MDEV-37052: JSON_TABLE stack overflow handling errors

main.json_debug_nonembedded_noasan fails because of stack
overrun on Debug + MSAN testing.

Since MDEV-33209 (09ea2dc788)
the the stack overflow errors are just injected instead of
frailer mechanisms to consume stack. These mechanims where
not carried forward to the JSON_TABLE functions where
the pattern was the same.

Related MDEV-34099 (cf1c381bb8) makes check_stack_overrun never fail
under Address Sanitizer (only).

The previous ALLOCATE_MEM_ON_STACK did in MemorySanitizer consume
memory, but check_stack_overrun did fail because its 16000 byte
safety margin was exceeded. The allocation of the 448 byte error
ER_STACK_OVERRUN_NEED_MORE is well within these bounds, however
under the safemalloc implementation, "backtrace" library call is called,
which does further allocation for every stack frame. This exceeds the stack.

Fixes:

JSON_TABLE functions that trigger on out of memory debug instrumentation
replaced with the mechanism from MDEV-33209.

The get_disallowed_table_deps_for_list in a non-Debug build returned
incorrectly 1, instead of -1 indicating the out of memory condition.

In json_table add_extra_deps never passed the out of memory error
condition to the caller and would continue to run in a loop, potentially
recursively under these near out of stack conditions.

The Memory, Undefined Behaviour, Address and Thread sanitizers provide
sufficient instrumentation and a backtrace so the safemalloc
functionality provides insufficent value with these. As such is
disabled under WITH_SAFEMALLOC=AUTO.

With all of thse corrected the main.json_debug_nonembedded_noasan no
longer needs its ASAN exclusion.

The JSON_TABLE tests in this test case was dropped in a merge from 10.6
so these tests are re-added.

CMakeLists.txt

@@ -342,7 +342,13 @@ SET(WITH_SAFEMALLOC "AUTO" CACHE STRING "Use safemalloc memory debugger. Will re
 
 IF(WITH_SAFEMALLOC MATCHES "ON")
   ADD_DEFINITIONS( -DSAFEMALLOC)
-ELSEIF(WITH_SAFEMALLOC MATCHES "AUTO" AND NOT WIN32 AND NOT WITH_VALGRIND)
+ELSEIF(WITH_SAFEMALLOC MATCHES "AUTO"
+       AND NOT WIN32
+       AND NOT WITH_VALGRIND
+       AND NOT WITH_ASAN
+       AND NOT WITH_UBSAN
+       AND NOT WITH_TSAN
+       AND NOT WITH_MSAN)
   SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -DSAFEMALLOC")
   SET(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -DSAFEMALLOC")
 ENDIF()

mysql-test/main/json_debug_nonembedded.result

@@ -3,6 +3,8 @@
 #
 SET @saved_dbug = @@debug_dbug;
 SET debug_dbug='+d,json_check_min_stack_requirement';
+SELECT * from JSON_TABLE('[{"a": 1, "b": [11,111]}, {"a": 2, "b": [22,222]}]', '$[*]' COLUMNS( a INT PATH '$.a')) as tt;
+ERROR HY000: Thread stack overrun:  'used bytes' used of a 'available' byte stack, and 'X' bytes needed. Consider increasing the thread_stack system variable.
 SET @json1= '{"key1":"val1"}';
 SET @json2= '{"key1":"val1"}';
 SELECT JSON_OVERLAPS(@json1, @json2);

mysql-test/main/json_debug_nonembedded.test

@@ -1,6 +1,5 @@
 -- source include/not_embedded.inc
 --source include/have_debug.inc
---source include/not_asan.inc
 
 --echo #
 --echo # MDEV-28762: recursive call of some json functions without stack control
@@ -9,6 +8,10 @@
 SET @saved_dbug = @@debug_dbug;
 SET debug_dbug='+d,json_check_min_stack_requirement';
 
+--replace_regex /overrun:  [0-9]* bytes used of a [0-9]* byte stack, and [0-9]* bytes needed/overrun:  'used bytes' used of a 'available' byte stack, and 'X' bytes needed/
+--error ER_STACK_OVERRUN_NEED_MORE
+SELECT * from JSON_TABLE('[{"a": 1, "b": [11,111]}, {"a": 2, "b": [22,222]}]', '$[*]' COLUMNS( a INT PATH '$.a')) as tt;
+
 SET @json1= '{"key1":"val1"}';
 SET @json2= '{"key1":"val1"}';
 

sql/item_jsonfunc.cc

@@ -21,7 +21,7 @@
 #include "sql_parse.h" // For check_stack_overrun
 
 #ifndef DBUG_OFF
-static int dbug_json_check_min_stack_requirement()
+int dbug_json_check_min_stack_requirement()
 {
   my_error(ER_STACK_OVERRUN_NEED_MORE, MYF(ME_FATAL),
            my_thread_stack_size, my_thread_stack_size, STACK_MIN_SIZE);

sql/json_table.cc

@@ -27,23 +27,11 @@
 #include "create_tmp_table.h"
 #include "sql_parse.h"
 
-#define HA_ERR_JSON_TABLE (HA_ERR_LAST+1)
+#ifndef DBUG_OFF
+int dbug_json_check_min_stack_requirement();
+#endif
 
-/*
+#define HA_ERR_JSON_TABLE (HA_ERR_LAST+1)
-  Allocating memory and *also* using it (reading and
-  writing from it) because some build instructions cause
-  compiler to optimize out stack_used_up. Since alloca()
-  here depends on stack_used_up, it doesnt get executed
-  correctly and causes json_debug_nonembedded to fail
-  ( --error ER_STACK_OVERRUN_NEED_MORE does not occur).
-*/
-#define ALLOCATE_MEM_ON_STACK(A) do \
-                              { \
-                                uchar *array= (uchar*)alloca(A); \
-                                array[0]= 1; \
-                                array[0]++; \
-                                array[0] ? array[0]++ : array[0]--; \
-                              } while(0)
 
 class table_function_handlerton
 {
@@ -119,13 +107,9 @@ int get_disallowed_table_deps_for_list(MEM_ROOT *mem_root,
   List_iterator<TABLE_LIST> li(*join_list);
 
   DBUG_EXECUTE_IF("json_check_min_stack_requirement",
-                  {
+                  return -dbug_json_check_min_stack_requirement(););
-                    long arbitrary_var;
-                    long stack_used_up= (available_stack_size(current_thd->thread_stack, &arbitrary_var));
-                    ALLOCATE_MEM_ON_STACK(my_thread_stack_size-stack_used_up-STACK_MIN_SIZE);
-                  });
   if (check_stack_overrun(current_thd, STACK_MIN_SIZE , NULL))
-    return 1;
+    return -1;
 
   while ((table= li++))
   {
@@ -1351,21 +1335,20 @@ void Table_function_json_table::fix_after_pullout(TABLE_LIST *sql_table,
 /*
   @brief
      Recursively make all tables in the join_list also depend on deps.
+
+  @return - boolean - true if error (out of memory).
 */
 
-static void add_extra_deps(List<TABLE_LIST> *join_list, table_map deps)
+static bool add_extra_deps(List<TABLE_LIST> *join_list, table_map deps)
 {
   TABLE_LIST *table;
   List_iterator<TABLE_LIST> li(*join_list);
 
   DBUG_EXECUTE_IF("json_check_min_stack_requirement",
-                  {
+                  dbug_json_check_min_stack_requirement(); return true;);
-                    long arbitrary_var;
-                    long stack_used_up= (available_stack_size(current_thd->thread_stack, &arbitrary_var));
-                    ALLOCATE_MEM_ON_STACK(my_thread_stack_size-stack_used_up-STACK_MIN_SIZE);
-                  });
   if (check_stack_overrun(current_thd, STACK_MIN_SIZE , NULL))
-    return;
+    return true;
+
   while ((table= li++))
   {
     table->dep_tables |= deps;
@@ -1373,9 +1356,11 @@ static void add_extra_deps(List<TABLE_LIST> *join_list, table_map deps)
     if ((nested_join= table->nested_join))
     {
        // set the deps inside, too
-       add_extra_deps(&nested_join->join_list, deps);
+       if (add_extra_deps(&nested_join->join_list, deps))
+         return true;
     }
   }
+  return false;
 }
 
 
@@ -1444,7 +1429,8 @@ static void add_extra_deps(List<TABLE_LIST> *join_list, table_map deps)
                        supply the JOIN's top-level table list.
   @param  nest_tables  Bitmap of all tables in the join list.
 
-  @return Bitmap of all outside references that tables in join_list have
+  @return Bitmap of all outside references that tables in join_list have,
+    or 0 on out of stack error.
 */
 
 table_map add_table_function_dependencies(List<TABLE_LIST> *join_list,
@@ -1455,11 +1441,7 @@ table_map add_table_function_dependencies(List<TABLE_LIST> *join_list,
   List_iterator<TABLE_LIST> li(*join_list);
 
   DBUG_EXECUTE_IF("json_check_min_stack_requirement",
-                  {
+                  if (dbug_json_check_min_stack_requirement()) return 0;);
-                    long arbitrary_var;
-                    long stack_used_up= (available_stack_size(current_thd->thread_stack, &arbitrary_var));
-                    ALLOCATE_MEM_ON_STACK(my_thread_stack_size-stack_used_up-STACK_MIN_SIZE);
-                  });
   if ((res=check_stack_overrun(current_thd, STACK_MIN_SIZE , NULL)))
     return res;
 
@@ -1481,7 +1463,10 @@ table_map add_table_function_dependencies(List<TABLE_LIST> *join_list,
   res= res & ~nest_tables & ~PSEUDO_TABLE_BITS;
   // Then, make all "peers" have them:
   if (res)
-    add_extra_deps(join_list,  res);
+  {
+    if (add_extra_deps(join_list,  res))
+      return 0;
+  }
 
   return res;
 }