MDEV-12416 OOM in create_virtual_tmp_table() makes the server crash

mysql-test/r/error_simulation.result

@@ -121,3 +121,12 @@ a
 2
 #cleanup
 DROP TABLE t1, pid_table;
+#
+# MDEV-12416 OOM in create_virtual_tmp_table() makes the server crash
+#
+CREATE FUNCTION f1(a INT) RETURNS INT RETURN a;
+SET SESSION debug_dbug="+d,simulate_create_virtual_tmp_table_out_of_memory";
+SELECT f1(1);
+Got one of the listed errors
+DROP FUNCTION f1;
+SET SESSION debug_dbug=DEFAULT;

mysql-test/t/error_simulation.test

@@ -148,3 +148,17 @@ SELECT a FROM t1 ORDER BY rand(1);
 
 --echo #cleanup
 DROP TABLE t1, pid_table;
+
+
+
+--echo #
+--echo # MDEV-12416 OOM in create_virtual_tmp_table() makes the server crash
+--echo #
+
+CREATE FUNCTION f1(a INT) RETURNS INT RETURN a;
+SET SESSION debug_dbug="+d,simulate_create_virtual_tmp_table_out_of_memory";
+# May fail with either ER_OUT_OF_RESOURCES or EE_OUTOFMEMORY
+--error ER_OUT_OF_RESOURCES, 5
+SELECT f1(1);
+DROP FUNCTION f1;
+SET SESSION debug_dbug=DEFAULT;

sql/sql_select.h

@@ -1993,7 +1993,7 @@ class Virtual_tmp_table: public TABLE
 
     This is needed to avoid memory leaks, as some fields can be BLOB
     variants and thus can have String onboard. Strings must be destructed
-    as they store data not the heap (not on MEM_ROOT).
+    as they store data on the heap (not on MEM_ROOT).
   */
   void destruct_fields()
   {
@@ -2025,6 +2025,7 @@ public:
     @param thd         - Current thread.
   */
   static void *operator new(size_t size, THD *thd) throw();
+  static void operator delete(void *ptr, size_t size) { TRASH(ptr, size); }
 
   Virtual_tmp_table(THD *thd)
   {
@@ -2035,7 +2036,8 @@ public:
 
   ~Virtual_tmp_table()
   {
-    destruct_fields();
+    if (s)
+      destruct_fields();
   }
 
   /**
@@ -2120,6 +2122,17 @@ create_virtual_tmp_table(THD *thd, List<Column_definition> &field_list)
   Virtual_tmp_table *table;
   if (!(table= new(thd) Virtual_tmp_table(thd)))
     return NULL;
+
+  /*
+    If "simulate_create_virtual_tmp_table_out_of_memory" debug option
+    is enabled, we now enable "simulate_out_of_memory". This effectively
+    makes table->init() fail on OOM inside multi_alloc_root().
+    This is done to test that ~Virtual_tmp_table() called from the "delete"
+    below correcly handles OOM.
+  */
+  DBUG_EXECUTE_IF("simulate_create_virtual_tmp_table_out_of_memory",
+                  DBUG_SET("+d,simulate_out_of_memory"););
+
   if (table->init(field_list.elements) ||
       table->add(field_list) ||
       table->open())