Merge 10.2 to 10.3

Temporarily disable main.cte_recursive due to hang in
an added test related to MDEV-15575.

mysql-test/main/cte_recursive.result

@@ -3070,6 +3070,142 @@ SELECT * FROM cte;
 2
 3
 #
+# MDEV-15575: using recursive cte with big_tables enabled
+#
+set big_tables=1;
+with recursive qn as
+(select 123 as a union all select 1+a from qn where a<130)
+select * from qn;
+a
+123
+124
+125
+126
+127
+128
+129
+130
+set big_tables=default;
+#
+# MDEV-15571: using recursive cte with big_tables enabled
+#
+set big_tables=1;
+with recursive qn as
+(
+select 1 as a from dual
+union all
+select a*2000 from qn where a<10000000000000000000
+)
+select * from qn;
+ERROR 22003: BIGINT value is out of range in '`qn`.`a` * 2000'
+set big_tables=default;
+#
+# MDEV-15556: using recursive cte with big_tables enabled
+#             when recursive tables are accessed by key
+#
+SET big_tables=1;
+CREATE TABLE t1 (id int, name char(10), leftpar int, rightpar int);
+INSERT INTO t1 VALUES
+(1, "A", 2, 3), (2, "LA", 4, 5), (4, "LLA", 6, 7),
+(6, "LLLA", NULL, NULL), (7, "RLLA", NULL, NULL), (5, "RLA", 8, 9),
+(8, "LRLA", NULL, NULL), (9, "RRLA", NULL, NULL), (3, "RA", 10, 11),
+(10, "LRA", 12, 13), (11, "RRA", 14, 15), (15, "RRRA", NULL, NULL),
+(16, "B", 17, 18), (17, "LB", NULL, NULL), (18, "RB", NULL, NULL);
+CREATE TABLE t2 SELECT * FROM t1 ORDER BY rand();
+WITH RECURSIVE tree_of_a AS
+(SELECT *, cast(id AS char(200)) AS path FROM t2 WHERE name="A"
+    UNION ALL
+SELECT t2.*, concat(tree_of_a.path,",",t2.id)
+FROM t2 JOIN tree_of_a ON t2.id=tree_of_a.leftpar
+UNION ALL
+SELECT t2.*, concat(tree_of_a.path,",",t2.id)
+FROM t2 JOIN tree_of_a ON t2.id=tree_of_a.rightpar)
+SELECT * FROM tree_of_a
+ORDER BY path;
+id	name	leftpar	rightpar	path
+1	A	2	3	1
+2	LA	4	5	1,2
+4	LLA	6	7	1,2,4
+6	LLLA	NULL	NULL	1,2,4,6
+7	RLLA	NULL	NULL	1,2,4,7
+5	RLA	8	9	1,2,5
+8	LRLA	NULL	NULL	1,2,5,8
+9	RRLA	NULL	NULL	1,2,5,9
+3	RA	10	11	1,3
+10	LRA	12	13	1,3,10
+11	RRA	14	15	1,3,11
+15	RRRA	NULL	NULL	1,3,11,15
+EXPLAIN WITH RECURSIVE tree_of_a AS
+(SELECT *, cast(id AS char(200)) AS path FROM t2 WHERE name="A"
+    UNION ALL
+SELECT t2.*, concat(tree_of_a.path,",",t2.id)
+FROM t2 JOIN tree_of_a ON t2.id=tree_of_a.leftpar
+UNION ALL
+SELECT t2.*, concat(tree_of_a.path,",",t2.id)
+FROM t2 JOIN tree_of_a ON t2.id=tree_of_a.rightpar)
+SELECT * FROM tree_of_a
+ORDER BY path;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	<derived2>	ALL	NULL	NULL	NULL	NULL	15	Using filesort
+2	DERIVED	t2	ALL	NULL	NULL	NULL	NULL	15	Using where
+3	RECURSIVE UNION	t2	ALL	NULL	NULL	NULL	NULL	15	Using where
+3	RECURSIVE UNION	<derived2>	ref	key0	key0	5	test.t2.id	2	
+4	RECURSIVE UNION	t2	ALL	NULL	NULL	NULL	NULL	15	Using where
+4	RECURSIVE UNION	<derived2>	ref	key0	key0	5	test.t2.id	2	
+NULL	UNION RESULT	<union2,3,4>	ALL	NULL	NULL	NULL	NULL	NULL	
+DROP TABLE t1,t2;
+SET big_tables=0;
+#
+# MDEV-15840: recursive tables are accessed by key
+#             (the same problem as for MDEV-15556)
+#
+CREATE TABLE t1 (p1 text,k2 int, p2 text, k1 int);
+INSERT INTO t1 select seq, seq, seq, seq from seq_1_to_1000;
+CREATE PROCEDURE getNums()
+BEGIN
+WITH RECURSIVE cte as
+(
+SELECT * FROM t1
+UNION
+SELECT c.* FROM t1 c JOIN cte p ON c.p1 = p.p2 AND c.k2 = p.k1
+)
+SELECT * FROM cte LIMIT 10;
+END |
+call getNums();
+p1	k2	p2	k1
+1	1	1	1
+2	2	2	2
+3	3	3	3
+4	4	4	4
+5	5	5	5
+6	6	6	6
+7	7	7	7
+8	8	8	8
+9	9	9	9
+10	10	10	10
+DROP PROCEDURE getNums;
+DROP TABLE t1;
+#
+# MDEV-15894: aggregate/winfow functions in non-recorsive part
+#
+create table t1(b int);
+insert into t1 values(10),(20),(10);
+with recursive qn as
+(select max(b) as a from t1 union
+select a from qn)
+select * from qn;
+a
+20
+with recursive qn as
+(select rank() over (order by b) as a from t1 union
+select a from qn)
+select * from qn;
+a
+1
+3
+drop table t1;
+# Start of 10.3 tests
+#
 # MDEV-14217 [db crash] Recursive CTE when SELECT includes new field
 #
 CREATE TEMPORARY TABLE a_tbl (

mysql-test/main/cte_recursive.test

@@ -2098,6 +2098,119 @@ WITH RECURSIVE cte AS
 	SELECT @c:=@c+1 FROM cte WHERE @c<3)
 SELECT * FROM cte;
 
+--echo #
+--echo # MDEV-15575: using recursive cte with big_tables enabled
+--echo #
+
+set big_tables=1;
+
+with recursive qn as
+(select 123 as a union all select 1+a from qn where a<130)
+select * from qn;
+
+set big_tables=default;
+
+--echo #
+--echo # MDEV-15571: using recursive cte with big_tables enabled
+--echo #
+
+set big_tables=1;
+
+--error ER_DATA_OUT_OF_RANGE
+with recursive qn as
+(
+  select 1 as a from dual
+  union all
+  select a*2000 from qn where a<10000000000000000000
+)
+select * from qn;
+
+set big_tables=default;
+
+--echo #
+--echo # MDEV-15556: using recursive cte with big_tables enabled
+--echo #             when recursive tables are accessed by key
+--echo #
+
+SET big_tables=1;
+
+CREATE TABLE t1 (id int, name char(10), leftpar int, rightpar int);
+INSERT INTO t1 VALUES
+  (1, "A", 2, 3), (2, "LA", 4, 5), (4, "LLA", 6, 7),
+  (6, "LLLA", NULL, NULL), (7, "RLLA", NULL, NULL), (5, "RLA", 8, 9),
+  (8, "LRLA", NULL, NULL), (9, "RRLA", NULL, NULL), (3, "RA", 10, 11),
+  (10, "LRA", 12, 13), (11, "RRA", 14, 15), (15, "RRRA", NULL, NULL),
+  (16, "B", 17, 18), (17, "LB", NULL, NULL), (18, "RB", NULL, NULL);
+
+CREATE TABLE t2 SELECT * FROM t1 ORDER BY rand();
+
+let $q=
+WITH RECURSIVE tree_of_a AS
+  (SELECT *, cast(id AS char(200)) AS path FROM t2 WHERE name="A"
+    UNION ALL
+  SELECT t2.*, concat(tree_of_a.path,",",t2.id)
+    FROM t2 JOIN tree_of_a ON t2.id=tree_of_a.leftpar
+  UNION ALL
+  SELECT t2.*, concat(tree_of_a.path,",",t2.id)
+    FROM t2 JOIN tree_of_a ON t2.id=tree_of_a.rightpar)
+SELECT * FROM tree_of_a
+ORDER BY path;
+
+eval $q;
+eval EXPLAIN $q;
+
+DROP TABLE t1,t2;
+
+SET big_tables=0;
+
+--echo #
+--echo # MDEV-15840: recursive tables are accessed by key
+--echo #             (the same problem as for MDEV-15556)
+--echo #
+
+--source include/have_sequence.inc
+
+CREATE TABLE t1 (p1 text,k2 int, p2 text, k1 int);
+INSERT INTO t1 select seq, seq, seq, seq from seq_1_to_1000;
+
+DELIMITER |;
+CREATE PROCEDURE getNums()
+BEGIN
+WITH RECURSIVE cte as
+(
+  SELECT * FROM t1
+  UNION
+  SELECT c.* FROM t1 c JOIN cte p ON c.p1 = p.p2 AND c.k2 = p.k1
+)
+SELECT * FROM cte LIMIT 10;
+END |
+
+DELIMITER ;|
+call getNums();
+
+DROP PROCEDURE getNums;
+DROP TABLE t1;
+
+--echo #
+--echo # MDEV-15894: aggregate/winfow functions in non-recorsive part
+--echo #
+
+create table t1(b int);
+insert into t1 values(10),(20),(10);
+
+with recursive qn as
+  (select max(b) as a from t1 union
+   select a from qn)
+select * from qn;
+
+with recursive qn as
+  (select rank() over (order by b) as a from t1 union
+   select a from qn)
+select * from qn;
+
+drop table t1;
+
+--echo # Start of 10.3 tests
 
 --echo #
 --echo # MDEV-14217 [db crash] Recursive CTE when SELECT includes new field

mysql-test/main/disabled.def

@@ -21,3 +21,4 @@ innodb-wl5522-debug-zip  : broken upstream
 innodb_bug12902967       : broken upstream
 file_contents            : MDEV-6526 these files are not installed anymore
 max_statement_time       : cannot possibly work, depends on timing
+cte_recursive            : Merge problem (MDEV-15575)

mysql-test/mysql-test-run.pl

@@ -3975,32 +3975,32 @@ sub run_testcase ($$) {
   }
 
   my $test= $tinfo->{suite}->start_test($tinfo);
-  # Set only when we have to keep waiting after expectedly died server
+  # Set to a list of processes we have to keep waiting (expectedly died servers)
-  my $keep_waiting_proc = 0;
+  my %keep_waiting_proc = ();
   my $print_timeout= start_timer($print_freq * 60);
 
   while (1)
   {
     my $proc;
-    if ($keep_waiting_proc)
+    if (scalar(keys(%keep_waiting_proc)) > 0)
     {
       # Any other process exited?
       $proc = My::SafeProcess->check_any();
       if ($proc)
       {
 	mtr_verbose ("Found exited process $proc");
+	$keep_waiting_proc{$proc} = 1;
       }
       else
       {
-	$proc = $keep_waiting_proc;
 	# Also check if timer has expired, if so cancel waiting
 	if ( has_expired($test_timeout) )
 	{
-	  $keep_waiting_proc = 0;
+	  %keep_waiting_proc = ();
 	}
       }
     }
-    if (! $keep_waiting_proc)
+    if (scalar(keys(%keep_waiting_proc)) == 0)
     {
       if($test_timeout > $print_timeout)
       {
@@ -4017,19 +4017,19 @@ sub run_testcase ($$) {
       else
       {
          $proc= My::SafeProcess->wait_any_timeout($test_timeout);
+         $keep_waiting_proc{$proc} = 1;
       }
     }
 
-    # Will be restored if we need to keep waiting
+    if (scalar(keys(%keep_waiting_proc)) == 0)
-    $keep_waiting_proc = 0;
-
-    unless ( defined $proc )
     {
       mtr_error("wait_any failed");
     }
-    mtr_verbose("Got $proc");
+    mtr_verbose("Got " . join(",", keys(%keep_waiting_proc)));
 
     mark_time_used('test');
+    my $expected_exit = 1;
+    foreach $proc (keys(%keep_waiting_proc)) {
       # ----------------------------------------------------
       # Was it the test program that exited
       # ----------------------------------------------------
@@ -4143,11 +4143,22 @@ sub run_testcase ($$) {
       # Check if it was an expected crash
       # ----------------------------------------------------
       my $check_crash = check_expected_crash_and_restart($proc);
-    if ($check_crash)
+      if ($check_crash == 0) # unexpected exit/crash of $proc
       {
-      # Keep waiting if it returned 2, if 1 don't wait or stop waiting.
+        $expected_exit = 0;
-      $keep_waiting_proc = 0 if $check_crash == 1;
+        last;
-      $keep_waiting_proc = $proc if $check_crash == 2;
+      }
+      elsif ($check_crash == 1) # $proc was started again by check_expected_crash_and_restart()
+      {
+        delete $keep_waiting_proc{$proc};
+      }
+      elsif ($check_crash == 2) # we must keep waiting
+      {
+        # do nothing
+      }
+    }
+
+    if ($expected_exit) {
       next;
     }
 

mysql-test/suite/innodb/r/undo_log.result

@@ -140,3 +140,16 @@ CHECK TABLE test_tab;
 Table	Op	Msg_type	Msg_text
 test.test_tab	check	status	OK
 DROP TABLE test_tab;
+SET @saved_frequency = @@GLOBAL.innodb_purge_rseg_truncate_frequency;
+SET GLOBAL innodb_purge_rseg_truncate_frequency = 1;
+CREATE TEMPORARY TABLE t2(i INT)ENGINE=InnoDB;
+CREATE TABLE t1(i TEXT NOT NULL) ENGINE=INNODB;
+BEGIN;
+INSERT t1 SET i=REPEAT('1234567890',840);
+UPDATE t1 SET i='';
+INSERT INTO t2 VALUES(2);
+ROLLBACK;
+InnoDB		0 transactions not purged
+DROP TABLE t1;
+DROP TABLE t2;
+SET GLOBAL innodb_purge_rseg_truncate_frequency = @saved_frequency;

mysql-test/suite/innodb/t/undo_log.test

@@ -137,3 +137,17 @@ ROLLBACK;
 SELECT COUNT(*) FROM test_tab;
 CHECK TABLE test_tab;
 DROP TABLE test_tab;
+
+SET @saved_frequency = @@GLOBAL.innodb_purge_rseg_truncate_frequency;
+SET GLOBAL innodb_purge_rseg_truncate_frequency = 1;
+CREATE TEMPORARY TABLE t2(i INT)ENGINE=InnoDB;
+CREATE TABLE t1(i TEXT NOT NULL) ENGINE=INNODB;
+BEGIN;
+INSERT t1 SET i=REPEAT('1234567890',840);
+UPDATE t1 SET i='';
+INSERT INTO t2 VALUES(2);
+ROLLBACK;
+--source include/wait_all_purged.inc
+DROP TABLE t1;
+DROP TABLE t2;
+SET GLOBAL innodb_purge_rseg_truncate_frequency = @saved_frequency;

mysql-test/suite/innodb_zip/r/cmp_per_index.result

@@ -72,8 +72,17 @@ index_name	PRIMARY
 compress_ops	65
 compress_ops_ok	65
 uncompress_ops	0
+SHOW CREATE TABLE t;
+Table	t
+Create Table	CREATE TABLE `t` (
+  `a` int(11) NOT NULL,
+  `b` varchar(512) DEFAULT NULL,
+  `c` varchar(16) DEFAULT NULL,
+  PRIMARY KEY (`a`),
+  KEY `b` (`b`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1 KEY_BLOCK_SIZE=2
 SET GLOBAL innodb_cmp_per_index_enabled=ON;
-SELECT COUNT(*) FROM t;
+SELECT COUNT(*) FROM t IGNORE INDEX(b);
 COUNT(*)	128
 SELECT
 database_name,
@@ -87,15 +96,9 @@ FROM information_schema.innodb_cmp_per_index
 ORDER BY 1, 2, 3;
 database_name	test
 table_name	t
-index_name	b
-compress_ops	0
-compress_ops_ok	0
-uncompress_ops	6
-database_name	test
-table_name	t
 index_name	PRIMARY
 compress_ops	0
 compress_ops_ok	0
-uncompress_ops	5
+uncompress_ops	4
 DROP TABLE t;
 SET GLOBAL innodb_cmp_per_index_enabled=default;

mysql-test/suite/innodb_zip/t/cmp_per_index.test

@@ -102,9 +102,11 @@ ORDER BY 1, 2, 3;
 
 -- source include/restart_mysqld.inc
 
+SHOW CREATE TABLE t;
+
 SET GLOBAL innodb_cmp_per_index_enabled=ON;
 
-SELECT COUNT(*) FROM t;
+SELECT COUNT(*) FROM t IGNORE INDEX(b);
 
 SELECT
 database_name,

sql/sql_cte.cc

@@ -1189,7 +1189,7 @@ bool st_select_lex::check_unrestricted_recursive(bool only_standard_compliant)
 
 
   /* Check conditions 3-4 for restricted specification*/
-  if (with_sum_func ||
+  if ((with_sum_func && !with_elem->is_anchor(this)) ||
       (with_elem->contains_sq_with_recursive_reference()))
     with_elem->get_owner()->add_unrestricted(
                               with_elem->get_mutually_recursive());
@@ -1414,7 +1414,7 @@ bool With_element::instantiate_tmp_tables()
   {
     if (!rec_table->is_created() &&
         instantiate_tmp_table(rec_table,
-                              rec_result->tmp_table_param.keyinfo,
+                              rec_table->s->key_info,
                               rec_result->tmp_table_param.start_recinfo,
                               &rec_result->tmp_table_param.recinfo,
                               0))

sql/sql_union.cc

@@ -413,19 +413,13 @@ select_union_recursive::create_result_table(THD *thd_arg,
   if (! (incr_table= create_tmp_table(thd_arg, &tmp_table_param, *column_types,
                                       (ORDER*) 0, false, 1,
                                       options, HA_POS_ERROR, &empty_clex_str,
-                                      !create_table, keep_row_order)))
+                                      true, keep_row_order)))
     return true;
 
   incr_table->keys_in_use_for_query.clear_all();
   for (uint i=0; i < table->s->fields; i++)
     incr_table->field[i]->flags &= ~PART_KEY_FLAG;
 
-  if (create_table)
-  {
-    incr_table->file->extra(HA_EXTRA_WRITE_CACHE);
-    incr_table->file->extra(HA_EXTRA_IGNORE_DUP_KEY);
-  }
-
   TABLE *rec_table= 0;
   if (! (rec_table= create_tmp_table(thd_arg, &tmp_table_param, *column_types,
                                      (ORDER*) 0, false, 1,
@@ -468,9 +462,12 @@ void select_union_recursive::cleanup()
   }
 
   if (incr_table)
+  {
+    if (incr_table->is_created())
     {
       incr_table->file->extra(HA_EXTRA_RESET_STATE);
       incr_table->file->ha_delete_all_rows();
+    }
     free_tmp_table(thd, incr_table);
   }
 
@@ -1659,15 +1656,23 @@ bool st_select_lex_unit::exec_recursive()
   if (!was_executed)
     save_union_explain(thd->lex->explain);
 
-  if ((saved_error= incr_table->file->ha_delete_all_rows()))
-    goto err;
-
   if (with_element->level == 0)
   {
+    if (!incr_table->is_created() &&
+        instantiate_tmp_table(incr_table,
+                              tmp_table_param->keyinfo,
+                              tmp_table_param->start_recinfo,
+                              &tmp_table_param->recinfo,
+                              0))
+      DBUG_RETURN(1);
+    incr_table->file->extra(HA_EXTRA_WRITE_CACHE);
+    incr_table->file->extra(HA_EXTRA_IGNORE_DUP_KEY);
     start= first_select();
     if (with_element->with_anchor)
       end= with_element->first_recursive;
   }
+  else if ((saved_error= incr_table->file->ha_delete_all_rows()))
+    goto err;
 
   for (st_select_lex *sl= start ; sl != end; sl= sl->next_select())
   {

storage/connect/plugutil.cpp

@@ -559,7 +559,7 @@ void *PlugSubAlloc(PGLOBAL g, void *memp, size_t size)
     if (trace(1))
       htrc("PlugSubAlloc: %s\n", g->Message);
 
-    abort();
+    throw 1234;
     } /* endif size OS32 code */
 
   /*********************************************************************/

storage/innobase/os/os0file.cc

@@ -866,8 +866,10 @@ os_file_get_block_size(
 		0, OPEN_EXISTING, 0, 0);
 
 	if (volume_handle == INVALID_HANDLE_VALUE) {
+		if (GetLastError() != ERROR_ACCESS_DENIED) {
 			os_file_handle_error_no_exit(volume,
 				"CreateFile()", FALSE);
+		}
 		goto end;
 	}
 
@@ -889,16 +891,7 @@ os_file_get_block_size(
 
 	if (!result) {
 		DWORD err = GetLastError();
-		if (err == ERROR_INVALID_FUNCTION || err == ERROR_NOT_SUPPORTED) {
+		if (err != ERROR_INVALID_FUNCTION && err != ERROR_NOT_SUPPORTED) {
-			// Don't report error, it is driver's fault, not ours or users.
-			// We handle this with fallback. Report wit info message, just once.
-			static bool write_info = true;
-			if (write_info) {
-				ib::info() << "DeviceIoControl(IOCTL_STORAGE_QUERY_PROPERTY)"
-					<< " unsupported on volume " << volume;
-				write_info = false;
-			}
-		} else {
 				os_file_handle_error_no_exit(volume,
 					"DeviceIoControl(IOCTL_STORAGE_QUERY_PROPERTY)", FALSE);
 		}

storage/innobase/trx/trx0roll.cc

@@ -979,7 +979,7 @@ trx_roll_pop_top_rec_of_trx(trx_t* trx, roll_ptr_t* roll_ptr, mem_heap_t* heap)
 		trx_roll_try_truncate(trx);
 	}
 
-	trx_undo_t*	undo;
+	trx_undo_t*	undo	= NULL;
 	trx_undo_t*	insert	= trx->rsegs.m_redo.old_insert;
 	trx_undo_t*	update	= trx->rsegs.m_redo.undo;
 	trx_undo_t*	temp	= trx->rsegs.m_noredo.undo;
@@ -994,17 +994,26 @@ trx_roll_pop_top_rec_of_trx(trx_t* trx, roll_ptr_t* roll_ptr, mem_heap_t* heap)
 
 	if (UNIV_LIKELY_NULL(insert)
 	    && !insert->empty && limit <= insert->top_undo_no) {
-		if (update && !update->empty
-		    && update->top_undo_no > insert->top_undo_no) {
-			undo = update;
-		} else {
 		undo = insert;
 	}
-	} else if (update && !update->empty && limit <= update->top_undo_no) {
+
+	if (update && !update->empty && update->top_undo_no >= limit) {
+		if (!undo) {
 			undo = update;
-	} else if (temp && !temp->empty && limit <= temp->top_undo_no) {
+		} else if (undo->top_undo_no < update->top_undo_no) {
+			undo = update;
+		}
+	}
+
+	if (temp && !temp->empty && temp->top_undo_no >= limit) {
+		if (!undo) {
 			undo = temp;
-	} else {
+		} else if (undo->top_undo_no < temp->top_undo_no) {
+			undo = temp;
+		}
+	}
+
+	if (undo == NULL) {
 		trx_roll_try_truncate(trx);
 		/* Mark any ROLLBACK TO SAVEPOINT completed, so that
 		if the transaction object is committed and reused

storage/maria/ma_delete_all.c

@@ -135,6 +135,9 @@ int maria_delete_all_rows(MARIA_HA *info)
       goto err;
   }
 
+  if (info->opt_flag & WRITE_CACHE_USED)
+    reinit_io_cache(&info->rec_cache, WRITE_CACHE, 0, 1, 1);
+
   _ma_writeinfo(info, WRITEINFO_UPDATE_KEYFILE);
 #ifdef HAVE_MMAP
   /* Map again */

storage/myisam/mi_delete_all.c

@@ -62,6 +62,10 @@ int mi_delete_all_rows(MI_INFO *info)
   if (mysql_file_chsize(info->dfile, 0, 0, MYF(MY_WME)) ||
       mysql_file_chsize(share->kfile, share->base.keystart, 0, MYF(MY_WME)))
     goto err;
+
+  if (info->opt_flag & WRITE_CACHE_USED)
+    reinit_io_cache(&info->rec_cache, WRITE_CACHE, 0, 1, 1);
+
   (void) _mi_writeinfo(info,WRITEINFO_UPDATE_KEYFILE);
   DBUG_RETURN(0);
 

storage/rocksdb/ha_rocksdb.cc

@@ -1772,6 +1772,17 @@ protected:
   bool m_is_delayed_snapshot = false;
   bool m_is_two_phase = false;
 
+private:
+  /* Number of RockDB savepoints taken */
+  int m_n_savepoints;
+  /*
+    Number of write operations this transaction had when we took the last
+    savepoint (the idea is not to take another savepoint if we haven't made
+    any changes)
+  */
+  ulonglong m_writes_at_last_savepoint;
+
+protected:
   THD *m_thd = nullptr;
 
   rocksdb::ReadOptions m_read_opts;
@@ -1800,6 +1811,14 @@ protected:
   get_iterator(const rocksdb::ReadOptions &options,
                rocksdb::ColumnFamilyHandle *column_family) = 0;
  
+protected:
+  /*
+    The following two are helper functions to be overloaded by child classes.
+    They should provide RocksDB's savepoint semantics.
+  */
+  virtual void do_set_savepoint() = 0;
+  virtual void do_rollback_to_savepoint() = 0;
+
 public:
   const char *m_mysql_log_file_name;
   my_off_t m_mysql_log_offset;
@@ -2173,6 +2192,50 @@ public:
   virtual bool is_tx_started() const = 0;
   virtual void start_tx() = 0;
   virtual void start_stmt() = 0;
+
+  void set_initial_savepoint() {
+    /*
+      Set the initial savepoint. If the first statement in the transaction
+      fails, we need something to roll back to, without rolling back the
+      entire transaction.
+    */
+    do_set_savepoint();
+    m_n_savepoints= 1;
+    m_writes_at_last_savepoint= m_write_count;
+  }
+
+  /*
+    Called when a "top-level" statement inside a transaction completes
+    successfully and its changes become part of the transaction's changes.
+  */
+  void make_stmt_savepoint_permanent() {
+
+    // Take another RocksDB savepoint only if we had changes since the last
+    // one. This is very important for long transactions doing lots of
+    // SELECTs.
+    if (m_writes_at_last_savepoint != m_write_count)
+    {
+      do_set_savepoint();
+      m_writes_at_last_savepoint= m_write_count;
+      m_n_savepoints++;
+    }
+  }
+
+
+  /*
+    Rollback to the savepoint we've set before the last statement
+  */
+  void rollback_to_stmt_savepoint() {
+    if (m_writes_at_last_savepoint != m_write_count) {
+      do_rollback_to_savepoint();
+      if (!--m_n_savepoints) {
+        do_set_savepoint();
+        m_n_savepoints= 1;
+      }
+      m_writes_at_last_savepoint= m_write_count;
+    }
+  }
+
   virtual void rollback_stmt() = 0;
 
   void set_tx_failed(bool failed_arg) { m_is_tx_failed = failed_arg; }
@@ -2462,9 +2525,20 @@ public:
 
     m_read_opts = rocksdb::ReadOptions();
 
+    set_initial_savepoint();
+
     m_ddl_transaction = false;
   }
 
+  /* Implementations of do_*savepoint based on rocksdB::Transaction savepoints */
+  void do_set_savepoint() override {
+    m_rocksdb_tx->SetSavePoint();
+  }
+
+  void do_rollback_to_savepoint() override {
+    m_rocksdb_tx->RollbackToSavePoint();
+  }
+
   /*
     Start a statement inside a multi-statement transaction.
 
@@ -2477,7 +2551,6 @@ public:
   void start_stmt() override {
     // Set the snapshot to delayed acquisition (SetSnapshotOnNextOperation)
     acquire_snapshot(false);
-    m_rocksdb_tx->SetSavePoint();
   }
 
   /*
@@ -2488,7 +2561,7 @@ public:
     /* TODO: here we must release the locks taken since the start_stmt() call */
     if (m_rocksdb_tx) {
       const rocksdb::Snapshot *const org_snapshot = m_rocksdb_tx->GetSnapshot();
-      m_rocksdb_tx->RollbackToSavePoint();
+      rollback_to_stmt_savepoint();
 
       const rocksdb::Snapshot *const cur_snapshot = m_rocksdb_tx->GetSnapshot();
       if (org_snapshot != cur_snapshot) {
@@ -2565,6 +2638,16 @@ private:
     return res;
   }
 
+protected:
+  /* Implementations of do_*savepoint based on rocksdB::WriteBatch savepoints */
+  void do_set_savepoint() override {
+    m_batch->SetSavePoint();
+  }
+
+  void do_rollback_to_savepoint() override {
+    m_batch->RollbackToSavePoint();
+  }
+
 public:
   bool is_writebatch_trx() const override { return true; }
 
@@ -2670,13 +2753,15 @@ public:
     write_opts.disableWAL = THDVAR(m_thd, write_disable_wal);
     write_opts.ignore_missing_column_families =
         THDVAR(m_thd, write_ignore_missing_column_families);
+
+    set_initial_savepoint();
   }
 
-  void start_stmt() override { m_batch->SetSavePoint(); }
+  void start_stmt() override {}
 
   void rollback_stmt() override {
     if (m_batch)
-      m_batch->RollbackToSavePoint();
+      rollback_to_stmt_savepoint();
   }
 
   explicit Rdb_writebatch_impl(THD *const thd)
@@ -2922,6 +3007,8 @@ static int rocksdb_prepare(handlerton* hton, THD* thd, bool prepare_tx)
 
     DEBUG_SYNC(thd, "rocksdb.prepared");
   }
+  else
+    tx->make_stmt_savepoint_permanent();
   return HA_EXIT_SUCCESS;
 }
 
@@ -3172,11 +3259,8 @@ static int rocksdb_commit(handlerton* hton, THD* thd, bool commit_tx)
     } else {
       /*
         We get here when committing a statement within a transaction.
-
-        We don't need to do anything here. tx->start_stmt() will notify
-        Rdb_transaction_impl that another statement has started.
       */
-      tx->set_tx_failed(false);
+      tx->make_stmt_savepoint_permanent();
     }
 
     if (my_core::thd_tx_isolation(thd) <= ISO_READ_COMMITTED) {
@@ -10064,8 +10148,9 @@ int ha_rocksdb::external_lock(THD *const thd, int lock_type) {
   }
 
   if (lock_type == F_UNLCK) {
-    Rdb_transaction *const tx = get_or_create_tx(thd);
+    Rdb_transaction *const tx = get_tx_from_thd(thd);
 
+    if (tx) {
       tx->io_perf_end_and_record(&m_io_perf);
       tx->m_n_mysql_tables_in_use--;
       if (tx->m_n_mysql_tables_in_use == 0 &&
@@ -10082,6 +10167,7 @@ int ha_rocksdb::external_lock(THD *const thd, int lock_type) {
           res = HA_ERR_INTERNAL_ERROR;
         }
       }
+    }
   } else {
     if (my_core::thd_tx_isolation(thd) < ISO_READ_COMMITTED ||
         my_core::thd_tx_isolation(thd) > ISO_REPEATABLE_READ) {

storage/rocksdb/mysql-test/rocksdb/r/transaction.result

@@ -934,3 +934,27 @@ value
 3
 rollback;
 drop table t1;
+#
+#  #802: MyRocks: Statement rollback doesnt work correctly for nested statements
+#
+create table t1 (a varchar(100)) engine=rocksdb;
+create table t2(a int) engine=rocksdb;
+insert into t2 values (1), (2);
+create table t3(a varchar(100)) engine=rocksdb;
+create function func() returns varchar(100) deterministic 
+begin
+insert into t3 values ('func-called');
+set @a= (select a from t2);
+return 'func-returned';
+end;//
+begin;
+insert into t1 values (func());
+ERROR 21000: Subquery returns more than 1 row
+select * from t1;
+a
+# The following must not produce 'func-called':
+select * from t3;
+a
+rollback;
+drop function func;
+drop table t1,t2,t3;

storage/rocksdb/mysql-test/rocksdb/t/transaction.test

@@ -103,3 +103,33 @@ update t1 set id=115 where id=3;
 rollback;
 
 drop table t1;
+
+--echo #
+--echo #  #802: MyRocks: Statement rollback doesnt work correctly for nested statements
+--echo #
+create table t1 (a varchar(100)) engine=rocksdb;
+create table t2(a int) engine=rocksdb;
+insert into t2 values (1), (2);
+
+create table t3(a varchar(100)) engine=rocksdb;
+
+delimiter //;
+create function func() returns varchar(100) deterministic 
+begin
+  insert into t3 values ('func-called');
+  set @a= (select a from t2);
+  return 'func-returned';
+end;//
+delimiter ;//
+
+begin;
+--error ER_SUBQUERY_NO_1_ROW 
+insert into t1 values (func());
+select * from t1;
+--echo # The following must not produce 'func-called':
+select * from t3;
+
+rollback;
+drop function func;
+drop table t1,t2,t3;
+