From 12bbf03cff20b5ed9aa0f3b81c391ffea6d1433f Mon Sep 17 00:00:00 2001
From: Michael Widenius <monty@askmonty.org>
Date: Tue, 11 Feb 2014 14:02:42 +0200
Subject: [PATCH] Fix for MDEV-5629: Failing assertion: state ==
 TRX_STATE_NOT_STARTED on concurrent CREATE OR REPLACE and transactional
 UPDATE

mysql-test/suite/innodb/r/row_lock.result:
  Test case for MDEV-5629
mysql-test/suite/innodb/t/row_lock.test:
  Test case for MDEV-5629
sql/filesort.cc:
  Don't call unlock_row() in case of errors
---
 mysql-test/suite/innodb/r/row_lock.result | 16 ++++++++
 mysql-test/suite/innodb/t/row_lock.test   | 48 +++++++++++++++++++++++
 sql/filesort.cc                           |  9 ++++-
 3 files changed, 71 insertions(+), 2 deletions(-)
 create mode 100644 mysql-test/suite/innodb/r/row_lock.result
 create mode 100644 mysql-test/suite/innodb/t/row_lock.test

diff --git a/mysql-test/suite/innodb/r/row_lock.result b/mysql-test/suite/innodb/r/row_lock.result
new file mode 100644
index 00000000000..c2fb7b90c3f
--- /dev/null
+++ b/mysql-test/suite/innodb/r/row_lock.result
@@ -0,0 +1,16 @@
+CREATE TABLE t1 (a INT, b INT) ENGINE=InnoDB;
+INSERT INTO t1 VALUES (1,1),(2,2);
+CREATE TABLE t2 (c INT, d INT) ENGINE=InnoDB;
+INSERT INTO t2 VALUES (1,1), (2,2);
+CREATE TABLE t3 (e INT) ENGINE=InnoDB;
+CREATE TABLE t4 ENGINE=InnoDB AS SELECT * FROM t2;
+BEGIN;
+UPDATE t1 SET a = 0 WHERE a = ( SELECT e FROM t3 );
+SET DEBUG_SYNC='srv_suspend_mysql_thread_enter SIGNAL waiting';
+UPDATE t4 SET d = 1 WHERE d in ( SELECT a FROM t1 ) ORDER BY c LIMIT 6;
+SET DEBUG_SYNC='now WAIT_FOR waiting';
+SET DEBUG_SYNC='RESET';
+UPDATE t4 SET d = 9;
+ERROR 40001: Deadlock found when trying to get lock; try restarting transaction
+commit;
+drop table t1,t2,t3,t4;
diff --git a/mysql-test/suite/innodb/t/row_lock.test b/mysql-test/suite/innodb/t/row_lock.test
new file mode 100644
index 00000000000..76ac4fec23f
--- /dev/null
+++ b/mysql-test/suite/innodb/t/row_lock.test
@@ -0,0 +1,48 @@
+#
+# Test of wrong call to unlock_row
+# MDEV-5629 Failing assertion: state == TRX_STATE_NOT_STARTED on
+# concurrent CREATE OR REPLACE and transactional UPDATE
+#
+
+--source include/have_innodb.inc
+--source include/have_debug_sync.inc
+
+CREATE TABLE t1 (a INT, b INT) ENGINE=InnoDB;
+INSERT INTO t1 VALUES (1,1),(2,2);
+
+CREATE TABLE t2 (c INT, d INT) ENGINE=InnoDB;
+INSERT INTO t2 VALUES (1,1), (2,2);
+
+CREATE TABLE t3 (e INT) ENGINE=InnoDB;
+CREATE TABLE t4 ENGINE=InnoDB AS SELECT * FROM t2;
+
+--connect (con11,localhost,root,,test)
+
+BEGIN;
+UPDATE t1 SET a = 0 WHERE a = ( SELECT e FROM t3 );
+
+--connect (con12,localhost,root,,test)
+
+SET DEBUG_SYNC='srv_suspend_mysql_thread_enter SIGNAL waiting';
+
+--send 
+UPDATE t4 SET d = 1 WHERE d in ( SELECT a FROM t1 ) ORDER BY c LIMIT 6;
+
+--connection con11
+
+# Wait for con12 to start waiting for con11
+
+SET DEBUG_SYNC='now WAIT_FOR waiting';
+SET DEBUG_SYNC='RESET';
+
+UPDATE t4 SET d = 9;
+
+--connection con12
+--error ER_LOCK_DEADLOCK
+--reap
+--connection con11
+commit;
+--connection default
+--disconnect con12
+--disconnect con11
+drop table t1,t2,t3,t4;
diff --git a/sql/filesort.cc b/sql/filesort.cc
index 1ee50ba134b..7b019022409 100644
--- a/sql/filesort.cc
+++ b/sql/filesort.cc
@@ -652,12 +652,17 @@ static ha_rows find_all_keys(SORTPARAM *param, SQL_SELECT *select,
       make_sortkey(param, next_sort_key, ref_pos);
       next_sort_key+= param->rec_length;
     }
-    else
-      file->unlock_row();
 
     /* It does not make sense to read more keys in case of a fatal error */
     if (thd->is_error())
       break;
+
+    /*
+      We need to this after checking the error as the transaction may have
+      rolled back in case of a deadlock
+    */
+    if (!write_record)
+      file->unlock_row();
   }
   if (!quick_select)
   {