Merge 10.2 into bb-10.2-ext

mysql-test/r/func_json.result

@@ -356,6 +356,12 @@ json_keys('foo')
 NULL
 Warnings:
 Warning	4038	Syntax error in JSON text in argument 1 to function 'json_keys' at position 1
+select json_keys('{"a":{"c":1, "d":2}, "b":2, "c":1, "a":3, "b":1, "c":2}');
+json_keys('{"a":{"c":1, "d":2}, "b":2, "c":1, "a":3, "b":1, "c":2}')
+["a", "b", "c"]
+select json_keys('{"c1": "value 1", "c1": "value 2"}');
+json_keys('{"c1": "value 1", "c1": "value 2"}')
+["c1"]
 SET @j = '["abc", [{"k": "10"}, "def"], {"x":"abc"}, {"y":"bcd"}]';
 select json_search(@j, 'one', 'abc');
 json_search(@j, 'one', 'abc')
@@ -648,6 +654,21 @@ NULL
 SELECT JSON_EXTRACT( '{"foo":"bar"}', '$[*]' );
 JSON_EXTRACT( '{"foo":"bar"}', '$[*]' )
 NULL
+select JSON_EXTRACT('{"name":"value"}', '$.name') = 'value';
+JSON_EXTRACT('{"name":"value"}', '$.name') = 'value'
+1
+select JSON_EXTRACT('{\"asdf\":true}', "$.\"asdf\"") = true;
+JSON_EXTRACT('{\"asdf\":true}', "$.\"asdf\"") = true
+1
+select JSON_EXTRACT('{\"asdf\":true}', "$.\"asdf\"") = false;
+JSON_EXTRACT('{\"asdf\":true}', "$.\"asdf\"") = false
+0
+select JSON_EXTRACT('{\"asdf\":true}', "$.\"asdf\"") = 1;
+JSON_EXTRACT('{\"asdf\":true}', "$.\"asdf\"") = 1
+1
+select JSON_EXTRACT('{\"input1\":\"\\u00f6\"}', '$.\"input1\"');
+JSON_EXTRACT('{\"input1\":\"\\u00f6\"}', '$.\"input1\"')
+"\u00f6"
 #
 # Start of 10.3 tests
 #

mysql-test/suite/encryption/r/innodb_encrypt_log.result

@@ -25,6 +25,7 @@ CREATE TEMPORARY TABLE t LIKE t0;
 INSERT INTO t VALUES
 (NULL,1,1,'private','secret'),(NULL,2,2,'sacred','success'),
 (NULL,3,3,'story','secure'),(NULL,4,4,'security','sacrament');
+SET GLOBAL innodb_change_buffering=none;
 SET GLOBAL innodb_flush_log_at_trx_commit=1;
 INSERT INTO t0
 SELECT NULL, t1.col_int, t1.col_int_key, t1.col_char, t1.col_char_key

mysql-test/suite/encryption/t/innodb_encrypt_log.test

@@ -32,6 +32,11 @@ INSERT INTO t VALUES
 (NULL,1,1,'private','secret'),(NULL,2,2,'sacred','success'),
 (NULL,3,3,'story','secure'),(NULL,4,4,'security','sacrament');
 
+# Prevent change buffering of key(col_char_key), so that
+# after the restart, the data ('secret','success','secure','sacrament')
+# cannot be emitted to the unencrypted redo log by change buffer merge.
+SET GLOBAL innodb_change_buffering=none;
+
 # Force a redo log flush at the next commit.
 SET GLOBAL innodb_flush_log_at_trx_commit=1;
 INSERT INTO t0

mysql-test/t/func_json.test

@@ -138,6 +138,11 @@ select json_keys('{"a":{"c":1, "d":2}, "b":2}');
 select json_keys('{"a":{"c":1, "d":2}, "b":2}', "$.a");
 select json_keys('{"a":{"c":1, "d":2}, "b":2}', "$.b");
 select json_keys('foo');
+#
+# mdev-12789 JSON_KEYS returns duplicate keys twice
+#
+select json_keys('{"a":{"c":1, "d":2}, "b":2, "c":1, "a":3, "b":1, "c":2}');
+select json_keys('{"c1": "value 1", "c1": "value 2"}');
 
 SET @j = '["abc", [{"k": "10"}, "def"], {"x":"abc"}, {"y":"bcd"}]';
 select json_search(@j, 'one', 'abc');
@@ -302,6 +307,16 @@ DROP TABLE t1;
 SELECT JSON_EXTRACT( '{"foo":"bar"}', '$[*].*' );
 SELECT JSON_EXTRACT( '{"foo":"bar"}', '$[*]' );
 
+#
+# MDEV-12604 Comparison of JSON_EXTRACT result differs with Mysql.
+#
+
+select JSON_EXTRACT('{"name":"value"}', '$.name') = 'value';
+select JSON_EXTRACT('{\"asdf\":true}', "$.\"asdf\"") = true;
+select JSON_EXTRACT('{\"asdf\":true}', "$.\"asdf\"") = false;
+select JSON_EXTRACT('{\"asdf\":true}', "$.\"asdf\"") = 1;
+select JSON_EXTRACT('{\"input1\":\"\\u00f6\"}', '$.\"input1\"');
+
 --echo #
 --echo # Start of 10.3 tests
 --echo #

sql/handler.cc

@@ -3356,6 +3356,12 @@ void handler::print_error(int error, myf errflag)
   DBUG_ENTER("handler::print_error");
   DBUG_PRINT("enter",("error: %d",error));
 
+  if (ha_thd()->transaction_rollback_request)
+  {
+    /* Ensure this becomes a true error */
+    errflag&= ~(ME_JUST_WARNING | ME_JUST_INFO);
+  }
+
   int textno= -1; // impossible value
   switch (error) {
   case EACCES:
@@ -3504,9 +3510,6 @@ void handler::print_error(int error, myf errflag)
   {
     String str, full_err_msg(ER_DEFAULT(ER_LOCK_DEADLOCK), system_charset_info);
 
-    /* cannot continue. the statement was already aborted in the engine */
-    SET_FATAL_ERROR;
-
     get_error_message(error, &str);
     full_err_msg.append(str);
     my_printf_error(ER_LOCK_DEADLOCK, "%s", errflag, full_err_msg.c_ptr_safe());

sql/item_cmpfunc.cc

@@ -545,6 +545,16 @@ bool Arg_comparator::set_cmp_func_string()
     if (owner->agg_arg_charsets_for_comparison(&m_compare_collation, a, b))
       return true;
   }
+
+  if ((*a)->is_json_type() ^ (*b)->is_json_type())
+  {
+    Item **j_item= (*a)->is_json_type() ? a : b;
+    Item *uf= new(thd->mem_root) Item_func_json_unquote(thd, *j_item);
+    if (!uf || uf->fix_fields(thd, &uf))
+      return 1;
+    *j_item= uf;
+  }
+
   a= cache_converted_constant(thd, a, &a_cache, compare_type_handler());
   b= cache_converted_constant(thd, b, &b_cache, compare_type_handler());
   return false;

sql/item_jsonfunc.cc

@@ -587,24 +587,40 @@ void Item_func_json_unquote::fix_length_and_dec()
 }
 
 
-String *Item_func_json_unquote::val_str(String *str)
+String *Item_func_json_unquote::read_json(json_engine_t *je)
 {
   String *js= args[0]->val_json(&tmp_s);
-  json_engine_t je;
-  int c_len;
 
   if ((null_value= args[0]->null_value))
-    return NULL;
+    return 0;
 
-  json_scan_start(&je, js->charset(),(const uchar *) js->ptr(),
+  json_scan_start(je, js->charset(),(const uchar *) js->ptr(),
                   (const uchar *) js->ptr() + js->length());
 
-  je.value_type= (enum json_value_types) -1; /* To report errors right. */
+  je->value_type= (enum json_value_types) -1; /* To report errors right. */
 
-  if (json_read_value(&je))
+  if (json_read_value(je))
     goto error;
 
-  if (je.value_type != JSON_VALUE_STRING)
+  return js;
+
+error:
+  if (je->value_type == JSON_VALUE_STRING)
+    report_json_error(js, je, 0);
+  return js;
+}
+
+
+String *Item_func_json_unquote::val_str(String *str)
+{
+  json_engine_t je;
+  int c_len;
+  String *js;
+
+  if (!(js= read_json(&je)))
+    return NULL;
+
+  if (je.s.error || je.value_type != JSON_VALUE_STRING)
     return js;
 
   str->length(0);
@@ -621,13 +637,86 @@ String *Item_func_json_unquote::val_str(String *str)
   return str;
 
 error:
-  if (je.value_type == JSON_VALUE_STRING)
   report_json_error(js, &je, 0);
-  /* We just return the argument's value in the case of error. */
   return js;
 }
 
 
+double Item_func_json_unquote::val_real()
+{
+  json_engine_t je;
+  double d= 0.0;
+  String *js;
+
+  if ((js= read_json(&je)) != NULL)
+  {
+    switch (je.value_type)
+    {
+      case JSON_VALUE_NUMBER:
+      {
+        char *end;
+        int err;
+        d= my_strntod(je.s.cs, (char *) je.value, je.value_len, &end, &err);
+        break;
+      }
+      case JSON_VALUE_TRUE:
+        d= 1.0;
+        break;
+      case JSON_VALUE_STRING:
+      {
+        char *end;
+        int err;
+        d= my_strntod(js->charset(), (char *) js->ptr(), js->length(),
+                      &end, &err);
+        break;
+      }
+      default:
+        break;
+    };
+  }
+
+  return d;
+}
+
+
+longlong Item_func_json_unquote::val_int()
+{
+  json_engine_t je;
+  longlong i= 0;
+  String *js;
+
+  if ((js= read_json(&je)) != NULL)
+  {
+    switch (je.value_type)
+    {
+      case JSON_VALUE_NUMBER:
+      {
+        char *end;
+        int err;
+        i= my_strntoll(je.s.cs, (char *) je.value, je.value_len, 10,
+                       &end, &err);
+        break;
+      }
+      case JSON_VALUE_TRUE:
+        i= 1;
+        break;
+      case JSON_VALUE_STRING:
+      {
+        char *end;
+        int err;
+        i= my_strntoll(js->charset(), (char *) js->ptr(), js->length(), 10,
+                       &end, &err);
+        break;
+      }
+      default:
+        break;
+    };
+  }
+
+  return i;
+}
+
+
 static int alloc_tmp_paths(THD *thd, uint n_paths,
                            json_path_with_flags **paths,String **tmp_paths)
 {
@@ -1397,6 +1486,8 @@ void Item_func_json_array::fix_length_and_dec()
   ulonglong char_length= 2;
   uint n_arg;
 
+  result_limit= 0;
+
   if (arg_count == 0)
   {
     collation.set(&my_charset_utf8_general_ci);
@@ -1413,7 +1504,6 @@ void Item_func_json_array::fix_length_and_dec()
 
   fix_char_length_ulonglong(char_length);
   tmp_val.set_charset(collation.collation);
-  result_limit= 0;
 }
 
 
@@ -2692,6 +2782,41 @@ void Item_func_json_keys::fix_length_and_dec()
 }
 
 
+/*
+  That function is for Item_func_json_keys::val_str exclusively.
+  It utilizes the fact the resulting string is in specific format:
+        ["key1", "key2"...]
+*/
+static int check_key_in_list(String *res,
+                             const uchar *key, int key_len)
+{
+  const uchar *c= (const uchar *) res->ptr() + 2; /* beginning '["' */
+  const uchar *end= (const uchar *) res->end() - 1; /* ending '"' */
+
+  while (c < end)
+  {
+    int n_char;
+    for (n_char=0; c[n_char] != '"' && n_char < key_len; n_char++)
+    {
+      if (c[n_char] != key[n_char])
+        break;
+    }
+    if (c[n_char] == '"')
+    {
+      if (n_char == key_len)
+        return 1;
+    }
+    else
+    {
+      while (c[n_char] != '"')
+        n_char++;
+    }
+    c+= n_char + 4; /* skip ', "' */
+  }
+  return 0;
+}
+
+
 String *Item_func_json_keys::val_str(String *str)
 {
   json_engine_t je;
@@ -2748,6 +2873,7 @@ skip_search:
   while (json_scan_next(&je) == 0 && je.state != JST_OBJ_END)
   {
     const uchar *key_start, *key_end;
+    int key_len;
 
     switch (je.state)
     {
@@ -2757,13 +2883,19 @@ skip_search:
       {
         key_end= je.s.c_str;
       } while (json_read_keyname_chr(&je) == 0);
-      if (je.s.error ||
+      if (je.s.error)
-          (n_keys > 0 && str->append(", ", 2)) ||
+        goto err_return;
+      key_len= key_end - key_start;
+
+      if (!check_key_in_list(str, key_start, key_len))
+      { 
+        if ((n_keys > 0 && str->append(", ", 2)) ||
           str->append("\"", 1) ||
-          append_simple(str, key_start, key_end - key_start) ||
+          append_simple(str, key_start, key_len) ||
           str->append("\"", 1))
         goto err_return;
         n_keys++;
+      }
       break;
     case JST_OBJ_START:
     case JST_ARRAY_START:

sql/item_jsonfunc.h

@@ -125,12 +125,14 @@ class Item_func_json_unquote: public Item_str_func
 {
 protected:
   String tmp_s;
-
+  String *read_json(json_engine_t *je);
 public:
   Item_func_json_unquote(THD *thd, Item *s): Item_str_func(thd, s) {}
   const char *func_name() const { return "json_unquote"; }
   void fix_length_and_dec();
   String *val_str(String *);
+  double val_real();
+  longlong val_int();
   Item *get_copy(THD *thd, MEM_ROOT *mem_root)
   { return get_item_copy<Item_func_json_unquote>(thd, mem_root, this); }
 };

storage/innobase/handler/ha_innodb.cc

@@ -9420,7 +9420,7 @@ ha_innobase::update_row(
 
 	innobase_srv_conc_enter_innodb(m_prebuilt);
 
-	error = row_update_for_mysql((byte*) old_row, m_prebuilt);
+	error = row_update_for_mysql(m_prebuilt);
 
 	if (error == DB_SUCCESS && autoinc) {
 		/* A value for an AUTO_INCREMENT column
@@ -9535,7 +9535,7 @@ ha_innobase::delete_row(
 
 	innobase_srv_conc_enter_innodb(m_prebuilt);
 
-	error = row_update_for_mysql((byte*) record, m_prebuilt);
+	error = row_update_for_mysql(m_prebuilt);
 
 	innobase_srv_conc_exit_innodb(m_prebuilt);
 

storage/innobase/include/row0mysql.h

@@ -272,13 +272,10 @@ row_table_got_default_clust_index(
 	const dict_table_t*	table);	/*!< in: table */
 
 /** Does an update or delete of a row for MySQL.
-@param[in]	mysql_rec	row in the MySQL format
 @param[in,out]	prebuilt	prebuilt struct in MySQL handle
 @return error code or DB_SUCCESS */
 dberr_t
-row_update_for_mysql(
+row_update_for_mysql(row_prebuilt_t* prebuilt)
-	const byte*		mysql_rec,
-	row_prebuilt_t*		prebuilt)
 	MY_ATTRIBUTE((warn_unused_result));
 
 /** This can only be used when srv_locks_unsafe_for_binlog is TRUE or this

storage/innobase/row/row0mysql.cc

@@ -740,6 +740,8 @@ row_mysql_handle_errors(
 {
 	dberr_t	err;
 
+	DBUG_ENTER("row_mysql_handle_errors");
+
 handle_new_error:
 	err = trx->error_state;
 
@@ -747,6 +749,9 @@ handle_new_error:
 
 	trx->error_state = DB_SUCCESS;
 
+	DBUG_LOG("trx", "handle error: " << ut_strerr(err)
+		 << ";id=" << ib::hex(trx->id) << ", " << trx);
+
 	switch (err) {
 	case DB_LOCK_WAIT_TIMEOUT:
 		if (row_rollback_on_timeout) {
@@ -795,7 +800,7 @@ handle_new_error:
 
 		*new_err = err;
 
-		return(true);
+		DBUG_RETURN(true);
 
 	case DB_DEADLOCK:
 	case DB_LOCK_TABLE_FULL:
@@ -840,7 +845,7 @@ handle_new_error:
 
 	trx->error_state = DB_SUCCESS;
 
-	return(false);
+	DBUG_RETURN(false);
 }
 
 /********************************************************************//**
@@ -1806,14 +1811,10 @@ public:
 
 
 /** Does an update or delete of a row for MySQL.
-@param[in]	mysql_rec	row in the MySQL format
 @param[in,out]	prebuilt	prebuilt struct in MySQL handle
 @return error code or DB_SUCCESS */
-static
 dberr_t
-row_update_for_mysql_using_upd_graph(
+row_update_for_mysql(row_prebuilt_t* prebuilt)
-	const byte*	mysql_rec,
-	row_prebuilt_t*	prebuilt)
 {
 	trx_savept_t	savept;
 	dberr_t		err;
@@ -1829,13 +1830,13 @@ row_update_for_mysql_using_upd_graph(
 	upd_cascade_t*	processed_cascades;
 	bool		got_s_lock	= false;
 
-	DBUG_ENTER("row_update_for_mysql_using_upd_graph");
+	DBUG_ENTER("row_update_for_mysql");
 
 	ut_ad(trx);
 	ut_a(prebuilt->magic_n == ROW_PREBUILT_ALLOCATED);
 	ut_a(prebuilt->magic_n2 == ROW_PREBUILT_ALLOCATED);
+	ut_a(prebuilt->template_type == ROW_MYSQL_WHOLE_ROW);
 	ut_ad(table->stat_initialized);
-	UT_NOT_USED(mysql_rec);
 
 	if (!table->is_readable()) {
 		return(row_mysql_get_table_status(table, trx, true));
@@ -2154,19 +2155,6 @@ error:
 	DBUG_RETURN(err);
 }
 
-/** Does an update or delete of a row for MySQL.
-@param[in]	mysql_rec	row in the MySQL format
-@param[in,out]	prebuilt	prebuilt struct in MySQL handle
-@return error code or DB_SUCCESS */
-dberr_t
-row_update_for_mysql(
-	const byte*		mysql_rec,
-	row_prebuilt_t*		prebuilt)
-{
-	ut_a(prebuilt->template_type == ROW_MYSQL_WHOLE_ROW);
-	return(row_update_for_mysql_using_upd_graph(mysql_rec, prebuilt));
-}
-
 /** This can only be used when srv_locks_unsafe_for_binlog is TRUE or this
 session is using a READ COMMITTED or READ UNCOMMITTED isolation level.
 Before calling this function row_search_for_mysql() must have

storage/innobase/trx/trx0purge.cc

@@ -293,14 +293,16 @@ trx_purge_add_update_undo_to_history(
 
 	After the purge thread has been given permission to exit,
 	in fast shutdown, we may roll back transactions (trx->undo_no==0)
-	in THD::cleanup() invoked from unlink_thd(). */
+	in THD::cleanup() invoked from unlink_thd(), and we may also
+	continue to execute user transactions. */
 	ut_ad(srv_undo_sources
 	      || ((srv_startup_is_before_trx_rollback_phase
 		   || trx_rollback_or_clean_is_active)
 		  && purge_sys->state == PURGE_STATE_INIT)
 	      || (srv_force_recovery >= SRV_FORCE_NO_BACKGROUND
 		  && purge_sys->state == PURGE_STATE_DISABLED)
-	      || (trx->undo_no == 0 && srv_fast_shutdown));
+	      || ((trx->undo_no == 0 || trx->in_mysql_trx_list)
+		  && srv_fast_shutdown));
 
 	/* Add the log as the first in the history list */
 	flst_add_first(rseg_header + TRX_RSEG_HISTORY,

storage/innobase/trx/trx0trx.cc

@@ -238,6 +238,7 @@ struct TrxFactory {
 
 		trx_init(trx);
 
+		DBUG_LOG("trx", "Init: " << trx);
 		trx->state = TRX_STATE_NOT_STARTED;
 
 		trx->dict_operation_lock_mode = 0;
@@ -452,6 +453,7 @@ trx_create_low()
 
 	/* Trx state can be TRX_STATE_FORCED_ROLLBACK if
 	the trx was forced to rollback before it's reused.*/
+	DBUG_LOG("trx", "Create: " << trx);
 	trx->state = TRX_STATE_NOT_STARTED;
 
 	heap = mem_heap_create(sizeof(ib_vector_t) + sizeof(void*) * 8);
@@ -630,6 +632,7 @@ trx_free_prepared(
 
 	ut_d(trx->in_rw_trx_list = FALSE);
 
+	DBUG_LOG("trx", "Free prepared: " << trx);
 	trx->state = TRX_STATE_NOT_STARTED;
 
 	/* Undo trx_resurrect_table_locks(). */
@@ -1753,6 +1756,7 @@ trx_commit_in_memory(
 		ut_ad(!(trx->in_innodb
 			& (TRX_FORCE_ROLLBACK | TRX_FORCE_ROLLBACK_ASYNC)));
 
+		DBUG_LOG("trx", "Autocommit in memory: " << trx);
 		trx->state = TRX_STATE_NOT_STARTED;
 
 	} else {
@@ -1888,8 +1892,10 @@ trx_commit_in_memory(
 	if (trx->abort) {
 
 		trx->abort = false;
+		DBUG_LOG("trx", "Abort: " << trx);
 		trx->state = TRX_STATE_FORCED_ROLLBACK;
 	} else {
+		DBUG_LOG("trx", "Commit in memory: " << trx);
 		trx->state = TRX_STATE_NOT_STARTED;
 	}
 
@@ -2061,6 +2067,7 @@ trx_cleanup_at_db_startup(
 	ut_ad(trx->is_recovered);
 	ut_ad(!trx->in_rw_trx_list);
 	ut_ad(!trx->in_mysql_trx_list);
+	DBUG_LOG("trx", "Cleanup at startup: " << trx);
 	trx->state = TRX_STATE_NOT_STARTED;
 }
 

strings/json_lib.c

@@ -253,7 +253,7 @@ static int read_4_hexdigits(json_string_t *s, uchar *dest)
     if ((c_len= json_next_char(s)) <= 0)
       return s->error= json_eos(s) ? JE_EOS : JE_BAD_CHR;
 
-    if (s->c_next >= 128 || (t= json_instr_chr_map[s->c_next]) >= S_F)
+    if (s->c_next >= 128 || (t= json_instr_chr_map[s->c_next]) > S_F)
       return s->error= JE_SYN;
 
     s->c_str+= c_len;