From 784becf3e161058f5453e90d2c9627472345dbe2 Mon Sep 17 00:00:00 2001
From: Sergei Golubchik <serg@mariadb.org>
Date: Tue, 29 Oct 2024 09:44:24 +0100
Subject: [PATCH] MDEV-35267 Server crashes in _ma_reset_history upon altering
 on Aria table with vector key under lock

ALTER TABLE needs to open hlindex tables early enough, right after they
were created, so that cleanup after an error would see and delete them.

But they need to be external_lock-ed only in copy_data_between_tables,
after mysql_trans_prepare_alter_copy_data().

Let's move locking out of hlindex_open() into hlindex_lock()
---
 mysql-test/main/vector_aria.result |  9 +++++++++
 mysql-test/main/vector_aria.test   | 10 ++++++++++
 sql/handler.cc                     |  7 ++++---
 sql/sql_base.cc                    | 15 ++++++++++++---
 sql/sql_table.cc                   |  7 ++++++-
 sql/table.h                        |  1 +
 6 files changed, 42 insertions(+), 7 deletions(-)

diff --git a/mysql-test/main/vector_aria.result b/mysql-test/main/vector_aria.result
index c6cd5fb5f3a..1a94098b364 100644
--- a/mysql-test/main/vector_aria.result
+++ b/mysql-test/main/vector_aria.result
@@ -8,3 +8,12 @@ check table t extended;
 Table	Op	Msg_type	Msg_text
 test.t	check	status	OK
 drop table t;
+#
+# MDEV-35267 Server crashes in _ma_reset_history upon altering on Aria table with vector key under lock
+#
+create table t (v vector(1) not null, vector(v)) engine=aria;
+insert into t values (0x31313131);
+lock table t write;
+alter table t force;
+unlock tables;
+drop table t;
diff --git a/mysql-test/main/vector_aria.test b/mysql-test/main/vector_aria.test
index 1cf6ab99a24..cacb96863a5 100644
--- a/mysql-test/main/vector_aria.test
+++ b/mysql-test/main/vector_aria.test
@@ -6,3 +6,13 @@ insert into t values (1, 0x30303030),(2, 0x31313131);
 alter table t add vector(f);
 check table t extended;
 drop table t;
+
+--echo #
+--echo # MDEV-35267 Server crashes in _ma_reset_history upon altering on Aria table with vector key under lock
+--echo #
+create table t (v vector(1) not null, vector(v)) engine=aria;
+insert into t values (0x31313131);
+lock table t write;
+alter table t force;
+unlock tables;
+drop table t;
diff --git a/sql/handler.cc b/sql/handler.cc
index 9084fe82210..b8b6e961557 100644
--- a/sql/handler.cc
+++ b/sql/handler.cc
@@ -5369,7 +5369,7 @@ int handler::ha_check(THD *thd, HA_CHECK_OPT *check_opt)
   for (uint i= table->s->keys; i < table->s->total_keys; i++)
   {
     DBUG_ASSERT(table->s->hlindexes() == 1);
-    if (table->hlindex_open(i))
+    if (table->hlindex_open(i) || table->hlindex_lock(i))
       return HA_ADMIN_FAILED;
     if ((error= table->hlindex->file->check(thd, check_opt)))
       return error;
@@ -5502,9 +5502,10 @@ handler::ha_truncate()
   mark_trx_read_write();
 
   int err= truncate();
-  if (!err && table->s->hlindexes())
+  for (uint i= table->s->keys; !err && i < table->s->total_keys; i++)
   {
-    if (!(err= table->hlindex_open(table->s->keys)))
+    DBUG_ASSERT(table->s->hlindexes() == 1);
+    if (!(err= table->hlindex_open(i)) && !(err= table->hlindex_lock(i)))
       err= table->hlindexes_on_delete_all(true);
   }
 
diff --git a/sql/sql_base.cc b/sql/sql_base.cc
index 6b0b996fa62..2e85e63fa99 100644
--- a/sql/sql_base.cc
+++ b/sql/sql_base.cc
@@ -9891,8 +9891,17 @@ int TABLE::hlindex_open(uint nr)
                     db_stat, EXTRA_RECORD, in_use->open_options, table, 0))
       return 1;
     hlindex= table;
+    hlindex->in_use= NULL;
   }
-  else if (hlindex->in_use == in_use)
+  return 0;
+}
+
+int TABLE::hlindex_lock(uint nr)
+{
+  DBUG_ASSERT(s->hlindexes() == 1);
+  DBUG_ASSERT(nr == s->keys);
+  DBUG_ASSERT(hlindex);
+  if (hlindex->in_use == in_use)
     return 0;
   hlindex->in_use= in_use;      // mark in use for this query
   hlindex->use_all_columns();
@@ -9920,7 +9929,7 @@ int TABLE::open_hlindexes_for_write()
 {
   DBUG_ASSERT(s->hlindexes() <= 1);
   for (uint i= s->keys; i < s->total_keys; i++)
-    if (hlindex_open(i))
+    if (hlindex_open(i) || hlindex_lock(i))
       return 1;
   return 0;
 }
@@ -9983,7 +9992,7 @@ int TABLE::hlindex_read_first(uint nr, Item *item, ulonglong limit)
   DBUG_ASSERT(s->hlindexes() == 1);
   DBUG_ASSERT(nr == s->keys);
 
-  if (hlindex_open(nr))
+  if (hlindex_open(nr) || hlindex_lock(nr))
     return HA_ERR_CRASHED;
 
   DBUG_ASSERT(hlindex->in_use == in_use);
diff --git a/sql/sql_table.cc b/sql/sql_table.cc
index 09f15bead6d..6d69051af65 100644
--- a/sql/sql_table.cc
+++ b/sql/sql_table.cc
@@ -11555,9 +11555,14 @@ alter_copy:
   new_table= thd->create_and_open_tmp_table(&frm, alter_ctx.get_tmp_path(),
                                             alter_ctx.new_db,
                                             alter_ctx.new_name, true);
-  if (!new_table || new_table->open_hlindexes_for_write())
+  if (!new_table)
     goto err_new_table_cleanup;
 
+  DBUG_ASSERT(new_table->s->hlindexes() <= 1);
+  for (uint i= new_table->s->keys; i < new_table->s->total_keys; i++)
+    if (new_table->hlindex_open(i))
+      goto err_new_table_cleanup;
+
   if (table->s->tmp_table != NO_TMP_TABLE)
   {
     /* in case of alter temp table send the tracker in OK packet */
diff --git a/sql/table.h b/sql/table.h
index 7253428908c..a30dfbe7a5e 100644
--- a/sql/table.h
+++ b/sql/table.h
@@ -1797,6 +1797,7 @@ public:
   inline ha_rows stat_records() { return used_stat_records; }
 
   int hlindex_open(uint nr);
+  int hlindex_lock(uint nr);
   int hlindex_read_first(uint nr, Item *item, ulonglong limit);
   int hlindex_read_next();
   int hlindex_read_end();