MDEV-25004 Missing row in FTS_DOC_ID_INDEX during DELETE HISTORY

1. In case of system-versioned table add row_end into FTS_DOC_ID index
   in fts_create_common_tables() and innobase_create_key_defs().
   fts_n_uniq() returns 1 or 2 depending on whether the table is
   system-versioned.

   After this patch recreate of FTS_DOC_ID index is required for
   existing system-versioned tables. If you see this message in error
   log or server warnings: "InnoDB: Table db/t1 contains 2 indexes
   inside InnoDB, which is different from the number of indexes 1
   defined in the MariaDB" use this command to fix the table:

      ALTER TABLE db.t1 FORCE;

2. Fix duplicate history for secondary unique index like it was done
   in MDEV-23644 for clustered index (932ec586aa). In case of
   existing history row which conflicts with currently inseted row we
   check in row_ins_scan_sec_index_for_duplicate() whether that row
   was inserted as part of current transaction. In that case we
   indicate with DB_FOREIGN_DUPLICATE_KEY that new history row is not
   needed and should be silently skipped.

3. Some parts of MDEV-21138 (7410ff436e) reverted. Skipping of
   FTS_DOC_ID index for history rows made problems with purge
   system. Now this is fixed differently by p.2.

4. wait_all_purged.inc checks that we didn't affect non-history rows
   so they are deleted and purged correctly.

Additional FTS fixes

  fts_init_get_doc_id(): exclude history rows from max_doc_id
  calculation. fts_init_get_doc_id() callback is used only for crash
  recovery.

  fts_add_doc_by_id(): set max value for row_end field.

  fts_read_stopword(): stopwords table can be system-versioned too. We
  now read stopwords only for current data.

  row_insert_for_mysql(): exclude history rows from doc_id validation.

  row_merge_read_clustered_index(): exclude history_rows from doc_id
  processing.

  fts_load_user_stopword(): for versioned table retrieve row_end field
  and skip history rows. For non-versioned table we retrieve 'value'
  field twice (just for uniformity).

FTS tests for System Versioning now include maybe_versioning.inc which
adds 3 combinations:

'vers'     for debug build sets sysvers_force and
	   sysvers_hide. sysvers_force makes every created table
	   system-versioned, sysvers_hide hides WITH SYSTEM VERSIONING
	   for SHOW CREATE.

	   Note: basic.test, stopword.test and versioning.test do not
	   require debug for 'vers' combination. This is controlled by
	   $modify_create_table in maybe_versioning.inc and these
	   tests run WITH SYSTEM VERSIONING explicitly which allows to
	   test 'vers' combination on non-debug builds.

'vers_trx' like 'vers' sets sysvers_force_trx and sysvers_hide. That
	   tests FTS with trx_id-based System Versioning.

'orig' 	   works like before: no System Versioning is added, no debug is
	   required.

Upgrade/downgrade test for System Versioning is done by
innodb_fts.versioning. It has 2 combinations:

'prepare' makes binaries in std_data (requires old server and OLD_BINDIR).
	  It tests upgrade/downgrade against old server as well.

'upgrade' tests upgrade against binaries in std_data.

Cleanups:

Removed innodb-fts-stopword.test as it duplicates stopword.test

mysql-test/suite/versioning/r/alter.result

@@ -375,6 +375,11 @@ a	b
 2	NULL
 3	1
 4	2
+alter table t add c int, drop system versioning;
+select * from t;
+a	b	c
+3	1	NULL
+4	2	NULL
 create or replace table t (a int) with system versioning;
 insert into t values (1), (2), (3);
 delete from t where a<3;

mysql-test/suite/versioning/r/debug.result

@@ -19,7 +19,7 @@ show create table tt2;
 Table	Create Table
 tt2	CREATE TEMPORARY TABLE `tt2` (
   `a` int(11) DEFAULT NULL
-) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci
+) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci WITH SYSTEM VERSIONING
 connect con1, localhost, root;
 create table t3 (a int);
 show create table t3;
@@ -32,7 +32,7 @@ show create table tt3;
 Table	Create Table
 tt3	CREATE TEMPORARY TABLE `tt3` (
   `a` int(11) DEFAULT NULL
-) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci
+) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci WITH SYSTEM VERSIONING
 disconnect con1;
 connection default;
 set debug_dbug='+d,sysvers_show';

mysql-test/suite/versioning/r/delete.result

@@ -146,6 +146,5 @@ delete from t1;
 select f1, f3, check_row_ts(row_start, row_end) from t1 for system_time all;
 f1	f3	check_row_ts(row_start, row_end)
 1	1	HISTORICAL ROW
-1	NULL	ERROR: row_end == row_start
 1	1	HISTORICAL ROW
 drop table t1;

mysql-test/suite/versioning/r/delete_history.result

@@ -1,3 +1,5 @@
+set @saved_frequency= @@global.innodb_purge_rseg_truncate_frequency;
+set global innodb_purge_rseg_truncate_frequency= 1;
 create table t (a int);
 delete history from t before system_time now();
 ERROR HY000: Table `t` is not system-versioned
@@ -165,3 +167,23 @@ x
 1
 drop prepare stmt;
 drop table t1;
+#
+# MDEV-25004 Missing row in FTS_DOC_ID_INDEX during DELETE HISTORY
+#
+create table t1 (a integer, c0 varchar(255), fulltext key (c0))
+with system versioning engine innodb;
+set system_versioning_alter_history= keep;
+alter table t1 drop system versioning;
+alter table t1 add system versioning;
+insert into t1 values (1, 'politician');
+update t1 set c0= 'criminal';
+InnoDB		0 transactions not purged
+delete history from t1;
+drop table t1;
+create table t1 (id int primary key, ftx varchar(255))
+with system versioning engine innodb;
+insert into t1 values (1, 'c');
+delete from t1;
+alter table t1 add fulltext key(ftx);
+drop table t1;
+set global innodb_purge_rseg_truncate_frequency= @saved_frequency;

mysql-test/suite/versioning/r/foreign.result

@@ -443,6 +443,43 @@ pk	f1	f2	left(f3, 4)	check_row_ts(row_start, row_end)
 1	8	8	SHOR	HISTORICAL ROW
 2	8	8	LONG	HISTORICAL ROW
 drop table t1;
+# Shorter case for clustered index (MDEV-25004)
+create table t1 (
+y int primary key, r int, f int, key (r),
+foreign key (f) references t1 (r) on delete set null)
+with system versioning engine innodb;
+insert into t1 values (1, 6, 6), (2, 6, 6);
+delete from t1;
+select *, check_row_ts(row_start, row_end) from t1 for system_time all;
+y	r	f	check_row_ts(row_start, row_end)
+1	6	6	HISTORICAL ROW
+2	6	6	HISTORICAL ROW
+drop tables t1;
+# Secondary unique index
+create table t1 (
+y int unique null, r int, f int, key (r),
+foreign key (f) references t1 (r) on delete set null)
+with system versioning engine innodb;
+insert into t1 values (1, 6, 6), (2, 6, 6);
+delete from t1;
+select *, check_row_ts(row_start, row_end) from t1 for system_time all;
+y	r	f	check_row_ts(row_start, row_end)
+1	6	6	HISTORICAL ROW
+2	6	6	HISTORICAL ROW
+drop tables t1;
+# Non-unique index cannot be fixed because it does not trigger duplicate error
+create table t1 (
+y int, r int, f int, key (y), key (r),
+foreign key (f) references t1 (r) on delete set null)
+with system versioning engine innodb;
+insert into t1 values (1, 6, 6), (2, 6, 6);
+delete from t1;
+select *, check_row_ts(row_start, row_end) from t1 for system_time all;
+y	r	f	check_row_ts(row_start, row_end)
+1	6	6	HISTORICAL ROW
+2	6	NULL	ERROR: row_end == row_start
+2	6	6	HISTORICAL ROW
+drop tables t1;
 #
 # MDEV-21555 Assertion secondary index is out of sync on delete from versioned table
 #

mysql-test/suite/versioning/t/alter.test

@@ -264,6 +264,8 @@ select * from t;
 select * from t for system_time all;
 insert into t values (4, 0);
 select * from t for system_time all;
+alter table t add c int, drop system versioning;
+select * from t;
 
 create or replace table t (a int) with system versioning;
 insert into t values (1), (2), (3);

mysql-test/suite/versioning/t/delete_history.test

@@ -2,6 +2,9 @@
 --source include/have_partition.inc
 --source suite/versioning/engines.inc
 
+set @saved_frequency= @@global.innodb_purge_rseg_truncate_frequency;
+set global innodb_purge_rseg_truncate_frequency= 1;
+
 create table t (a int);
 --error ER_VERS_NOT_VERSIONED
 delete history from t before system_time now();
@@ -164,4 +167,26 @@ select * from t1;
 drop prepare stmt;
 drop table t1;
 
+--echo #
+--echo # MDEV-25004 Missing row in FTS_DOC_ID_INDEX during DELETE HISTORY
+--echo #
+create table t1 (a integer, c0 varchar(255), fulltext key (c0))
+with system versioning engine innodb;
+set system_versioning_alter_history= keep;
+alter table t1 drop system versioning;
+alter table t1 add system versioning;
+insert into t1 values (1, 'politician');
+update t1 set c0= 'criminal';
+--source suite/innodb/include/wait_all_purged.inc
+delete history from t1;
+drop table t1;
+
+create table t1 (id int primary key, ftx varchar(255))
+with system versioning engine innodb;
+insert into t1 values (1, 'c');
+delete from t1;
+alter table t1 add fulltext key(ftx);
+drop table t1;
+
+set global innodb_purge_rseg_truncate_frequency= @saved_frequency;
 --source suite/versioning/common_finish.inc

mysql-test/suite/versioning/t/foreign.test

@@ -476,6 +476,39 @@ select pk, f1, f2, left(f3, 4), check_row_ts(row_start, row_end) from t1 for sys
 # cleanup
 drop table t1;
 
+--echo # Shorter case for clustered index (MDEV-25004)
+create table t1 (
+  y int primary key, r int, f int, key (r),
+  foreign key (f) references t1 (r) on delete set null)
+with system versioning engine innodb;
+
+insert into t1 values (1, 6, 6), (2, 6, 6);
+delete from t1;
+select *, check_row_ts(row_start, row_end) from t1 for system_time all;
+drop tables t1;
+
+--echo # Secondary unique index
+create table t1 (
+  y int unique null, r int, f int, key (r),
+  foreign key (f) references t1 (r) on delete set null)
+with system versioning engine innodb;
+
+insert into t1 values (1, 6, 6), (2, 6, 6);
+delete from t1;
+select *, check_row_ts(row_start, row_end) from t1 for system_time all;
+drop tables t1;
+
+--echo # Non-unique index cannot be fixed because it does not trigger duplicate error
+create table t1 (
+  y int, r int, f int, key (y), key (r),
+  foreign key (f) references t1 (r) on delete set null)
+with system versioning engine innodb;
+
+insert into t1 values (1, 6, 6), (2, 6, 6);
+delete from t1;
+select *, check_row_ts(row_start, row_end) from t1 for system_time all;
+drop tables t1;
+
 --echo #
 --echo # MDEV-21555 Assertion secondary index is out of sync on delete from versioned table
 --echo #