MDEV-29721 Inconsistency upon inserting history with visible period columns

don't set vers_write=false if one vers column was used explicitly,
instead do vers_update_fields() for columns that do not have explicit
value. So, if row_start has the value and row_end not, row_end will
get max by default.

mysql-test/suite/versioning/r/insert.result

@@ -170,11 +170,9 @@ ERROR HY000: Incorrect row_start value: '1980-01-01 00:00:01.000000'
 insert into t1(x, row_start, row_end) values (7, '1980-01-01 00:00:11', '1980-01-01 00:00:11');
 ERROR HY000: Incorrect row_start value: '1980-01-01 00:00:11.000000'
 insert into t1(x, row_start) values (8, '1980-01-01 00:00:22');
-ERROR HY000: Incorrect row_start value: '1980-01-01 00:00:22.000000'
 insert into t1(x, row_end) values (9, '1980-01-01 00:00:33');
-ERROR HY000: Incorrect row_start value: '0000-00-00 00:00:00.000000'
+ERROR HY000: Incorrect row_start value: 'now'
 insert into t1(x, row_end) values (10, TIMESTAMP'2038-01-19 03:14:07.999999');
-ERROR HY000: Incorrect row_start value: '0000-00-00 00:00:00.000000'
 select x, check_row_ts(row_start, row_end) from t1 for system_time all order by x;
 x	check_row_ts(row_start, row_end)
 1	HISTORICAL ROW
@@ -182,6 +180,8 @@ x	check_row_ts(row_start, row_end)
 3	HISTORICAL ROW
 4	CURRENT ROW
 5	HISTORICAL ROW
+8	CURRENT ROW
+10	CURRENT ROW
 select x, row_start, row_end from t1 for system_time all
 where x > 1 and row_end < TIMESTAMP'2038-01-19 03:14:07.999999' order by x, row_start, row_end;
 x	row_start	row_end
@@ -207,6 +207,8 @@ x	check_row_ts(row_start, row_end)
 3	HISTORICAL ROW
 4	CURRENT ROW
 5	HISTORICAL ROW
+8	CURRENT ROW
+10	CURRENT ROW
 select x, row_start, row_end from t2 for system_time all
 where x > 1 and row_end < TIMESTAMP'2038-01-19 03:14:07.999999' order by x, row_start, row_end;
 x	row_start	row_end
@@ -229,10 +231,8 @@ ERROR 42S22: Unknown column 'row_start' in 'field list'
 replace into t2 (a, row_start, row_end) select x, row_start, row_end from t1;
 ERROR 42S22: Unknown column 'row_start' in 'field list'
 set @@system_versioning_insert_history= 1;
-replace into t2 (a, row_start) values (1, '1980-01-01 00:00:00');
-ERROR HY000: Incorrect row_start value: '1980-01-01 00:00:00.000000'
 replace into t2 (a, row_end) values (0, '1980-01-01 00:00:00');
-ERROR HY000: Incorrect row_start value: '0000-00-00 00:00:00.000000'
+ERROR HY000: Incorrect row_start value: 'now'
 replace into t2 (a, row_start, row_end) values (1, '1980-01-01 00:00:00', '1980-01-01 00:00:01');
 select a, row_start, row_end from t2 for system_time all order by a, row_start, row_end;
 a	row_start	row_end
@@ -243,63 +243,75 @@ select a, row_start, row_end from t2 for system_time all order by a, row_start,
 a	row_start	row_end
 1	1980-01-01 00:00:00.000000	1980-01-01 00:00:01.000000
 1	1980-01-01 00:00:00.000000	1990-01-01 00:00:01.000000
-# But changing row_start via REPLACE is possible:
+# REPLACE is DELETE + INSERT
+set timestamp=unix_timestamp('2020-10-10 10:10:10');
 replace into t2 (a, row_start, row_end) values (1, '1971-01-01 00:00:00', '1980-01-01 00:00:01');
+set timestamp=default;
 select a, row_start, row_end from t2 for system_time all order by a, row_start, row_end;
 a	row_start	row_end
 1	1971-01-01 00:00:00.000000	1980-01-01 00:00:01.000000
 1	1980-01-01 00:00:00.000000	1990-01-01 00:00:01.000000
+1	1980-01-01 00:00:00.000000	2020-10-10 10:10:10.000000
 replace into t2 (a, row_start, row_end) select x, row_start, row_end from t1 for system_time all
 where x > 1 and row_end < TIMESTAMP'2038-01-19 03:14:07.999999';
 select a, row_start, row_end from t2 for system_time all order by a, row_start, row_end;
 a	row_start	row_end
 1	1971-01-01 00:00:00.000000	1980-01-01 00:00:01.000000
 1	1980-01-01 00:00:00.000000	1990-01-01 00:00:01.000000
+1	1980-01-01 00:00:00.000000	2020-10-10 10:10:10.000000
 3	1980-01-01 00:00:00.000000	1980-01-01 00:00:01.000000
 5	1980-01-01 00:00:00.000000	1980-01-01 00:00:01.000000
 # LOAD DATA
 select x, row_start, row_end into outfile 'DATAFILE' from t1 for system_time all;
-create or replace table t3 like t1;
+create or replace table t2 like t1;
-show create table t3;
+show create table t2;
 Table	Create Table
-t3	CREATE TABLE `t3` (
+t2	CREATE TABLE `t2` (
   `x` int(11) NOT NULL,
   PRIMARY KEY (`x`)
 ) ENGINE=DEFAULT_ENGINE DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci WITH SYSTEM VERSIONING
 set @@system_versioning_insert_history= 1;
-show create table t3;
+show create table t2;
 Table	Create Table
-t3	CREATE TABLE `t3` (
+t2	CREATE TABLE `t2` (
   `x` int(11) NOT NULL,
   PRIMARY KEY (`x`)
 ) ENGINE=DEFAULT_ENGINE DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci WITH SYSTEM VERSIONING
-load data infile 'DATAFILE' into table t3 (x, row_start, row_end);
+load data infile 'DATAFILE' into table t2 (x, row_start, row_end);
-select x, check_row_ts(row_start, row_end) from t3 for system_time all order by x;
+select x, check_row_ts(row_start, row_end) from t2 for system_time all order by x;
 x	check_row_ts(row_start, row_end)
 1	HISTORICAL ROW
 2	CURRENT ROW
 3	HISTORICAL ROW
 4	CURRENT ROW
 5	HISTORICAL ROW
-select row_start = '1980-01-01 00:00:00', row_end = '1980-01-01 00:00:01' from t3 for system_time all where x = 3;
+8	CURRENT ROW
+10	CURRENT ROW
+select row_start = '1980-01-01 00:00:00', row_end = '1980-01-01 00:00:01' from t2 for system_time all where x = 3;
 row_start = '1980-01-01 00:00:00'	row_end = '1980-01-01 00:00:01'
 1	1
 # Honor secure_timestamp option
 # restart: --secure-timestamp=YES
 set @@system_versioning_insert_history= 1;
-insert into t1(x, row_start, row_end) values (8, '1980-01-01 00:00:00', '1980-01-01 00:00:01');
+insert into t3(z, row_start, row_end) values (8, '1980-01-01 00:00:00', '1980-01-01 00:00:01');
+ERROR HY000: The MariaDB server is running with the --secure-timestamp=YES option so it cannot execute this statement
+insert into t3 values (8, '1980-01-01 00:00:00', '1980-01-01 00:00:01');
 ERROR HY000: The MariaDB server is running with the --secure-timestamp=YES option so it cannot execute this statement
 # restart: --secure-timestamp=REPLICATION
 create user nobody;
 grant all privileges on test.* to nobody;
 set @@system_versioning_insert_history= 1;
-insert into test.t1(x, row_start, row_end) values (9, '1980-01-01 00:00:00', '1980-01-01 00:00:01');
+insert into test.t3(z, row_start, row_end) values (9, '1980-01-01 00:00:00', '1980-01-01 00:00:01');
+ERROR 42000: Access denied; you need (at least one of) the BINLOG REPLAY privilege(s) for this operation
+insert into test.t3 values (9, '1980-01-01 00:00:00', '1980-01-01 00:00:01');
 ERROR 42000: Access denied; you need (at least one of) the BINLOG REPLAY privilege(s) for this operation
 # restart: --secure-timestamp=SUPER
 set @@system_versioning_insert_history= 1;
-insert into test.t1(x, row_start, row_end) values (10, '1980-01-01 00:00:00', '1980-01-01 00:00:01');
+insert into test.t3(z, row_start, row_end) values (10, '1980-01-01 00:00:00', '1980-01-01 00:00:01');
 set @@system_versioning_insert_history= 1;
-insert into test.t1(x, row_start, row_end) values (7, '1980-01-01 00:00:00', '1980-01-01 00:00:01');
+insert into test.t3(z, row_start, row_end) values (7, '1980-01-01 00:00:00', '1980-01-01 00:00:01');
+ERROR 42000: Access denied; you need (at least one of) the SUPER, BINLOG REPLAY privilege(s) for this operation
+insert into test.t3 values (7, '1980-01-01 00:00:00', '1980-01-01 00:00:01');
 ERROR 42000: Access denied; you need (at least one of) the SUPER, BINLOG REPLAY privilege(s) for this operation
 use test;
 # restart: --secure-timestamp=NO

mysql-test/suite/versioning/t/insert.test

@@ -134,13 +134,10 @@ insert into t1 set x= 5, row_start= '1980-01-01 00:00:00', row_end= '1980-01-01
 insert into t1(x, row_start, row_end) values (6, '1980-01-01 00:00:01', '1980-01-01 00:00:00');
 --error ER_WRONG_VALUE
 insert into t1(x, row_start, row_end) values (7, '1980-01-01 00:00:11', '1980-01-01 00:00:11');
---error ER_WRONG_VALUE
 insert into t1(x, row_start) values (8, '1980-01-01 00:00:22');
-# NOTE: having row_start=0 might be useful and can mean
+--replace_regex /'202\d-\d\d-\d\d .*'/'now'/
-# "there is no information on when history was started" (an opposite to row_end=MAX_TIMESTAMP)
 --error ER_WRONG_VALUE
 insert into t1(x, row_end) values (9, '1980-01-01 00:00:33');
---error ER_WRONG_VALUE
 eval insert into t1(x, row_end) values (10, $MAX_TIMESTAMP);
 select x, check_row_ts(row_start, row_end) from t1 for system_time all order by x;
 eval select x, row_start, row_end from t1 for system_time all
@@ -180,18 +177,18 @@ replace into t2 (a, row_start, row_end) values (1, '1980-01-01 00:00:00', '1980-
 --error ER_BAD_FIELD_ERROR
 replace into t2 (a, row_start, row_end) select x, row_start, row_end from t1;
 set @@system_versioning_insert_history= 1;
---error ER_WRONG_VALUE
+--replace_regex /'202\d-\d\d-\d\d .*'/'now'/
-replace into t2 (a, row_start) values (1, '1980-01-01 00:00:00');
 --error ER_WRONG_VALUE
 replace into t2 (a, row_end) values (0, '1980-01-01 00:00:00');
 replace into t2 (a, row_start, row_end) values (1, '1980-01-01 00:00:00', '1980-01-01 00:00:01');
 select a, row_start, row_end from t2 for system_time all order by a, row_start, row_end;
 --echo # Changing row_end via REPLACE is NOT possible, we just insert new row:
-# NOTE: because multiple versions of history row with a=1 may exist, so what REPLACE should change?
 replace into t2 (a, row_start, row_end) values (1, '1980-01-01 00:00:00', '1990-01-01 00:00:01');
 select a, row_start, row_end from t2 for system_time all order by a, row_start, row_end;
---echo # But changing row_start via REPLACE is possible:
+--echo # REPLACE is DELETE + INSERT
+set timestamp=unix_timestamp('2020-10-10 10:10:10');
 replace into t2 (a, row_start, row_end) values (1, '1971-01-01 00:00:00', '1980-01-01 00:00:01');
+set timestamp=default;
 select a, row_start, row_end from t2 for system_time all order by a, row_start, row_end;
 eval replace into t2 (a, row_start, row_end) select x, row_start, row_end from t1 for system_time all
 where x > 1 and row_end < $MAX_TIMESTAMP;
@@ -201,16 +198,16 @@ select a, row_start, row_end from t2 for system_time all order by a, row_start,
 --let DATAFILE= $MYSQLTEST_VARDIR/tmp/test_versioning_t3.data
 --replace_result $DATAFILE DATAFILE
 eval select x, row_start, row_end into outfile '$DATAFILE' from t1 for system_time all;
-create or replace table t3 like t1;
+create or replace table t2 like t1;
 --replace_result $default_engine DEFAULT_ENGINE
-show create table t3;
+show create table t2;
 set @@system_versioning_insert_history= 1;
 --replace_result $default_engine DEFAULT_ENGINE
-show create table t3;
+show create table t2;
 --replace_result $DATAFILE DATAFILE
-eval load data infile '$DATAFILE' into table t3 (x, row_start, row_end);
+eval load data infile '$DATAFILE' into table t2 (x, row_start, row_end);
-select x, check_row_ts(row_start, row_end) from t3 for system_time all order by x;
+select x, check_row_ts(row_start, row_end) from t2 for system_time all order by x;
-select row_start = '1980-01-01 00:00:00', row_end = '1980-01-01 00:00:01' from t3 for system_time all where x = 3;
+select row_start = '1980-01-01 00:00:00', row_end = '1980-01-01 00:00:01' from t2 for system_time all where x = 3;
 --remove_file $DATAFILE
 
 --echo # Honor secure_timestamp option
@@ -218,7 +215,9 @@ select row_start = '1980-01-01 00:00:00', row_end = '1980-01-01 00:00:01' from t
 --source include/restart_mysqld.inc
 set @@system_versioning_insert_history= 1;
 --error ER_OPTION_PREVENTS_STATEMENT
-insert into t1(x, row_start, row_end) values (8, '1980-01-01 00:00:00', '1980-01-01 00:00:01');
+insert into t3(z, row_start, row_end) values (8, '1980-01-01 00:00:00', '1980-01-01 00:00:01');
+--error ER_OPTION_PREVENTS_STATEMENT
+insert into t3 values (8, '1980-01-01 00:00:00', '1980-01-01 00:00:01');
 --let $restart_parameters= --secure-timestamp=REPLICATION
 --source include/restart_mysqld.inc
 create user nobody;
@@ -226,16 +225,20 @@ grant all privileges on test.* to nobody;
 change_user nobody;
 set @@system_versioning_insert_history= 1;
 --error ER_SPECIFIC_ACCESS_DENIED_ERROR
-insert into test.t1(x, row_start, row_end) values (9, '1980-01-01 00:00:00', '1980-01-01 00:00:01');
+insert into test.t3(z, row_start, row_end) values (9, '1980-01-01 00:00:00', '1980-01-01 00:00:01');
+--error ER_SPECIFIC_ACCESS_DENIED_ERROR
+insert into test.t3 values (9, '1980-01-01 00:00:00', '1980-01-01 00:00:01');
 change_user root;
 --let $restart_parameters= --secure-timestamp=SUPER
 --source include/restart_mysqld.inc
 set @@system_versioning_insert_history= 1;
-insert into test.t1(x, row_start, row_end) values (10, '1980-01-01 00:00:00', '1980-01-01 00:00:01');
+insert into test.t3(z, row_start, row_end) values (10, '1980-01-01 00:00:00', '1980-01-01 00:00:01');
 change_user nobody;
 set @@system_versioning_insert_history= 1;
 --error ER_SPECIFIC_ACCESS_DENIED_ERROR
-insert into test.t1(x, row_start, row_end) values (7, '1980-01-01 00:00:00', '1980-01-01 00:00:01');
+insert into test.t3(z, row_start, row_end) values (7, '1980-01-01 00:00:00', '1980-01-01 00:00:01');
+--error ER_SPECIFIC_ACCESS_DENIED_ERROR
+insert into test.t3 values (7, '1980-01-01 00:00:00', '1980-01-01 00:00:01');
 change_user root;
 use test;
 --let $restart_parameters= --secure-timestamp=NO

sql/sql_base.cc

@@ -6313,10 +6313,7 @@ find_field_in_table(THD *thd, TABLE *table, const char *name, size_t length,
         thd->column_usage != COLUMNS_READ)
     {
       if (thd->vers_insert_history(field))
-      {
         DBUG_ASSERT(table->versioned());
-        table->vers_write= false;
-      }
       else if (field->invisible == INVISIBLE_SYSTEM)
         DBUG_RETURN((Field*)0);
     }
@@ -8842,12 +8839,10 @@ static bool vers_update_or_validate_fields(TABLE *table)
 {
   if (!table->versioned())
     return 0;
+  DBUG_ASSERT(table->vers_write);
 
-  if (table->vers_write)
+  if (table->vers_update_fields())
-  {
-    table->vers_update_fields();
     return 0;
-  }
 
   Field *row_start= table->vers_start_field();
   Field *row_end= table->vers_end_field();
@@ -8933,8 +8928,11 @@ fill_record(THD *thd, TABLE *table_arg, List<Item> &fields, List<Item> &values,
     if (table->next_number_field &&
         rfield->field_index ==  table->next_number_field->field_index)
       table->auto_increment_field_not_null= TRUE;
+
     const bool skip_sys_field= rfield->vers_sys_field() &&
-      (update || table->vers_write);
+      (update || table->versioned(VERS_TRX_ID) ||
+       !(thd->variables.option_bits & OPTION_INSERT_HISTORY));
+
     if ((rfield->vcol_info || skip_sys_field) &&
         !value->vcol_assignment_allowed_value() &&
         table->s->table_category != TABLE_CATEGORY_TEMPORARY)
@@ -8949,11 +8947,14 @@ fill_record(THD *thd, TABLE *table_arg, List<Item> &fields, List<Item> &values,
 
     if (rfield->stored_in_db())
     {
-      if (!skip_sys_field &&
+      if (!skip_sys_field)
-          unlikely(value->save_in_field(rfield, 0) < 0) && !ignore_errors)
       {
-        my_message(ER_UNKNOWN_ERROR, ER_THD(thd, ER_UNKNOWN_ERROR), MYF(0));
+        if (value->save_in_field(rfield, 0) < 0 && !ignore_errors)
-        goto err;
+        {
+          my_message(ER_UNKNOWN_ERROR, ER_THD(thd, ER_UNKNOWN_ERROR), MYF(0));
+          goto err;
+        }
+        rfield->set_has_explicit_value();
       }
       /*
         In sql MODE_SIMULTANEOUS_ASSIGNMENT,
@@ -8964,7 +8965,6 @@ fill_record(THD *thd, TABLE *table_arg, List<Item> &fields, List<Item> &values,
         rfield->move_field_offset((my_ptrdiff_t) (table->record[1] -
                                                   table->record[0]));
     }
-    rfield->set_has_explicit_value();
   }
 
   if (update && thd->variables.sql_mode & MODE_SIMULTANEOUS_ASSIGNMENT)
@@ -9217,7 +9217,8 @@ fill_record(THD *thd, TABLE *table, Field **ptr, List<Item> &values,
     DBUG_ASSERT(value);
 
     const bool skip_sys_field= field->vers_sys_field() &&
-                               table->vers_write;
+                      (table->versioned(VERS_TRX_ID) ||
+                       !(thd->variables.option_bits & OPTION_INSERT_HISTORY));
 
     if (field->field_index == autoinc_index)
       table->auto_increment_field_not_null= TRUE;
@@ -9229,10 +9230,11 @@ fill_record(THD *thd, TABLE *table, Field **ptr, List<Item> &values,
                           ER_WARNING_NON_DEFAULT_VALUE_FOR_GENERATED_COLUMN,
                           ER_THD(thd, ER_WARNING_NON_DEFAULT_VALUE_FOR_GENERATED_COLUMN),
                           field->field_name.str, table->s->table_name.str);
-      if (skip_sys_field)
-        continue;
     }
 
+    if (skip_sys_field)
+      continue;
+
     if (use_value)
       value->save_val(field);
     else

sql/sql_insert.cc

@@ -1749,11 +1749,10 @@ int mysql_prepare_insert(THD *thd, TABLE_LIST *table_list,
     if (duplic == DUP_REPLACE && table_list->set_insert_values(thd->mem_root))
       DBUG_RETURN(1);
 
-  Field *row_start= table->vers_start_field();
+    Field *row_start= table->vers_start_field();
-  Field *row_end= table->vers_end_field();
+    Field *row_end= table->vers_end_field();
-    if (!fields.elements && !(row_start->invisible && row_end->invisible) &&
+    if (!fields.elements && !(row_start->invisible && row_end->invisible))
-        thd->vers_insert_history(row_start))
+      thd->vers_insert_history(row_start); // check privileges
-      table->vers_write= false;
   }
 
   if (!select_insert)
@@ -4194,6 +4193,7 @@ int select_insert::send_data(List<Item> &values)
   bool error=0;
 
   thd->count_cuted_fields= CHECK_FIELD_WARN;	// Calculate cuted fields
+  table->reset_default_fields();
   store_values(values);
   if (table->default_field &&
       unlikely(table->update_default_fields(info.ignore)))

sql/sql_load.cc

@@ -1106,6 +1106,7 @@ read_sep_field(THD *thd, COPY_INFO &info, TABLE_LIST *table_list,
       }
     }
     restore_record(table, s->default_values);
+    table->reset_default_fields();
 
     while ((item= it++))
     {

sql/table.cc

@@ -9153,9 +9153,11 @@ bool TABLE::check_period_overlaps(const KEY &key,
   return true;
 }
 
-void TABLE::vers_update_fields()
+/* returns true if vers_end_field was updated */
+bool TABLE::vers_update_fields()
 {
-  if (versioned(VERS_TIMESTAMP))
+  bool res= false;
+  if (versioned(VERS_TIMESTAMP) && !vers_start_field()->has_explicit_value())
   {
     if (vers_start_field()->store_timestamp(in_use->query_start(),
                                           in_use->query_start_sec_part()))
@@ -9165,11 +9167,16 @@ void TABLE::vers_update_fields()
     vers_start_field()->set_has_explicit_value();
   }
 
-  vers_end_field()->set_max();
+  if (!versioned(VERS_TIMESTAMP) || !vers_end_field()->has_explicit_value())
-  vers_end_field()->set_has_explicit_value();
+  {
+    vers_end_field()->set_max();
+    vers_end_field()->set_has_explicit_value();
+    res= true;
+  }
 
   if (vfield)
     update_virtual_fields(file, VCOL_UPDATE_FOR_READ);
+  return res;
 }
 
 

sql/table.h

@@ -1842,7 +1842,7 @@ public:
   static bool check_period_overlaps(const KEY &key, const uchar *lhs, const uchar *rhs);
   int delete_row();
   /* Used in majority of DML (called from fill_record()) */
-  void vers_update_fields();
+  bool vers_update_fields();
   /* Used in DELETE, DUP REPLACE and insert history row */
   void vers_update_end();
   void find_constraint_correlated_indexes();