diff --git a/mysql-test/main/analyze.result b/mysql-test/main/analyze.result index a1332abd177..17325863acd 100644 --- a/mysql-test/main/analyze.result +++ b/mysql-test/main/analyze.result @@ -71,3 +71,31 @@ optimize table t1 extended; ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'extended' at line 1 drop table t1; End of 5.0 tests +# +# Test analyze of text column (not yet supported) +# +set optimizer_use_condition_selectivity=4; +set histogram_type='single_prec_hb'; +set histogram_size=255; +create table t1 (a int not null, t tinytext, tx text); +insert into t1 select seq+1, repeat('X',seq*5), repeat('X',seq*10) from seq_0_to_50; +insert into t1 select seq+100, repeat('X',5), "" from seq_1_to_10; +analyze table t1; +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze Warning Engine-independent statistics are not collected for column 't' +test.t1 analyze Warning Engine-independent statistics are not collected for column 'tx' +test.t1 analyze status OK +explain select count(*) from t1 where t='XXXXXX'; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 61 Using where +select column_name, min_value, max_value, hist_size from mysql.column_stats where table_name='t1'; +column_name min_value max_value hist_size +a 1 110 255 +drop table t1; +set use_stat_tables=default; +set histogram_type=default; +set histogram_size=default; +# +# End of 10.6 tests +# diff --git a/mysql-test/main/analyze.test b/mysql-test/main/analyze.test index 85a88158162..3c7179fbf19 100644 --- a/mysql-test/main/analyze.test +++ b/mysql-test/main/analyze.test @@ -1,3 +1,5 @@ +--source include/have_sequence.inc + # # Bug #10901 Analyze Table on new table destroys table # This is minimal test case to get error @@ -87,3 +89,28 @@ optimize table t1 extended; drop table t1; --echo End of 5.0 tests + +--echo # +--echo # Test analyze of text column (not yet supported) +--echo # + +set optimizer_use_condition_selectivity=4; +set histogram_type='single_prec_hb'; +set histogram_size=255; + +create table t1 (a int not null, t tinytext, tx text); +insert into t1 select seq+1, repeat('X',seq*5), repeat('X',seq*10) from seq_0_to_50; +insert into t1 select seq+100, repeat('X',5), "" from seq_1_to_10; +analyze table t1; +explain select count(*) from t1 where t='XXXXXX'; +select column_name, min_value, max_value, hist_size from mysql.column_stats where table_name='t1'; + +drop table t1; + +set use_stat_tables=default; +set histogram_type=default; +set histogram_size=default; + +--echo # +--echo # End of 10.6 tests +--echo # diff --git a/mysql-test/main/join.result b/mysql-test/main/join.result index 427b06aabbe..104e7a8c83c 100644 --- a/mysql-test/main/join.result +++ b/mysql-test/main/join.result @@ -1280,7 +1280,7 @@ pk v pk v SHOW STATUS LIKE 'Handler_read_%'; Variable_name Value Handler_read_first 0 -Handler_read_key 14 +Handler_read_key 1 Handler_read_last 0 Handler_read_next 0 Handler_read_prev 0 diff --git a/mysql-test/main/join_outer.result b/mysql-test/main/join_outer.result index 6bcaee31e54..9722211fadf 100644 --- a/mysql-test/main/join_outer.result +++ b/mysql-test/main/join_outer.result @@ -1804,7 +1804,7 @@ sum(t3.b) show status like "handler_read%"; Variable_name Value Handler_read_first 0 -Handler_read_key 13 +Handler_read_key 4 Handler_read_last 0 Handler_read_next 5 Handler_read_prev 0 @@ -1819,7 +1819,7 @@ sum(t3.b) show status like "handler_read%"; Variable_name Value Handler_read_first 0 -Handler_read_key 7 +Handler_read_key 4 Handler_read_last 0 Handler_read_next 5 Handler_read_prev 0 diff --git a/mysql-test/main/join_outer_jcl6.result b/mysql-test/main/join_outer_jcl6.result index 3d73ebdc9ba..26865a72d47 100644 --- a/mysql-test/main/join_outer_jcl6.result +++ b/mysql-test/main/join_outer_jcl6.result @@ -1811,7 +1811,7 @@ sum(t3.b) show status like "handler_read%"; Variable_name Value Handler_read_first 0 -Handler_read_key 13 +Handler_read_key 4 Handler_read_last 0 Handler_read_next 5 Handler_read_prev 0 @@ -1826,7 +1826,7 @@ sum(t3.b) show status like "handler_read%"; Variable_name Value Handler_read_first 0 -Handler_read_key 7 +Handler_read_key 4 Handler_read_last 0 Handler_read_next 5 Handler_read_prev 0 diff --git a/mysql-test/main/partition_explicit_prune.result b/mysql-test/main/partition_explicit_prune.result index 07af2d58a42..8b49210d119 100644 --- a/mysql-test/main/partition_explicit_prune.result +++ b/mysql-test/main/partition_explicit_prune.result @@ -350,7 +350,6 @@ WHERE VARIABLE_NAME LIKE 'HANDLER_%' AND VARIABLE_VALUE > 0; VARIABLE_NAME VARIABLE_VALUE HANDLER_COMMIT 1 HANDLER_READ_FIRST 1 -HANDLER_READ_KEY 8 HANDLER_TMP_WRITE 24 # Should be 1 commit # 4 locks (1 ha_partition + 1 ha_innobase) x 2 (lock/unlock) @@ -777,7 +776,7 @@ SELECT * FROM INFORMATION_SCHEMA.SESSION_STATUS WHERE VARIABLE_NAME LIKE 'HANDLER_%' AND VARIABLE_VALUE > 0; VARIABLE_NAME VARIABLE_VALUE HANDLER_COMMIT 1 -HANDLER_READ_KEY 8 +HANDLER_READ_KEY 6 HANDLER_READ_RND_NEXT 2 HANDLER_TMP_WRITE 24 HANDLER_UPDATE 2 diff --git a/mysql-test/main/stat_tables.result b/mysql-test/main/stat_tables.result index 379e9737e1c..1c43dd268a8 100644 --- a/mysql-test/main/stat_tables.result +++ b/mysql-test/main/stat_tables.result @@ -903,4 +903,82 @@ id select_type table type possible_keys key key_len ref rows r_rows filtered r_f 1 SIMPLE t1 ALL NULL NULL NULL NULL 10 10.00 2.78 10.00 Using where drop table t1; set @@global.histogram_size=@save_histogram_size; +# # End of 10.4 tests +# +# +# MDEV-29693 ANALYZE TABLE still flushes table definition cache +# when engine-independent statistics is used +# +create table t1 (a int); +insert into t1 select seq from seq_0_to_99; +analyze table t1 persistent for all; +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status OK +analyze table t1 persistent for all; +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status Table is already up to date +explain extended select count(*) from t1 where a < 50; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 100 50.51 Using where +Warnings: +Note 1003 select count(0) AS `count(*)` from `test`.`t1` where `test`.`t1`.`a` < 50 +connect con1, localhost, root,,; +connection con1; +explain extended select count(*) from t1 where a < 50; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 100 50.51 Using where +Warnings: +Note 1003 select count(0) AS `count(*)` from `test`.`t1` where `test`.`t1`.`a` < 50 +connection default; +update t1 set a= a +100; +# Explain shows outdated statistics: +explain extended select count(*) from t1 where a < 50; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 100 50.51 Using where +Warnings: +Note 1003 select count(0) AS `count(*)` from `test`.`t1` where `test`.`t1`.`a` < 50 +connection con1; +explain extended select count(*) from t1 where a < 50; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 100 50.51 Using where +Warnings: +Note 1003 select count(0) AS `count(*)` from `test`.`t1` where `test`.`t1`.`a` < 50 +connection default; +analyze table t1 persistent for all; +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status OK +# Now explain shows updated statistics: +explain extended select count(*) from t1 where a < 50; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 100 1.00 Using where +Warnings: +Note 1003 select count(0) AS `count(*)` from `test`.`t1` where `test`.`t1`.`a` < 50 +connection con1; +explain extended select count(*) from t1 where a < 50; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 100 1.00 Using where +Warnings: +Note 1003 select count(0) AS `count(*)` from `test`.`t1` where `test`.`t1`.`a` < 50 +connection con1; +# Run update and analyze in con1: +update t1 set a= a - 150; +analyze table t1 persistent for all; +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status OK +connection default; +# Explain shows updated statistics: +explain extended select count(*) from t1 where a < 50; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 100 99.22 Using where +Warnings: +Note 1003 select count(0) AS `count(*)` from `test`.`t1` where `test`.`t1`.`a` < 50 +disconnect con1; +drop table t1; +# +# End of 10.6 tests +# diff --git a/mysql-test/main/stat_tables.test b/mysql-test/main/stat_tables.test index 7488ccb6877..4d4a969f49c 100644 --- a/mysql-test/main/stat_tables.test +++ b/mysql-test/main/stat_tables.test @@ -1,9 +1,9 @@ # Tests will be skipped for the view protocol because the view protocol creates # an additional util connection and other statistics data --- source include/no_view_protocol.inc - +--source include/no_view_protocol.inc --source include/have_stat_tables.inc --source include/have_partition.inc +--source include/have_sequence.inc select @@global.use_stat_tables; select @@session.use_stat_tables; @@ -640,4 +640,64 @@ drop table t1; set @@global.histogram_size=@save_histogram_size; +--echo # --echo # End of 10.4 tests +--echo # + +--echo # +--echo # MDEV-29693 ANALYZE TABLE still flushes table definition cache +--echo # when engine-independent statistics is used +--echo # + +create table t1 (a int); +insert into t1 select seq from seq_0_to_99; +analyze table t1 persistent for all; +analyze table t1 persistent for all; + +explain extended select count(*) from t1 where a < 50; + +connect (con1, localhost, root,,); +--connection con1 +explain extended select count(*) from t1 where a < 50; + +let $open_tables=`select variable_value from information_schema.global_status where variable_name="OPENED_TABLES"`; + +--connection default +update t1 set a= a +100; + +--echo # Explain shows outdated statistics: +explain extended select count(*) from t1 where a < 50; +--connection con1 +explain extended select count(*) from t1 where a < 50; + +--connection default +analyze table t1 persistent for all; +--echo # Now explain shows updated statistics: +explain extended select count(*) from t1 where a < 50; +--connection con1 +explain extended select count(*) from t1 where a < 50; + +--connection con1 +--echo # Run update and analyze in con1: +update t1 set a= a - 150; +analyze table t1 persistent for all; + +--connection default +--echo # Explain shows updated statistics: +explain extended select count(*) from t1 where a < 50; + +disconnect con1; + +let $new_open_tables=`select variable_value from information_schema.global_status where variable_name="OPENED_TABLES"`; + +if ($open_tables != $new_open_tables) +{ +--let $diff=`select $new_open_tables - $open_tables` +--echo "Fail: Test opened $diff new tables, 0 was expected" +} + +drop table t1; + +--echo # +--echo # End of 10.6 tests +--echo # diff --git a/mysql-test/main/stat_tables_flush.result b/mysql-test/main/stat_tables_flush.result new file mode 100644 index 00000000000..9a88d5d388d --- /dev/null +++ b/mysql-test/main/stat_tables_flush.result @@ -0,0 +1,91 @@ +# +# Check that ANALYZE TABLE is remembered by MyISAM and Aria +# +create table t1 (a int) engine=myisam; +insert into t1 select seq from seq_0_to_99; +analyze table t1 persistent for all; +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status OK +flush tables; +analyze table t1 persistent for all; +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status Table is already up to date +update t1 set a=100 where a=1; +analyze table t1 persistent for all; +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status OK +update t1 set a=100 where a=2; +flush tables; +analyze table t1 persistent for all; +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status OK +# Aria transactional=0 +ALTER TABLE t1 ENGINE=aria transactional=0; +analyze table t1 persistent for all; +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status OK +update t1 set a=100 where a=10; +analyze table t1 persistent for all; +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status OK +analyze table t1 persistent for all; +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status Table is already up to date +flush tables; +analyze table t1 persistent for all; +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status Table is already up to date +update t1 set a=100 where a=11; +analyze table t1 persistent for all; +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status OK +update t1 set a=100 where a=12; +flush tables; +analyze table t1 persistent for all; +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status Table is already up to date +# Aria transactional=1 +ALTER TABLE t1 ENGINE=aria transactional=1; +analyze table t1 persistent for all; +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status OK +update t1 set a=100 where a=20; +analyze table t1 persistent for all; +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status OK +analyze table t1 persistent for all; +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status Table is already up to date +flush tables; +analyze table t1 persistent for all; +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status Table is already up to date +update t1 set a=100 where a=21; +analyze table t1 persistent for all; +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status OK +update t1 set a=100 where a=22; +flush tables; +analyze table t1 persistent for all; +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status OK +drop table t1; +# +# End of 10.5 tests +# diff --git a/mysql-test/main/stat_tables_flush.test b/mysql-test/main/stat_tables_flush.test new file mode 100644 index 00000000000..abbbb0d6467 --- /dev/null +++ b/mysql-test/main/stat_tables_flush.test @@ -0,0 +1,50 @@ +--source include/have_sequence.inc + +--echo # +--echo # Check that ANALYZE TABLE is remembered by MyISAM and Aria +--echo # + +create table t1 (a int) engine=myisam; +insert into t1 select seq from seq_0_to_99; +analyze table t1 persistent for all; +flush tables; +analyze table t1 persistent for all; +update t1 set a=100 where a=1; +analyze table t1 persistent for all; +update t1 set a=100 where a=2; +flush tables; +analyze table t1 persistent for all; + +--echo # Aria transactional=0 +ALTER TABLE t1 ENGINE=aria transactional=0; +analyze table t1 persistent for all; +update t1 set a=100 where a=10; +analyze table t1 persistent for all; +analyze table t1 persistent for all; +flush tables; +analyze table t1 persistent for all; +update t1 set a=100 where a=11; +analyze table t1 persistent for all; +update t1 set a=100 where a=12; +flush tables; +analyze table t1 persistent for all; + +--echo # Aria transactional=1 + +ALTER TABLE t1 ENGINE=aria transactional=1; +analyze table t1 persistent for all; +update t1 set a=100 where a=20; +analyze table t1 persistent for all; +analyze table t1 persistent for all; +flush tables; +analyze table t1 persistent for all; +update t1 set a=100 where a=21; +analyze table t1 persistent for all; +update t1 set a=100 where a=22; +flush tables; +analyze table t1 persistent for all; +drop table t1; + +--echo # +--echo # End of 10.5 tests +--echo # diff --git a/mysql-test/main/stat_tables_innodb.result b/mysql-test/main/stat_tables_innodb.result index 5b62f228b1f..163a417f810 100644 --- a/mysql-test/main/stat_tables_innodb.result +++ b/mysql-test/main/stat_tables_innodb.result @@ -935,7 +935,85 @@ id select_type table type possible_keys key key_len ref rows r_rows filtered r_f 1 SIMPLE t1 ALL NULL NULL NULL NULL 10 10.00 2.78 10.00 Using where drop table t1; set @@global.histogram_size=@save_histogram_size; +# # End of 10.4 tests +# +# +# MDEV-29693 ANALYZE TABLE still flushes table definition cache +# when engine-independent statistics is used +# +create table t1 (a int); +insert into t1 select seq from seq_0_to_99; +analyze table t1 persistent for all; +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status OK +analyze table t1 persistent for all; +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status OK +explain extended select count(*) from t1 where a < 50; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 100 50.51 Using where +Warnings: +Note 1003 select count(0) AS `count(*)` from `test`.`t1` where `test`.`t1`.`a` < 50 +connect con1, localhost, root,,; +connection con1; +explain extended select count(*) from t1 where a < 50; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 100 50.51 Using where +Warnings: +Note 1003 select count(0) AS `count(*)` from `test`.`t1` where `test`.`t1`.`a` < 50 +connection default; +update t1 set a= a +100; +# Explain shows outdated statistics: +explain extended select count(*) from t1 where a < 50; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 100 50.51 Using where +Warnings: +Note 1003 select count(0) AS `count(*)` from `test`.`t1` where `test`.`t1`.`a` < 50 +connection con1; +explain extended select count(*) from t1 where a < 50; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 100 50.51 Using where +Warnings: +Note 1003 select count(0) AS `count(*)` from `test`.`t1` where `test`.`t1`.`a` < 50 +connection default; +analyze table t1 persistent for all; +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status OK +# Now explain shows updated statistics: +explain extended select count(*) from t1 where a < 50; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 100 1.00 Using where +Warnings: +Note 1003 select count(0) AS `count(*)` from `test`.`t1` where `test`.`t1`.`a` < 50 +connection con1; +explain extended select count(*) from t1 where a < 50; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 100 1.00 Using where +Warnings: +Note 1003 select count(0) AS `count(*)` from `test`.`t1` where `test`.`t1`.`a` < 50 +connection con1; +# Run update and analyze in con1: +update t1 set a= a - 150; +analyze table t1 persistent for all; +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status OK +connection default; +# Explain shows updated statistics: +explain extended select count(*) from t1 where a < 50; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 100 99.22 Using where +Warnings: +Note 1003 select count(0) AS `count(*)` from `test`.`t1` where `test`.`t1`.`a` < 50 +disconnect con1; +drop table t1; +# +# End of 10.6 tests +# set global innodb_stats_persistent= @innodb_stats_persistent_save; set global innodb_stats_persistent_sample_pages= @innodb_stats_persistent_sample_pages_save; diff --git a/sql/field.cc b/sql/field.cc index 4a7ee0f3ce0..3d11d0b878f 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -1997,13 +1997,35 @@ int Field::store_to_statistical_minmax_field(Field *field, String *val) } -int Field::store_from_statistical_minmax_field(Field *stat_field, String *str) +int Field::store_from_statistical_minmax_field(Field *stat_field, String *str, + MEM_ROOT *mem) { stat_field->val_str(str); return store_text(str->ptr(), str->length(), &my_charset_bin); } +/* + Same as above, but store the string in the statistics mem_root to make it + easy to free everything by just freeing the mem_root. +*/ + +int Field_blob::store_from_statistical_minmax_field(Field *stat_field, + String *str, + MEM_ROOT *mem) +{ + String *tmp= stat_field->val_str(str); + uchar *ptr; + if (!(ptr= (uchar*) memdup_root(mem, tmp->ptr(), tmp->length()))) + { + set_ptr((uint32) 0, NULL); + return 1; + } + set_ptr(tmp->length(), ptr); + return 0; +} + + /** Pack the field into a format suitable for storage and transfer. diff --git a/sql/field.h b/sql/field.h index 9534cfabeed..e4e88408f8a 100644 --- a/sql/field.h +++ b/sql/field.h @@ -1011,7 +1011,8 @@ public: field statistical table field str value buffer */ - virtual int store_from_statistical_minmax_field(Field *field, String *str); + virtual int store_from_statistical_minmax_field(Field *field, String *str, + MEM_ROOT *mem); #ifdef HAVE_MEM_CHECK /** @@ -4469,6 +4470,8 @@ public: } bool make_empty_rec_store_default_value(THD *thd, Item *item) override; int store(const char *to, size_t length, CHARSET_INFO *charset) override; + int store_from_statistical_minmax_field(Field *stat_field, String *str, + MEM_ROOT *mem) override; using Field_str::store; void hash_not_null(Hasher *hasher) override; double val_real() override; diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 843f9b5cbae..634e4e91bd7 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -919,6 +919,7 @@ PSI_mutex_key key_LOCK_gtid_waiting; PSI_mutex_key key_LOCK_after_binlog_sync; PSI_mutex_key key_LOCK_prepare_ordered, key_LOCK_commit_ordered; PSI_mutex_key key_TABLE_SHARE_LOCK_share; +PSI_mutex_key key_TABLE_SHARE_LOCK_statistics; PSI_mutex_key key_LOCK_ack_receiver; PSI_mutex_key key_TABLE_SHARE_LOCK_rotation; @@ -986,6 +987,7 @@ static PSI_mutex_info all_server_mutexes[]= { &key_structure_guard_mutex, "Query_cache::structure_guard_mutex", 0}, { &key_TABLE_SHARE_LOCK_ha_data, "TABLE_SHARE::LOCK_ha_data", 0}, { &key_TABLE_SHARE_LOCK_share, "TABLE_SHARE::LOCK_share", 0}, + { &key_TABLE_SHARE_LOCK_statistics, "TABLE_SHARE::LOCK_statistics", 0}, { &key_TABLE_SHARE_LOCK_rotation, "TABLE_SHARE::LOCK_rotation", 0}, { &key_LOCK_error_messages, "LOCK_error_messages", PSI_FLAG_GLOBAL}, { &key_LOCK_prepare_ordered, "LOCK_prepare_ordered", PSI_FLAG_GLOBAL}, diff --git a/sql/mysqld.h b/sql/mysqld.h index 3014da58b3d..cab7dafdc19 100644 --- a/sql/mysqld.h +++ b/sql/mysqld.h @@ -334,6 +334,7 @@ extern PSI_mutex_key key_BINLOG_LOCK_index, key_BINLOG_LOCK_xid_list, key_relay_log_info_log_space_lock, key_relay_log_info_run_lock, key_rpl_group_info_sleep_lock, key_structure_guard_mutex, key_TABLE_SHARE_LOCK_ha_data, + key_TABLE_SHARE_LOCK_statistics, key_LOCK_start_thread, key_LOCK_error_messages, key_PARTITION_LOCK_auto_inc; diff --git a/sql/sql_admin.cc b/sql/sql_admin.cc index 00d7e5efecd..b7a845f04c9 100644 --- a/sql/sql_admin.cc +++ b/sql/sql_admin.cc @@ -923,8 +923,14 @@ static bool mysql_admin_table(THD* thd, TABLE_LIST* tables, bitmap_clear_all(tab->read_set); for (uint fields= 0; *field_ptr; field_ptr++, fields++) { + /* + Note that type() always return MYSQL_TYPE_BLOB for + all blob types. Another function needs to be added + if we in the future want to distingush between blob + types here. + */ enum enum_field_types type= (*field_ptr)->type(); - if (type < MYSQL_TYPE_MEDIUM_BLOB || + if (type < MYSQL_TYPE_TINY_BLOB || type > MYSQL_TYPE_BLOB) tab->field[fields]->register_field_in_read_map(); else @@ -952,7 +958,7 @@ static bool mysql_admin_table(THD* thd, TABLE_LIST* tables, } pos--; enum enum_field_types type= tab->field[pos]->type(); - if (type < MYSQL_TYPE_MEDIUM_BLOB || + if (type < MYSQL_TYPE_TINY_BLOB || type > MYSQL_TYPE_BLOB) tab->field[pos]->register_field_in_read_map(); else @@ -984,6 +990,8 @@ static bool mysql_admin_table(THD* thd, TABLE_LIST* tables, tab->keys_in_use_for_query.set_bit(--pos); } } + /* Ensure that number of records are updated */ + table->table->file->info(HA_STATUS_VARIABLE); if (!(compl_result_code= alloc_statistics_for_table(thd, table->table)) && !(compl_result_code= @@ -1279,13 +1287,8 @@ send_result_message: if (table->table && !table->view) { - /* - Don't skip flushing if we are collecting EITS statistics. - */ - const bool skip_flush= - (operator_func == &handler::ha_analyze) && - (table->table->file->ha_table_flags() & HA_ONLINE_ANALYZE) && - !collect_eis; + /* Skip FLUSH TABLES if we are doing analyze */ + const bool skip_flush= (operator_func == &handler::ha_analyze); if (table->table->s->tmp_table) { /* @@ -1305,6 +1308,13 @@ send_result_message: table->table= 0; // For query cache query_cache_invalidate3(thd, table, 0); } + else if (collect_eis && skip_flush && compl_result_code == HA_ADMIN_OK) + { + TABLE_LIST *save_next_global= table->next_global; + table->next_global= 0; + read_statistics_for_tables(thd, table, true /* force_reload */); + table->next_global= save_next_global; + } } /* Error path, a admin command failed. */ if (thd->transaction_rollback_request || fatal_error) diff --git a/sql/sql_base.cc b/sql/sql_base.cc index 691925c82b2..4299d631059 100644 --- a/sql/sql_base.cc +++ b/sql/sql_base.cc @@ -5323,7 +5323,8 @@ bool open_and_lock_tables(THD *thd, const DDL_options_st &options, goto err; /* Don't read statistics tables when opening internal tables */ - if (!(flags & MYSQL_OPEN_IGNORE_LOGGING_FORMAT)) + if (!(flags & (MYSQL_OPEN_IGNORE_LOGGING_FORMAT | + MYSQL_OPEN_IGNORE_ENGINE_STATS))) (void) read_statistics_for_tables_if_needed(thd, tables); if (derived) diff --git a/sql/sql_base.h b/sql/sql_base.h index 0bfef1ca7ee..a67d28f47c9 100644 --- a/sql/sql_base.h +++ b/sql/sql_base.h @@ -130,6 +130,9 @@ TABLE *open_ltable(THD *thd, TABLE_LIST *table_list, thr_lock_type update, */ #define MYSQL_OPEN_IGNORE_LOGGING_FORMAT 0x20000 +/* Don't use statistics tables */ +#define MYSQL_OPEN_IGNORE_ENGINE_STATS 0x40000 + /** Please refer to the internals manual. */ #define MYSQL_OPEN_REOPEN (MYSQL_OPEN_IGNORE_FLUSH |\ MYSQL_OPEN_IGNORE_GLOBAL_READ_LOCK |\ diff --git a/sql/sql_insert.cc b/sql/sql_insert.cc index 132be65a848..73044c60c20 100644 --- a/sql/sql_insert.cc +++ b/sql/sql_insert.cc @@ -580,7 +580,8 @@ bool open_and_lock_for_insert_delayed(THD *thd, TABLE_LIST *table_list) Open tables used for sub-selects or in stored functions, will also cache these functions. */ - if (open_and_lock_tables(thd, table_list->next_global, TRUE, 0)) + if (open_and_lock_tables(thd, table_list->next_global, TRUE, + MYSQL_OPEN_IGNORE_ENGINE_STATS)) { end_delayed_insert(thd); error= TRUE; @@ -2751,6 +2752,9 @@ TABLE *Delayed_insert::get_local_table(THD* client_thd) /* Ensure we don't use the table list of the original table */ copy->pos_in_table_list= 0; + /* We don't need statistics for insert delayed */ + copy->stats_cb= 0; + /* Make a copy of all fields. The copied fields need to point into the copied record. This is done diff --git a/sql/sql_show.cc b/sql/sql_show.cc index e90c52ceabd..d12dbcebde5 100644 --- a/sql/sql_show.cc +++ b/sql/sql_show.cc @@ -6750,7 +6750,7 @@ static int get_schema_stat_record(THD *thd, TABLE_LIST *tables, KEY *key_info=show_table->s->key_info; if (show_table->file) { - (void) read_statistics_for_tables(thd, tables); + (void) read_statistics_for_tables(thd, tables, false); show_table->file->info(HA_STATUS_VARIABLE | HA_STATUS_NO_LOCK | HA_STATUS_CONST | diff --git a/sql/sql_statistics.cc b/sql/sql_statistics.cc index d0b6ac20d1d..e09370f94c4 100644 --- a/sql/sql_statistics.cc +++ b/sql/sql_statistics.cc @@ -62,7 +62,7 @@ /* Currently there are only 3 persistent statistical tables */ static const uint STATISTICS_TABLES= 3; -/* +/* The names of the statistical tables in this array must correspond the definitions of the tables in the file ../scripts/mysql_system_tables.sql */ @@ -74,6 +74,21 @@ static const LEX_CSTRING stat_table_name[STATISTICS_TABLES]= }; +TABLE_STATISTICS_CB::TABLE_STATISTICS_CB(): + usage_count(0), table_stats(0), total_hist_size(0), + stats_available(TABLE_STAT_NO_STATS) +{ + init_sql_alloc(PSI_INSTRUMENT_ME, &mem_root, TABLE_ALLOC_BLOCK_SIZE, 0, + MYF(0)); +} + +TABLE_STATISTICS_CB::~TABLE_STATISTICS_CB() +{ + DBUG_ASSERT(usage_count == 0); + free_root(&mem_root, MYF(0)); +} + + /** @details The function builds a list of TABLE_LIST elements for system statistical @@ -349,8 +364,7 @@ public: Reading statistical data from a statistical table is performed by the following pattern. First a table dependent method sets the values of the - the fields that comprise the lookup key. Then an implementation of the - method get_stat_values() declared in Stat_table as a pure virtual method + the fields that comprise the lookup key. Then, get_stat_values(...) call finds the row from the statistical table by the set key. If the row is found the values of statistical fields are read from this row and are distributed in the internal structures. @@ -447,8 +461,8 @@ protected: KEY *stat_key_info; /* Structure for the index to access stat_table */ /* Table for which statistical data is read / updated */ - TABLE *table; - TABLE_SHARE *table_share; /* Table share for 'table */ + const TABLE *table; + const TABLE_SHARE *table_share; /* Table share for 'table */ const LEX_CSTRING *db_name; /* Name of the database containing 'table' */ const LEX_CSTRING *table_name; /* Name of the table 'table' */ @@ -485,7 +499,7 @@ public: statistics has been collected. */ - Stat_table(TABLE *stat, TABLE *tab) + Stat_table(TABLE *stat, const TABLE *tab) :stat_table(stat), table(tab) { table_share= tab->s; @@ -528,7 +542,8 @@ public: The method is called by the update_table_name_key_parts function. */ - virtual void change_full_table_name(const LEX_CSTRING *db, const LEX_CSTRING *tab)= 0; + virtual void change_full_table_name(const LEX_CSTRING *db, + const LEX_CSTRING *tab)= 0; /** @@ -547,19 +562,6 @@ public: virtual void store_stat_fields()= 0; - /** - @brief - Read statistical data from fields of the statistical table - - @details - This is a purely virtual method. - The implementation for any derived read shall read the appropriate - statistical data from the corresponding fields of stat_table. - */ - - virtual void get_stat_values()= 0; - - /** @brief Find a record in the statistical table by a primary key @@ -746,7 +748,8 @@ private: table_name_field= stat_table->field[TABLE_STAT_TABLE_NAME]; } - void change_full_table_name(const LEX_CSTRING *db, const LEX_CSTRING *tab) + void change_full_table_name(const LEX_CSTRING *db, + const LEX_CSTRING *tab) override { db_name_field->store(db->str, db->length, system_charset_info); table_name_field->store(tab->str, tab->length, system_charset_info); @@ -762,7 +765,7 @@ public: must be passed as a value for the parameter 'stat'. */ - Table_stat(TABLE *stat, TABLE *tab) :Stat_table(stat, tab) + Table_stat(TABLE *stat, const TABLE *tab) :Stat_table(stat, tab) { common_init_table_stat(); } @@ -816,7 +819,7 @@ public: the field write_stat.cardinality' from the TABLE structure for 'table'. */ - void store_stat_fields() + void store_stat_fields() override { Field *stat_field= stat_table->field[TABLE_STAT_CARDINALITY]; if (table->collected_stats->cardinality_is_null) @@ -834,21 +837,19 @@ public: Read statistical data from statistical fields of table_stat @details - This implementation of a purely virtual method first looks for a record - the statistical table table_stat by its primary key set the record - buffer with the help of Table_stat::set_key_fields. Then, if the row is - found the function reads the value of the column 'cardinality' of the table - table_stat and sets the value of the flag read_stat.cardinality_is_null - and the value of the field read_stat.cardinality' from the TABLE structure - for 'table' accordingly. - */ + Find a record in mysql.table_stat that has statistics for this table. + We search for record using a PK lookup. The lookup values are in the stat + table's record buffer, they were put there by Table_stat::set_key_fields. - void get_stat_values() + The result is stored in *read_stats. + */ + + bool get_stat_values(Table_statistics *read_stats) { - Table_statistics *read_stats= table_share->stats_cb.table_stats; + bool res; read_stats->cardinality_is_null= TRUE; read_stats->cardinality= 0; - if (find_stat()) + if ((res= find_stat())) { Field *stat_field= stat_table->field[TABLE_STAT_CARDINALITY]; if (!stat_field->is_null()) @@ -857,8 +858,8 @@ public: read_stats->cardinality= stat_field->val_int(); } } + return res; } - }; @@ -890,7 +891,8 @@ private: column_name_field= stat_table->field[COLUMN_STAT_COLUMN_NAME]; } - void change_full_table_name(const LEX_CSTRING *db, const LEX_CSTRING *tab) + void change_full_table_name(const LEX_CSTRING *db, + const LEX_CSTRING *tab) override { db_name_field->store(db->str, db->length, system_charset_info); table_name_field->store(tab->str, tab->length, system_charset_info); @@ -906,7 +908,7 @@ public: column_stats must be passed as a value for the parameter 'stat'. */ - Column_stat(TABLE *stat, TABLE *tab) :Stat_table(stat, tab) + Column_stat(TABLE *stat, const TABLE *tab) :Stat_table(stat, tab) { common_init_column_stat_table(); } @@ -1019,7 +1021,7 @@ public: length of the column. */ - void store_stat_fields() + void store_stat_fields() override { StringBuffer val; @@ -1085,120 +1087,107 @@ public: Read statistical data from statistical fields of column_stats @details - This implementation of a purely virtual method first looks for a record - in the statistical table column_stats by its primary key set in the record - buffer with the help of Column_stat::set_key_fields. Then, if the row is + Find a record in mysql.column_stats that has statistics for this column. + We search for record using a PK lookup. The lookup values are in the stat + table's record buffer. Then, if the row is found, the function reads the values of the columns 'min_value', 'max_value', 'nulls_ratio', 'avg_length', 'avg_frequency', 'hist_size' and - 'hist_type" of the table column_stat and sets accordingly the value of - the bitmap read_stat.column_stat_nulls' and the values of the fields - min_value, max_value, nulls_ratio, avg_length, avg_frequency, hist_size and - hist_type of the structure read_stat from the Field structure for the field - 'table_field'. - */ + 'hist_type" of the table column_stat and sets the members of *read_stats + accordingly. + */ - void get_stat_values() + bool get_stat_values(Column_statistics *read_stats, MEM_ROOT *mem_root, + bool want_histograms) { - table_field->read_stats->set_all_nulls(); + bool res; + read_stats->set_all_nulls(); - if (table_field->read_stats->min_value) - table_field->read_stats->min_value->set_null(); - if (table_field->read_stats->max_value) - table_field->read_stats->max_value->set_null(); + if (read_stats->min_value) + read_stats->min_value->set_null(); + if (read_stats->max_value) + read_stats->max_value->set_null(); - if (find_stat()) + if ((res= find_stat())) { char buff[MAX_FIELD_WIDTH]; String val(buff, sizeof(buff), &my_charset_bin); for (uint i= COLUMN_STAT_MIN_VALUE; i <= COLUMN_STAT_HIST_TYPE; i++) - { + { Field *stat_field= stat_table->field[i]; if (!stat_field->is_null() && (i > COLUMN_STAT_MAX_VALUE || (i == COLUMN_STAT_MIN_VALUE && - table_field->read_stats->min_value) || + read_stats->min_value) || (i == COLUMN_STAT_MAX_VALUE && - table_field->read_stats->max_value))) + read_stats->max_value))) { - table_field->read_stats->set_not_null(i); + read_stats->set_not_null(i); switch (i) { case COLUMN_STAT_MIN_VALUE: { - Field *field= table_field->read_stats->min_value; + Field *field= read_stats->min_value; field->set_notnull(); if (table_field->type() == MYSQL_TYPE_BIT) field->store(stat_field->val_int(), true); else - field->store_from_statistical_minmax_field(stat_field, &val); + field->store_from_statistical_minmax_field(stat_field, &val, + mem_root); break; } case COLUMN_STAT_MAX_VALUE: { - Field *field= table_field->read_stats->max_value; + Field *field= read_stats->max_value; field->set_notnull(); if (table_field->type() == MYSQL_TYPE_BIT) field->store(stat_field->val_int(), true); else - field->store_from_statistical_minmax_field(stat_field, &val); + field->store_from_statistical_minmax_field(stat_field, &val, + mem_root); break; } case COLUMN_STAT_NULLS_RATIO: - table_field->read_stats->set_nulls_ratio(stat_field->val_real()); + read_stats->set_nulls_ratio(stat_field->val_real()); break; case COLUMN_STAT_AVG_LENGTH: - table_field->read_stats->set_avg_length(stat_field->val_real()); + read_stats->set_avg_length(stat_field->val_real()); break; case COLUMN_STAT_AVG_FREQUENCY: - table_field->read_stats->set_avg_frequency(stat_field->val_real()); + read_stats->set_avg_frequency(stat_field->val_real()); break; case COLUMN_STAT_HIST_SIZE: - table_field->read_stats->histogram.set_size(stat_field->val_int()); - break; + read_stats->histogram.set_size(stat_field->val_int()); + break; case COLUMN_STAT_HIST_TYPE: Histogram_type hist_type= (Histogram_type) (stat_field->val_int() - 1); - table_field->read_stats->histogram.set_type(hist_type); - break; + read_stats->histogram.set_type(hist_type); + break; } } } + + if (want_histograms) + { + char buff[MAX_FIELD_WIDTH]; + String val(buff, sizeof(buff), &my_charset_bin), *result; + uint hist_size; + if ((hist_size= read_stats->histogram.get_size())) + { + uchar *histogram_buf= (uchar *) alloc_root(mem_root, hist_size); + if (!histogram_buf) + return false; /* purecov: inspected */ + read_stats->histogram.set_values(histogram_buf); + read_stats->set_not_null(COLUMN_STAT_HISTOGRAM); + result= stat_table->field[COLUMN_STAT_HISTOGRAM]->val_str(&val); + memcpy(histogram_buf, result->ptr(), hist_size); + } + } } + return res; } - - - /** - @brief - Read histogram from of column_stats - - @details - This method first looks for a record in the statistical table column_stats - by its primary key set the record buffer with the help of - Column_stat::set_key_fields. Then, if the row is found, the function reads - the value of the column 'histogram' of the table column_stat and sets - accordingly the corresponding bit in the bitmap read_stat.column_stat_nulls. - The method assumes that the value of histogram size and the pointer to - the histogram location has been already set in the fields size and values - of read_stats->histogram. - */ - - void get_histogram_value() - { - if (find_stat()) - { - char buff[MAX_FIELD_WIDTH]; - String val(buff, sizeof(buff), &my_charset_bin); - uint fldno= COLUMN_STAT_HISTOGRAM; - Field *stat_field= stat_table->field[fldno]; - table_field->read_stats->set_not_null(fldno); - stat_field->val_str(&val); - memcpy(table_field->read_stats->histogram.get_values(), - val.ptr(), table_field->read_stats->histogram.get_size()); - } - } - }; @@ -1233,7 +1222,8 @@ private: prefix_arity_field= stat_table->field[INDEX_STAT_PREFIX_ARITY]; } - void change_full_table_name(const LEX_CSTRING *db, const LEX_CSTRING *tab) + void change_full_table_name(const LEX_CSTRING *db, + const LEX_CSTRING *tab) override { db_name_field->store(db->str, db->length, system_charset_info); table_name_field->store(tab->str, tab->length, system_charset_info); @@ -1251,7 +1241,7 @@ public: for the parameter 'stat'. */ - Index_stat(TABLE *stat, TABLE*tab) :Stat_table(stat, tab) + Index_stat(TABLE *stat, const TABLE *tab) :Stat_table(stat, tab) { common_init_index_stat_table(); } @@ -1355,7 +1345,7 @@ public: equal to 0, the value of the column is set to NULL. */ - void store_stat_fields() + void store_stat_fields() override { Field *stat_field= stat_table->field[INDEX_STAT_AVG_FREQUENCY]; double avg_frequency= @@ -1375,29 +1365,29 @@ public: Read statistical data from statistical fields of index_stats @details - This implementation of a purely virtual method first looks for a record the - statistical table index_stats by its primary key set the record buffer with - the help of Index_stat::set_key_fields. If the row is found the function - reads the value of the column 'avg_freguency' of the table index_stat and - sets the value of read_stat.avg_frequency[Index_stat::prefix_arity] - from the KEY_INFO structure 'table_key_info' accordingly. If the value of - the column is NULL, read_stat.avg_frequency[Index_stat::prefix_arity] is - set to 0. Otherwise, read_stat.avg_frequency[Index_stat::prefix_arity] is - set to the value of the column. - */ + Find a record in mysql.index_stats that has statistics for the index prefix + of interest (the prefix length is in this->prefix_arity). + We search for record using a PK lookup. The lookup values are in the stat + table's record buffer. - void get_stat_values() + The result is stored in read_stats->avg_frequency[this->prefix_arity]. + If mysql.index_stats doesn't have the value or has SQL NULL, we store the + value of 0. + */ + + bool get_stat_values(Index_statistics *read_stats) { double avg_frequency= 0; - if(find_stat()) + bool res; + if ((res= find_stat())) { Field *stat_field= stat_table->field[INDEX_STAT_AVG_FREQUENCY]; if (!stat_field->is_null()) avg_frequency= stat_field->val_real(); } - table_key_info->read_stats->set_avg_frequency(prefix_arity-1, avg_frequency); - } - + read_stats->set_avg_frequency(prefix_arity-1, avg_frequency); + return res; + } }; @@ -1727,7 +1717,6 @@ public: { return table_field->collected_stats->histogram.get_values(); } - }; @@ -1787,8 +1776,6 @@ class Index_prefix_calc: public Sql_alloc private: - /* Table containing index specified by index_info */ - TABLE *index_table; /* Info for the index i for whose prefix 'avg_frequency' is calculated */ KEY *index_info; /* The maximum number of the components in the prefixes of interest */ @@ -1825,7 +1812,7 @@ public: bool is_partial_fields_present; Index_prefix_calc(THD *thd, TABLE *table, KEY *key_info) - : index_table(table), index_info(key_info), prefixes(0), empty(true), + : index_info(key_info), prefixes(0), empty(true), calc_state(NULL), is_single_comp_pk(false), is_partial_fields_present(false) { uint i; @@ -1859,8 +1846,8 @@ public: } if (!(state->last_prefix= - new (thd->mem_root) Cached_item_field(thd, - key_info->key_part[i].field))) + new (thd->mem_root) + Cached_item_field(thd, key_info->key_part[i].field))) break; state->entry_count= state->prefix_count= 0; prefixes++; @@ -1971,12 +1958,12 @@ public: */ static -void create_min_max_statistical_fields_for_table(TABLE *table) +void create_min_max_statistical_fields_for_table(THD *thd, TABLE *table) { uint rec_buff_length= table->s->rec_buff_length; if ((table->collected_stats->min_max_record_buffers= - (uchar *) alloc_root(&table->mem_root, 2*rec_buff_length))) + (uchar *) alloc_root(thd->mem_root, 2*rec_buff_length))) { uchar *record= table->collected_stats->min_max_record_buffers; memset(record, 0, 2*rec_buff_length); @@ -1990,7 +1977,7 @@ void create_min_max_statistical_fields_for_table(TABLE *table) my_ptrdiff_t diff= record-table->record[0]; if (!bitmap_is_set(table->read_set, table_field->field_index)) continue; - if (!(fld= table_field->clone(&table->mem_root, table, diff))) + if (!(fld= table_field->clone(thd->mem_root, table, diff))) continue; if (i == 0) table_field->collected_stats->min_value= fld; @@ -2007,22 +1994,20 @@ void create_min_max_statistical_fields_for_table(TABLE *table) Create fields for min/max values to read column statistics @param - thd Thread handler + thd Thread handler @param - table_share Table share the fields are created for + table_share Table share the fields are created for @param - is_safe TRUE <-> at any time only one thread can perform the function + stats_cb TABLE_STATISTICS_CB object whose mem_root is used for allocations @details - The function first allocates record buffers to store min/max values - for 'table_share's fields. Then for each field f it creates Field structures + The function first allocates record buffers to store min/max values for + fields in the table. For each field f it creates Field structures that points to these buffers rather that to the record buffer as the Field object for f does. The pointers of the created fields are placed in the read_stats structure of the Field object for f. - The function allocates the buffers for min/max values in the table share - memory. - If the parameter is_safe is TRUE then it is guaranteed that at any given time - only one thread is executed the code of the function. + The function allocates the buffers for min/max values in the stats_cb + memory. @note The buffers allocated when min/max values are used to collect statistics @@ -2030,14 +2015,14 @@ void create_min_max_statistical_fields_for_table(TABLE *table) are used when statistics on min/max values for column is read as they are allocated in different mem_roots. The same is true for the fields created for min/max values. -*/ +*/ -static -void create_min_max_statistical_fields_for_table_share(THD *thd, - TABLE_SHARE *table_share) +static void +create_min_max_statistical_fields(THD *thd, + const TABLE_SHARE *table_share, + TABLE_STATISTICS_CB *stats_cb) { - TABLE_STATISTICS_CB *stats_cb= &table_share->stats_cb; - Table_statistics *stats= stats_cb->table_stats; + Table_statistics *stats= stats_cb->table_stats; if (stats->min_max_record_buffers) return; @@ -2052,7 +2037,10 @@ void create_min_max_statistical_fields_for_table_share(THD *thd, for (uint i=0; i < 2; i++, record+= rec_buff_length) { - for (Field **field_ptr= table_share->field; *field_ptr; field_ptr++) + Column_statistics *column_stats= stats_cb->table_stats->column_stats; + for (Field **field_ptr= table_share->field; + *field_ptr; + field_ptr++, column_stats++) { Field *fld; Field *table_field= *field_ptr; @@ -2060,9 +2048,9 @@ void create_min_max_statistical_fields_for_table_share(THD *thd, if (!(fld= table_field->clone(&stats_cb->mem_root, NULL, diff))) continue; if (i == 0) - table_field->read_stats->min_value= fld; + column_stats->min_value= fld; else - table_field->read_stats->max_value= fld; + column_stats->max_value= fld; } } } @@ -2081,10 +2069,10 @@ void create_min_max_statistical_fields_for_table_share(THD *thd, The function allocates the memory for the statistical data on 'table' with the intention to collect the data there. The memory is allocated for the statistics on the table, on the table's columns, and on the table's - indexes. The memory is allocated in the table's mem_root. + indexes. The memory is allocated in the thd's mem_root. @retval - 0 If the memory for all statistical data has been successfully allocated + 0 If the memory for all statistical data has been successfully allocated @retval 1 Otherwise @@ -2097,56 +2085,40 @@ void create_min_max_statistical_fields_for_table_share(THD *thd, int alloc_statistics_for_table(THD* thd, TABLE *table) { Field **field_ptr; - - DBUG_ENTER("alloc_statistics_for_table"); - - uint columns= 0; - for (field_ptr= table->field; *field_ptr; field_ptr++) - { - if (bitmap_is_set(table->read_set, (*field_ptr)->field_index)) - columns++; - } - - Table_statistics *table_stats= - (Table_statistics *) alloc_root(&table->mem_root, - sizeof(Table_statistics)); - - Column_statistics_collected *column_stats= - (Column_statistics_collected *) alloc_root(&table->mem_root, - sizeof(Column_statistics_collected) * - columns); - + uint fields= bitmap_bits_set(table->read_set); uint keys= table->s->keys; - Index_statistics *index_stats= - (Index_statistics *) alloc_root(&table->mem_root, - sizeof(Index_statistics) * keys); - uint key_parts= table->s->ext_key_parts; - ulonglong *idx_avg_frequency= (ulonglong*) alloc_root(&table->mem_root, - sizeof(ulonglong) * key_parts); - uint hist_size= thd->variables.histogram_size; Histogram_type hist_type= (Histogram_type) (thd->variables.histogram_type); - uchar *histogram= NULL; - if (hist_size > 0) - { - if ((histogram= (uchar *) alloc_root(&table->mem_root, - hist_size * columns))) - bzero(histogram, hist_size * columns); + Table_statistics *table_stats; + Column_statistics_collected *column_stats; + Index_statistics *index_stats; + ulonglong *idx_avg_frequency; + uchar *histogram; + DBUG_ENTER("alloc_statistics_for_table"); - } - - if (!table_stats || !column_stats || !index_stats || !idx_avg_frequency || - (hist_size && !histogram)) + if (!multi_alloc_root(thd->mem_root, + &table_stats, sizeof(*table_stats), + &column_stats, sizeof(*column_stats) * fields, + &index_stats, sizeof(*index_stats) * keys, + &idx_avg_frequency, + sizeof(idx_avg_frequency) * key_parts, + &histogram, hist_size * fields, + NullS)) DBUG_RETURN(1); + if (hist_size > 0) + bzero(histogram, hist_size * fields); + else + histogram= 0; + table->collected_stats= table_stats; table_stats->column_stats= column_stats; table_stats->index_stats= index_stats; table_stats->idx_avg_frequency= idx_avg_frequency; table_stats->histograms= histogram; - memset(column_stats, 0, sizeof(Column_statistics) * columns); + bzero(column_stats, sizeof(Column_statistics) * fields); for (field_ptr= table->field; *field_ptr; field_ptr++) { @@ -2158,6 +2130,8 @@ int alloc_statistics_for_table(THD* thd, TABLE *table) histogram+= hist_size; (*field_ptr)->collected_stats= column_stats++; } + else + (*field_ptr)->collected_stats= 0; } memset(idx_avg_frequency, 0, sizeof(ulonglong) * key_parts); @@ -2172,7 +2146,7 @@ int alloc_statistics_for_table(THD* thd, TABLE *table) idx_avg_frequency+= key_info->ext_key_parts; } - create_min_max_statistical_fields_for_table(table); + create_min_max_statistical_fields_for_table(thd, table); DBUG_RETURN(0); } @@ -2186,6 +2160,8 @@ int alloc_statistics_for_table(THD* thd, TABLE *table) thd Thread handler @param table_share Table share for which the memory for statistical data is allocated + @param + stats_cb TABLE_STATISTICS_CB object for storing the statistical data @note The function allocates the memory for the statistical data on a table in the @@ -2214,87 +2190,49 @@ int alloc_statistics_for_table(THD* thd, TABLE *table) guarantee the correctness of the allocation. */ -static int alloc_statistics_for_table_share(THD* thd, TABLE_SHARE *table_share) +static int +alloc_engine_independent_statistics(THD *thd, const TABLE_SHARE *table_share, + TABLE_STATISTICS_CB *stats_cb) { - Field **field_ptr; - KEY *key_info, *end; - TABLE_STATISTICS_CB *stats_cb= &table_share->stats_cb; - - DBUG_ENTER("alloc_statistics_for_table_share"); - Table_statistics *table_stats= stats_cb->table_stats; - if (!table_stats) - { - table_stats= (Table_statistics *) alloc_root(&stats_cb->mem_root, - sizeof(Table_statistics)); - if (!table_stats) - DBUG_RETURN(1); - memset(table_stats, 0, sizeof(Table_statistics)); - stats_cb->table_stats= table_stats; - } - uint fields= table_share->fields; - Column_statistics *column_stats= table_stats->column_stats; - if (!column_stats) - { - column_stats= (Column_statistics *) alloc_root(&stats_cb->mem_root, - sizeof(Column_statistics) * - (fields+1)); - if (column_stats) - { - memset(column_stats, 0, sizeof(Column_statistics) * (fields+1)); - table_stats->column_stats= column_stats; - for (field_ptr= table_share->field; - *field_ptr; - field_ptr++, column_stats++) - { - (*field_ptr)->read_stats= column_stats; - (*field_ptr)->read_stats->min_value= NULL; - (*field_ptr)->read_stats->max_value= NULL; - } - create_min_max_statistical_fields_for_table_share(thd, table_share); - } - } - uint keys= table_share->keys; - Index_statistics *index_stats= table_stats->index_stats; - if (!index_stats) - { - index_stats= (Index_statistics *) alloc_root(&stats_cb->mem_root, - sizeof(Index_statistics) * - keys); - if (index_stats) - { - table_stats->index_stats= index_stats; - for (key_info= table_share->key_info, end= key_info + keys; - key_info < end; - key_info++, index_stats++) - { - key_info->read_stats= index_stats; - } - } - } - uint key_parts= table_share->ext_key_parts; - ulonglong *idx_avg_frequency= table_stats->idx_avg_frequency; - if (!idx_avg_frequency) + Index_statistics *index_stats; + ulonglong *idx_avg_frequency; + DBUG_ENTER("alloc_engine_independent_statistics"); + + Column_statistics *column_stats; + if (!multi_alloc_root(&stats_cb->mem_root, + &table_stats, sizeof(Table_statistics), + &column_stats, sizeof(Column_statistics) * fields, + &index_stats, sizeof(Index_statistics) * keys, + &idx_avg_frequency, + sizeof(idx_avg_frequency) * key_parts, + NullS)) + DBUG_RETURN(1); + + /* Zero variables but not the gaps between them */ + bzero(table_stats, sizeof(Table_statistics)); + bzero(column_stats, sizeof(Column_statistics) * fields); + bzero(index_stats, sizeof(Index_statistics) * keys); + bzero(idx_avg_frequency, sizeof(idx_avg_frequency) * key_parts); + + stats_cb->table_stats= table_stats; + table_stats->column_stats= column_stats; + table_stats->index_stats= index_stats; + table_stats->idx_avg_frequency= idx_avg_frequency; + + create_min_max_statistical_fields(thd, table_share, stats_cb); + + for (KEY *key_info= table_share->key_info, *end= key_info + keys; + key_info < end; + key_info++, index_stats++) { - idx_avg_frequency= (ulonglong*) alloc_root(&stats_cb->mem_root, - sizeof(ulonglong) * key_parts); - if (idx_avg_frequency) - { - memset(idx_avg_frequency, 0, sizeof(ulonglong) * key_parts); - table_stats->idx_avg_frequency= idx_avg_frequency; - for (key_info= table_share->key_info, end= key_info + keys; - key_info < end; - key_info++) - { - key_info->read_stats->init_avg_frequency(idx_avg_frequency); - idx_avg_frequency+= key_info->ext_key_parts; - } - } + index_stats->init_avg_frequency(idx_avg_frequency); + idx_avg_frequency+= key_info->ext_key_parts; } - DBUG_RETURN(column_stats && index_stats && idx_avg_frequency ? 0 : 1); + DBUG_RETURN(0); } @@ -2516,7 +2454,6 @@ int collect_statistics_for_index(THD *thd, TABLE *table, uint index) { int rc= 0; KEY *key_info= &table->key_info[index]; - DBUG_ENTER("collect_statistics_for_index"); /* No statistics for FULLTEXT indexes. */ @@ -2626,7 +2563,6 @@ int collect_statistics_for_table(THD *thd, TABLE *table) handler *file=table->file; double sample_fraction= thd->variables.sample_percentage / 100; const ha_rows MIN_THRESHOLD_FOR_SAMPLING= 50000; - DBUG_ENTER("collect_statistics_for_table"); table->collected_stats->cardinality_is_null= TRUE; @@ -2791,6 +2727,12 @@ int update_statistics_for_table(THD *thd, TABLE *table) if (open_stat_tables(thd, tables, TRUE)) DBUG_RETURN(rc); + /* + Ensure that no one is reading satistics while we are writing them + This ensures that statistics is always read consistently + */ + mysql_mutex_lock(&table->s->LOCK_statistics); + save_binlog_format= thd->set_current_stmt_binlog_format_stmt(); /* Update the statistical table table_stats */ @@ -2842,6 +2784,7 @@ int update_statistics_for_table(THD *thd, TABLE *table) rc= 1; new_trans.restore_old_transaction(); + mysql_mutex_unlock(&table->s->LOCK_statistics); DBUG_RETURN(rc); } @@ -2851,11 +2794,14 @@ int update_statistics_for_table(THD *thd, TABLE *table) Read statistics for a table from the persistent statistical tables @param - thd The thread handle + thd The thread handle @param - table The table to read statistics on + table The table to read statistics on. @param - stat_tables The array of TABLE_LIST objects for statistical tables + stat_tables The array of TABLE_LIST objects for statistical tables + @param + force_reload Flag to require reloading the statistics from the tables + even if it has been already loaded @details For each statistical table the function looks for the rows from this @@ -2869,9 +2815,9 @@ int update_statistics_for_table(THD *thd, TABLE *table) The function is called by read_statistics_for_tables_if_needed(). @retval - 0 If data has been successfully read for the table + pointer to object If data has been successfully read for the table @retval - 1 Otherwise + 0 Otherwise @note Objects of the helper classes Table_stat, Column_stat and Index_stat @@ -2880,99 +2826,134 @@ int update_statistics_for_table(THD *thd, TABLE *table) */ static -int read_statistics_for_table(THD *thd, TABLE *table, TABLE_LIST *stat_tables) +TABLE_STATISTICS_CB* +read_statistics_for_table(THD *thd, TABLE *table, + TABLE_LIST *stat_tables, bool force_reload, + bool want_histograms) { + bool found; uint i; TABLE *stat_table; Field *table_field; Field **field_ptr; KEY *key_info, *key_info_end; TABLE_SHARE *table_share= table->s; - DBUG_ENTER("read_statistics_for_table"); - DEBUG_SYNC(thd, "statistics_mem_alloc_start1"); - DEBUG_SYNC(thd, "statistics_mem_alloc_start2"); - if (!table_share->stats_cb.start_stats_load()) - DBUG_RETURN(table_share->stats_cb.stats_are_ready() ? 0 : 1); - - if (alloc_statistics_for_table_share(thd, table_share)) + if (!force_reload && table_share->stats_cb) { - table_share->stats_cb.abort_stats_load(); - DBUG_RETURN(1); + if (table->stats_cb == table_share->stats_cb) + DBUG_RETURN(table->stats_cb); // Use current + table->update_engine_independent_stats(); // Copy table_share->stats_cb + DBUG_RETURN(table->stats_cb); + } + + /* + Read data into a new TABLE_STATISTICS_CB object and replace + TABLE_SHARE::stats_cb with this new one once the reading is finished + */ + TABLE_STATISTICS_CB *new_stats_cb; + if (!(new_stats_cb= new TABLE_STATISTICS_CB)) + DBUG_RETURN(0); /* purecov: inspected */ + + if (alloc_engine_independent_statistics(thd, table_share, new_stats_cb)) + { + /* purecov: begin inspected */ + delete new_stats_cb; + DBUG_RETURN(0); + /* purecov: end */ } /* Don't write warnings for internal field conversions */ Check_level_instant_set check_level_save(thd, CHECK_FIELD_IGNORE); /* Read statistics from the statistical table table_stats */ - Table_statistics *read_stats= table_share->stats_cb.table_stats; + Table_statistics *read_stats= new_stats_cb->table_stats; stat_table= stat_tables[TABLE_STAT].table; Table_stat table_stat(stat_table, table); table_stat.set_key_fields(); - table_stat.get_stat_values(); - + if (table_stat.get_stat_values(new_stats_cb->table_stats)) + new_stats_cb->stats_available|= TABLE_STAT_TABLE; + /* Read statistics from the statistical table column_stats */ stat_table= stat_tables[COLUMN_STAT].table; ulong total_hist_size= 0; Column_stat column_stat(stat_table, table); - for (field_ptr= table_share->field; *field_ptr; field_ptr++) + Column_statistics *column_statistics= new_stats_cb->table_stats->column_stats; + found= 0; + for (field_ptr= table_share->field; + *field_ptr; + field_ptr++, column_statistics++) { table_field= *field_ptr; column_stat.set_key_fields(table_field); - column_stat.get_stat_values(); - total_hist_size+= table_field->read_stats->histogram.get_size(); + found|= column_stat.get_stat_values(column_statistics, + &new_stats_cb->mem_root, + want_histograms); + total_hist_size+= column_statistics->histogram.get_size(); } - table_share->stats_cb.total_hist_size= total_hist_size; + if (found) + { + new_stats_cb->stats_available|= TABLE_STAT_COLUMN; + if (total_hist_size && want_histograms) + new_stats_cb->stats_available|= TABLE_STAT_HISTOGRAM; + } + + new_stats_cb->total_hist_size= total_hist_size; /* Read statistics from the statistical table index_stats */ stat_table= stat_tables[INDEX_STAT].table; Index_stat index_stat(stat_table, table); + Index_statistics *index_statistics= new_stats_cb->table_stats->index_stats; for (key_info= table_share->key_info, key_info_end= key_info + table_share->keys; - key_info < key_info_end; key_info++) + key_info < key_info_end; key_info++, index_statistics++) { uint key_parts= key_info->ext_key_parts; + found= 0; for (i= 0; i < key_parts; i++) { index_stat.set_key_fields(key_info, i+1); - index_stat.get_stat_values(); + found|= index_stat.get_stat_values(index_statistics); } - + if (found) + new_stats_cb->stats_available|= TABLE_STAT_INDEX; + key_part_map ext_key_part_map= key_info->ext_key_part_map; if (key_info->user_defined_key_parts != key_info->ext_key_parts && - key_info->read_stats->get_avg_frequency(key_info->user_defined_key_parts) == 0) + index_statistics->get_avg_frequency(key_info->user_defined_key_parts) == 0) { KEY *pk_key_info= table_share->key_info + table_share->primary_key; uint k= key_info->user_defined_key_parts; uint pk_parts= pk_key_info->user_defined_key_parts; ha_rows n_rows= read_stats->cardinality; - double k_dist= n_rows / key_info->read_stats->get_avg_frequency(k-1); + double k_dist= n_rows / index_statistics->get_avg_frequency(k-1); uint m= 0; + Index_statistics *pk_read_stats= (new_stats_cb->table_stats->index_stats + + table_share->primary_key); for (uint j= 0; j < pk_parts; j++) { if (!(ext_key_part_map & 1 << j)) { for (uint l= k; l < k + m; l++) { - double avg_frequency= - pk_key_info->read_stats->get_avg_frequency(j-1); + double avg_frequency= pk_read_stats->get_avg_frequency(j-1); set_if_smaller(avg_frequency, 1); - double val= pk_key_info->read_stats->get_avg_frequency(j) / - avg_frequency; - key_info->read_stats->set_avg_frequency (l, val); + double val= (pk_read_stats->get_avg_frequency(j) / + avg_frequency); + index_statistics->set_avg_frequency (l, val); } } else { - double avg_frequency= pk_key_info->read_stats->get_avg_frequency(j); - key_info->read_stats->set_avg_frequency(k + m, avg_frequency); + double avg_frequency= pk_read_stats->get_avg_frequency(j); + index_statistics->set_avg_frequency(k + m, avg_frequency); m++; } } for (uint l= k; l < k + m; l++) { - double avg_frequency= key_info->read_stats->get_avg_frequency(l); + double avg_frequency= index_statistics->get_avg_frequency(l); if (avg_frequency == 0 || read_stats->cardinality_is_null) avg_frequency= 1; else if (avg_frequency > 1) @@ -2980,116 +2961,14 @@ int read_statistics_for_table(THD *thd, TABLE *table, TABLE_LIST *stat_tables) avg_frequency/= k_dist; set_if_bigger(avg_frequency, 1); } - key_info->read_stats->set_avg_frequency(l, avg_frequency); + index_statistics->set_avg_frequency(l, avg_frequency); } } } - - table_share->stats_cb.end_stats_load(); - DBUG_RETURN(0); + DBUG_RETURN(new_stats_cb); } -/** - @breif - Cleanup of min/max statistical values for table share -*/ - -void delete_stat_values_for_table_share(TABLE_SHARE *table_share) -{ - TABLE_STATISTICS_CB *stats_cb= &table_share->stats_cb; - Table_statistics *table_stats= stats_cb->table_stats; - if (!table_stats) - return; - Column_statistics *column_stats= table_stats->column_stats; - if (!column_stats) - return; - - for (Field **field_ptr= table_share->field; - *field_ptr; - field_ptr++, column_stats++) - { - if (column_stats->min_value) - { - delete column_stats->min_value; - column_stats->min_value= NULL; - } - if (column_stats->max_value) - { - delete column_stats->max_value; - column_stats->max_value= NULL; - } - } -} - - -/** - @brief - Read histogram for a table from the persistent statistical tables - - @param - thd The thread handle - @param - table The table to read histograms for - @param - stat_tables The array of TABLE_LIST objects for statistical tables - - @details - For the statistical table columns_stats the function looks for the rows - from this table that contain statistical data on 'table'. If such rows - are found the histograms from them are read into the memory allocated - for histograms of 'table'. Later at the query processing these histogram - are supposed to be used by the optimizer. - The parameter stat_tables should point to an array of TABLE_LIST - objects for all statistical tables linked into a list. All statistical - tables are supposed to be opened. - The function is called by read_statistics_for_tables_if_needed(). - - @retval - 0 If data has been successfully read for the table - @retval - 1 Otherwise - - @note - Objects of the helper Column_stat are employed read histogram - from the statistical table column_stats now. -*/ - -static -int read_histograms_for_table(THD *thd, TABLE *table, TABLE_LIST *stat_tables) -{ - TABLE_STATISTICS_CB *stats_cb= &table->s->stats_cb; - DBUG_ENTER("read_histograms_for_table"); - - if (stats_cb->start_histograms_load()) - { - uchar *histogram= (uchar *) alloc_root(&stats_cb->mem_root, - stats_cb->total_hist_size); - if (!histogram) - { - stats_cb->abort_histograms_load(); - DBUG_RETURN(1); - } - memset(histogram, 0, stats_cb->total_hist_size); - - Column_stat column_stat(stat_tables[COLUMN_STAT].table, table); - for (Field **field_ptr= table->s->field; *field_ptr; field_ptr++) - { - Field *table_field= *field_ptr; - if (uint hist_size= table_field->read_stats->histogram.get_size()) - { - column_stat.set_key_fields(table_field); - table_field->read_stats->histogram.set_values(histogram); - column_stat.get_histogram_value(); - histogram+= hist_size; - } - } - stats_cb->end_histograms_load(); - } - table->histograms_are_read= true; - DBUG_RETURN(0); -} - /** @brief Read statistics for tables from a table list if it is needed @@ -3127,65 +3006,97 @@ int read_statistics_for_tables_if_needed(THD *thd, TABLE_LIST *tables) case SQLCOM_CREATE_TABLE: case SQLCOM_SET_OPTION: case SQLCOM_DO: - return read_statistics_for_tables(thd, tables); + return read_statistics_for_tables(thd, tables, 0); default: return 0; } } -static void dump_stats_from_share_to_table(TABLE *table) -{ - TABLE_SHARE *table_share= table->s; - KEY *key_info= table_share->key_info; - KEY *key_info_end= key_info + table_share->keys; - KEY *table_key_info= table->key_info; - for ( ; key_info < key_info_end; key_info++, table_key_info++) - table_key_info->read_stats= key_info->read_stats; +/* + Update TABLE field and key objects with pointers to + the current statistical data in table->stats_cb +*/ - Field **field_ptr= table_share->field; - Field **table_field_ptr= table->field; - for ( ; *field_ptr; field_ptr++, table_field_ptr++) - (*table_field_ptr)->read_stats= (*field_ptr)->read_stats; + +void TABLE_STATISTICS_CB::update_stats_in_table(TABLE *table) +{ + DBUG_ASSERT(table->stats_cb == this); + + /* + Table_statistics doesn't need to be updated: set_statistics_for_table() + sets TABLE::used_stat_records from table->stats_cb.table_stats.cardinality + */ + + KEY *key_info= table->key_info; + KEY *key_info_end= key_info + table->s->keys; + Index_statistics *index_stats= table_stats->index_stats; + + for ( ; key_info < key_info_end; key_info++, index_stats++) + key_info->read_stats= index_stats; + + Field **field_ptr= table->field; + Column_statistics *column_stats= table_stats->column_stats; + + for ( ; *field_ptr; field_ptr++, column_stats++) + (*field_ptr)->read_stats= column_stats; + /* Mark that stats are now usable */ table->stats_is_read= true; } -int read_statistics_for_tables(THD *thd, TABLE_LIST *tables) +int +read_statistics_for_tables(THD *thd, TABLE_LIST *tables, bool force_reload) { TABLE_LIST stat_tables[STATISTICS_TABLES]; - - DBUG_ENTER("read_statistics_for_tables"); - - if (thd->bootstrap || thd->variables.use_stat_tables == NEVER) - DBUG_RETURN(0); - bool found_stat_table= false; bool statistics_for_tables_is_needed= false; + bool want_histograms= thd->variables.optimizer_use_condition_selectivity > 3; + DBUG_ENTER("read_statistics_for_tables"); + + if (thd->bootstrap || thd->variables.use_stat_tables == NEVER || !tables) + DBUG_RETURN(0); for (TABLE_LIST *tl= tables; tl; tl= tl->next_global) { + TABLE *table= tl->table; TABLE_SHARE *table_share; - if (!tl->is_view_or_derived() && tl->table && (table_share= tl->table->s) && - table_share->tmp_table == NO_TMP_TABLE) + + /* Skip tables that can't have statistics. */ + if (tl->is_view_or_derived() || !table || !(table_share= table->s)) + continue; + /* Skip temporary tables */ + if (table_share->tmp_table != NO_TMP_TABLE) + continue; + + if (table_share->table_category == TABLE_CATEGORY_USER) { - if (table_share->table_category == TABLE_CATEGORY_USER) + /* Force reloading means we always read all stats tables. */ + if (force_reload || !table_share->stats_cb) + { + statistics_for_tables_is_needed= true; + continue; + } + + /* Stats versions don't match, take a reference under a mutex. */ + if (table->stats_cb != table_share->stats_cb) + { + table->update_engine_independent_stats(); + table->stats_cb->update_stats_in_table(table); + } + /* + We need to read histograms if they exist but have not yet been + loaded into memory. + */ + if (want_histograms && + table->stats_cb->histograms_exists() && + !(table->stats_cb->stats_available & TABLE_STAT_HISTOGRAM)) { - if (table_share->stats_cb.stats_are_ready()) - { - if (!tl->table->stats_is_read) - dump_stats_from_share_to_table(tl->table); - tl->table->histograms_are_read= - table_share->stats_cb.histograms_are_ready(); - if (table_share->stats_cb.histograms_are_ready() || - thd->variables.optimizer_use_condition_selectivity <= 3) - continue; - } statistics_for_tables_is_needed= true; } - else if (is_stat_table(&tl->db, &tl->alias)) - found_stat_table= true; } + else if (is_stat_table(&tl->db, &tl->alias)) + found_stat_table= true; } DEBUG_SYNC(thd, "statistics_read_start"); @@ -3205,20 +3116,50 @@ int read_statistics_for_tables(THD *thd, TABLE_LIST *tables) for (TABLE_LIST *tl= tables; tl; tl= tl->next_global) { + TABLE *table= tl->table; TABLE_SHARE *table_share; - if (!tl->is_view_or_derived() && tl->table && (table_share= tl->table->s) && - table_share->tmp_table == NO_TMP_TABLE && - table_share->table_category == TABLE_CATEGORY_USER) + + /* Skip tables that can't have statistics. */ + if (tl->is_view_or_derived() || !table || !(table_share= table->s) || + table_share->tmp_table != NO_TMP_TABLE || + table_share->table_category != TABLE_CATEGORY_USER) + continue; + + if (force_reload || !table_share->stats_cb || + table->stats_cb != table_share->stats_cb || + (want_histograms && table->stats_cb->histograms_exists() && + !(table->stats_cb->stats_available & TABLE_STAT_HISTOGRAM))) { - if (!tl->table->stats_is_read) + TABLE_STATISTICS_CB *stats_cb; + DEBUG_SYNC(thd, "read_statistics_for_table_start1"); + DEBUG_SYNC(thd, "read_statistics_for_table_start2"); + + /* + The following lock is here to ensure that if a lot of threads are + accessing the table at the same time after a ANALYZE TABLE, + only one thread is loading the data from the the stats tables + and the others threads are reusing the loaded data. + */ + mysql_mutex_lock(&table_share->LOCK_statistics); + if (!(stats_cb= read_statistics_for_table(thd, table, stat_tables, + force_reload, want_histograms))) { - if (!read_statistics_for_table(thd, tl->table, stat_tables)) - dump_stats_from_share_to_table(tl->table); - else - continue; + /* purecov: begin inspected */ + mysql_mutex_unlock(&table_share->LOCK_statistics); + continue; + /* purecov: end */ } - if (thd->variables.optimizer_use_condition_selectivity > 3) - (void) read_histograms_for_table(thd, tl->table, stat_tables); + + if (stats_cb->unused()) + { + /* New object created, update share to use it */ + table_share->update_engine_independent_stats(stats_cb); + table->update_engine_independent_stats(); + } + mysql_mutex_unlock(&table_share->LOCK_statistics); + table->stats_cb->update_stats_in_table(table); + table->stats_is_read= (stats_cb->stats_available != + TABLE_STAT_NO_STATS); } } @@ -3377,7 +3318,8 @@ int delete_statistics_for_column(THD *thd, TABLE *tab, Field *col) @param thd The thread handle @param tab The table the index belongs to - @param key_info The descriptor of the index whose statistics is to be deleted + @param key_info The descriptor of the index whose statistics is to be + deleted @param ext_prefixes_only Delete statistics only on the index prefixes extended by the components of the primary key @@ -3385,7 +3327,8 @@ int delete_statistics_for_column(THD *thd, TABLE *tab, Field *col) The function delete statistics on the index specified by 'key_info' defined on the table 'tab' from the statistical table index_stats. - @retval 0 If all deletions are successful or we couldn't open statistics table + @retval 0 If all deletions are successful or we couldn't open statistics + table @retval 1 Otherwise @note @@ -3625,11 +3568,12 @@ int rename_column_in_stat_tables(THD *thd, TABLE *tab, Field *col, void set_statistics_for_table(THD *thd, TABLE *table) { - TABLE_STATISTICS_CB *stats_cb= &table->s->stats_cb; - Table_statistics *read_stats= stats_cb->table_stats; + TABLE_STATISTICS_CB *stats_cb= table->stats_cb; + + Table_statistics *read_stats= stats_cb ? stats_cb->table_stats : 0; table->used_stat_records= (!check_eits_preferred(thd) || - !table->stats_is_read || read_stats->cardinality_is_null) ? + !table->stats_is_read || !read_stats || read_stats->cardinality_is_null) ? table->file->stats.records : read_stats->cardinality; /* diff --git a/sql/sql_statistics.h b/sql/sql_statistics.h index 35b3aa33acc..f5ccb26af7b 100644 --- a/sql/sql_statistics.h +++ b/sql/sql_statistics.h @@ -116,9 +116,9 @@ bool check_eits_preferred(THD *thd) } int read_statistics_for_tables_if_needed(THD *thd, TABLE_LIST *tables); -int read_statistics_for_tables(THD *thd, TABLE_LIST *tables); +int read_statistics_for_tables(THD *thd, TABLE_LIST *tables, + bool force_reload); int collect_statistics_for_table(THD *thd, TABLE *table); -void delete_stat_values_for_table_share(TABLE_SHARE *table_share); int alloc_statistics_for_table(THD *thd, TABLE *table); int update_statistics_for_table(THD *thd, TABLE *table); int delete_statistics_for_table(THD *thd, const LEX_CSTRING *db, const LEX_CSTRING *tab); @@ -308,7 +308,7 @@ public: /* Array of records per key for index prefixes */ ulonglong *idx_avg_frequency; - uchar *histograms; /* Sequence of histograms */ + uchar *histograms; /* Sequence of histograms */ }; diff --git a/sql/table.cc b/sql/table.cc index 2a6b758a027..4badef7d5a1 100644 --- a/sql/table.cc +++ b/sql/table.cc @@ -373,14 +373,13 @@ TABLE_SHARE *alloc_table_share(const char *db, const char *table_name, table_alias_charset->strnncoll(key, 6, "mysql", 6) == 0) share->not_usable_by_query_cache= 1; - init_sql_alloc(PSI_INSTRUMENT_ME, &share->stats_cb.mem_root, - TABLE_ALLOC_BLOCK_SIZE, 0, MYF(0)); - memcpy((char*) &share->mem_root, (char*) &mem_root, sizeof(mem_root)); mysql_mutex_init(key_TABLE_SHARE_LOCK_share, &share->LOCK_share, MY_MUTEX_INIT_SLOW); mysql_mutex_init(key_TABLE_SHARE_LOCK_ha_data, &share->LOCK_ha_data, MY_MUTEX_INIT_FAST); + mysql_mutex_init(key_TABLE_SHARE_LOCK_statistics, + &share->LOCK_statistics, MY_MUTEX_INIT_SLOW); DBUG_EXECUTE_IF("simulate_big_table_id", if (last_table_id < UINT_MAX32) @@ -481,15 +480,19 @@ void TABLE_SHARE::destroy() ha_share= NULL; // Safety } - delete_stat_values_for_table_share(this); + if (stats_cb) + { + stats_cb->usage_count--; + delete stats_cb; + } delete sequence; - free_root(&stats_cb.mem_root, MYF(0)); /* The mutexes are initialized only for shares that are part of the TDC */ if (tmp_table == NO_TMP_TABLE) { mysql_mutex_destroy(&LOCK_share); mysql_mutex_destroy(&LOCK_ha_data); + mysql_mutex_destroy(&LOCK_statistics); } my_hash_free(&name_hash); @@ -4527,6 +4530,72 @@ partititon_err: } +/** + Free engine stats + + This is only called from closefrm() when the TABLE object is destroyed +**/ + +void TABLE::free_engine_stats() +{ + bool free_stats= 0; + TABLE_STATISTICS_CB *stats= stats_cb; + mysql_mutex_lock(&s->LOCK_share); + free_stats= --stats->usage_count == 0; + mysql_mutex_unlock(&s->LOCK_share); + if (free_stats) + delete stats; +} + + +/* + Use engine stats from table_share if table_share has been updated +*/ + +void TABLE::update_engine_independent_stats() +{ + bool free_stats= 0; + TABLE_STATISTICS_CB *org_stats= stats_cb; + DBUG_ASSERT(stats_cb != s->stats_cb); + + if (stats_cb != s->stats_cb) + { + mysql_mutex_lock(&s->LOCK_share); + if (org_stats) + free_stats= --org_stats->usage_count == 0; + if ((stats_cb= s->stats_cb)) + stats_cb->usage_count++; + mysql_mutex_unlock(&s->LOCK_share); + if (free_stats) + delete org_stats; + } +} + + +/* + Update engine stats in table share to use new stats +*/ + +void +TABLE_SHARE::update_engine_independent_stats(TABLE_STATISTICS_CB *new_stats) +{ + TABLE_STATISTICS_CB *free_stats= 0; + DBUG_ASSERT(new_stats->usage_count == 0); + + mysql_mutex_lock(&LOCK_share); + if (stats_cb) + { + if (!--stats_cb->usage_count) + free_stats= stats_cb; + } + stats_cb= new_stats; + new_stats->usage_count++; + mysql_mutex_unlock(&LOCK_share); + if (free_stats) + delete free_stats; +} + + /* Free information allocated by openfrm @@ -4565,6 +4634,12 @@ int closefrm(TABLE *table) table->part_info= 0; } #endif + if (table->stats_cb) + { + DBUG_ASSERT(table->s->tmp_table == NO_TMP_TABLE); + table->free_engine_stats(); + } + free_root(&table->mem_root, MYF(0)); DBUG_RETURN(error); } diff --git a/sql/table.h b/sql/table.h index af09b03c9e0..382692cb7ca 100644 --- a/sql/table.h +++ b/sql/table.h @@ -624,94 +624,55 @@ enum open_frm_error { from persistent statistical tables */ + +#define TABLE_STAT_NO_STATS 0 +#define TABLE_STAT_TABLE 1 +#define TABLE_STAT_COLUMN 2 +#define TABLE_STAT_INDEX 4 +#define TABLE_STAT_HISTOGRAM 8 + +/* + EITS statistics information for a table. + + This data is loaded from mysql.{table|index|column}_stats tables and + then most of the time is owned by table's TABLE_SHARE object. + + Individual TABLE objects also have pointer to this object, and we do + reference counting to know when to free it. See + TABLE::update_engine_stats(), TABLE::free_engine_stats(), + TABLE_SHARE::update_engine_stats(), TABLE_SHARE::destroy(). + These implement a "shared pointer"-like functionality. + + When new statistics is loaded, we create new TABLE_STATISTICS_CB and make + the TABLE_SHARE point to it. Some TABLE object may still be using older + TABLE_STATISTICS_CB objects. Reference counting allows to free + TABLE_STATISTICS_CB when it is no longer used. +*/ + class TABLE_STATISTICS_CB { - class Statistics_state - { - enum state_codes - { - EMPTY, /** data is not loaded */ - LOADING, /** data is being loaded in some connection */ - READY /** data is loaded and available for use */ - }; - int32 state; - - public: - /** No state copy */ - Statistics_state &operator=(const Statistics_state &) { return *this; } - - /** Checks if data loading have been completed */ - bool is_ready() const - { - return my_atomic_load32_explicit(const_cast(&state), - MY_MEMORY_ORDER_ACQUIRE) == READY; - } - - /** - Sets mutual exclusion for data loading - - If stats are in LOADING state, waits until state change. - - @return - @retval true atomic EMPTY -> LOADING transfer completed, ok to load - @retval false stats are in READY state, no need to load - */ - bool start_load() - { - for (;;) - { - int32 expected= EMPTY; - if (my_atomic_cas32_weak_explicit(&state, &expected, LOADING, - MY_MEMORY_ORDER_RELAXED, - MY_MEMORY_ORDER_RELAXED)) - return true; - if (expected == READY) - return false; - (void) LF_BACKOFF(); - } - } - - /** Marks data available for subsequent use */ - void end_load() - { - DBUG_ASSERT(my_atomic_load32_explicit(&state, MY_MEMORY_ORDER_RELAXED) == - LOADING); - my_atomic_store32_explicit(&state, READY, MY_MEMORY_ORDER_RELEASE); - } - - /** Restores empty state on error (e.g. OOM) */ - void abort_load() - { - DBUG_ASSERT(my_atomic_load32_explicit(&state, MY_MEMORY_ORDER_RELAXED) == - LOADING); - my_atomic_store32_explicit(&state, EMPTY, MY_MEMORY_ORDER_RELAXED); - } - }; - - class Statistics_state stats_state; - class Statistics_state hist_state; + uint usage_count; // Instances of this stat public: + TABLE_STATISTICS_CB(); + ~TABLE_STATISTICS_CB(); MEM_ROOT mem_root; /* MEM_ROOT to allocate statistical data for the table */ Table_statistics *table_stats; /* Structure to access the statistical data */ ulong total_hist_size; /* Total size of all histograms */ + uint stats_available; - bool histograms_are_ready() const + bool histograms_exists() const { - return !total_hist_size || hist_state.is_ready(); + return total_hist_size != 0; } - - bool start_histograms_load() + bool unused() { - return total_hist_size && hist_state.start_load(); + return usage_count == 0; } - - void end_histograms_load() { hist_state.end_load(); } - void abort_histograms_load() { hist_state.abort_load(); } - bool stats_are_ready() const { return stats_state.is_ready(); } - bool start_stats_load() { return stats_state.start_load(); } - void end_stats_load() { stats_state.end_load(); } - void abort_stats_load() { stats_state.abort_load(); } + /* Copy (latest) state from TABLE_SHARE to TABLE */ + void update_stats_in_table(TABLE *table); + friend struct TABLE; + friend struct TABLE_SHARE; }; /** @@ -734,6 +695,7 @@ struct TABLE_SHARE TYPELIB *intervals; /* pointer to interval info */ mysql_mutex_t LOCK_ha_data; /* To protect access to ha_data */ mysql_mutex_t LOCK_share; /* To protect TABLE_SHARE */ + mysql_mutex_t LOCK_statistics; /* To protect against concurrent load */ TDC_element *tdc; @@ -750,7 +712,17 @@ struct TABLE_SHARE uint *blob_field; /* Index to blobs in Field arrray*/ LEX_CUSTRING vcol_defs; /* definitions of generated columns */ - TABLE_STATISTICS_CB stats_cb; + /* + EITS statistics data from the last time the table was opened or ANALYZE + table was run. + This is typically same as any related TABLE::stats_cb until ANALYZE + table is run. + This pointer is only to be de-referenced under LOCK_share as the + pointer can change by another thread running ANALYZE TABLE. + Without using a LOCK_share one can check if the statistics has been + updated by checking if TABLE::stats_cb != TABLE_SHARE::stats_cb. + */ + TABLE_STATISTICS_CB *stats_cb; uchar *default_values; /* row with default values */ LEX_CSTRING comment; /* Comment about table */ @@ -1174,7 +1146,6 @@ struct TABLE_SHARE void set_overlapped_keys(); void set_ignored_indexes(); key_map usable_indexes(THD *thd); - bool old_long_hash_function() const { return mysql_version < 100428 || @@ -1189,6 +1160,7 @@ struct TABLE_SHARE Item_func_hash *make_long_hash_func(THD *thd, MEM_ROOT *mem_root, List *field_list) const; + void update_engine_independent_stats(TABLE_STATISTICS_CB *stat); }; /* not NULL, but cannot be dereferenced */ @@ -1577,6 +1549,7 @@ public: and can be useful for range optimizer. */ Item *notnull_cond; + TABLE_STATISTICS_CB *stats_cb; inline void reset() { bzero((void*)this, sizeof(*this)); } void init(THD *thd, TABLE_LIST *tl); @@ -1606,6 +1579,8 @@ public: void mark_columns_used_by_virtual_fields(void); void mark_check_constraint_columns_for_read(void); int verify_constraints(bool ignore_failure); + void free_engine_stats(); + void update_engine_independent_stats(); inline void column_bitmaps_set(MY_BITMAP *read_set_arg) { read_set= read_set_arg; diff --git a/storage/maria/ma_locking.c b/storage/maria/ma_locking.c index c8e6394179c..9084be1d29d 100644 --- a/storage/maria/ma_locking.c +++ b/storage/maria/ma_locking.c @@ -395,7 +395,15 @@ int _ma_mark_file_changed(register MARIA_SHARE *share) if (!share->base.born_transactional) { if (!_MA_ALREADY_MARKED_FILE_CHANGED) - return _ma_mark_file_changed_now(share); + { + int res= _ma_mark_file_changed_now(share); + /* + Ensure that STATE_NOT_ANALYZED is reset on table changes + */ + share->state.changed|= (STATE_CHANGED | STATE_NOT_ANALYZED | + STATE_NOT_OPTIMIZED_KEYS); + return res; + } } else { @@ -409,10 +417,10 @@ int _ma_mark_file_changed(register MARIA_SHARE *share) (STATE_CHANGED | STATE_NOT_ANALYZED | STATE_NOT_OPTIMIZED_KEYS))) { - mysql_mutex_lock(&share->intern_lock); + mysql_mutex_lock(&share->intern_lock); share->state.changed|=(STATE_CHANGED | STATE_NOT_ANALYZED | - STATE_NOT_OPTIMIZED_KEYS); - mysql_mutex_unlock(&share->intern_lock); + STATE_NOT_OPTIMIZED_KEYS); + mysql_mutex_unlock(&share->intern_lock); } } return 0; @@ -430,7 +438,7 @@ int _ma_mark_file_changed_now(register MARIA_SHARE *share) if (! _MA_ALREADY_MARKED_FILE_CHANGED) { share->state.changed|=(STATE_CHANGED | STATE_NOT_ANALYZED | - STATE_NOT_OPTIMIZED_KEYS); + STATE_NOT_OPTIMIZED_KEYS); if (!share->global_changed) { share->changed= share->global_changed= 1; diff --git a/storage/myisam/mi_locking.c b/storage/myisam/mi_locking.c index 33a1c86c0d7..cee1c326b3e 100644 --- a/storage/myisam/mi_locking.c +++ b/storage/myisam/mi_locking.c @@ -603,12 +603,15 @@ int _mi_mark_file_changed(MI_INFO *info) { uchar buff[3]; register MYISAM_SHARE *share=info->s; + uint32 state; DBUG_ENTER("_mi_mark_file_changed"); - if (!(share->state.changed & STATE_CHANGED) || ! share->global_changed) + state= share->state.changed; + share->state.changed|= (STATE_CHANGED | STATE_NOT_ANALYZED | + STATE_NOT_OPTIMIZED_KEYS); + + if (!(state & STATE_CHANGED) || ! share->global_changed) { - share->state.changed|=(STATE_CHANGED | STATE_NOT_ANALYZED | - STATE_NOT_OPTIMIZED_KEYS); if (!share->global_changed) { share->global_changed=1;