From b9f5793176d722da8ecbe073c8e4888e79638793 Mon Sep 17 00:00:00 2001 From: Monty Date: Thu, 14 Mar 2024 17:59:00 +0100 Subject: [PATCH] MDEV-9101 Limit size of created disk temporary files and tables Two new variables added: - max_tmp_space_usage : Limits the the temporary space allowance per user - max_total_tmp_space_usage: Limits the temporary space allowance for all users. New status variables: tmp_space_used & max_tmp_space_used New field in information_schema.process_list: TMP_SPACE_USED The temporary space is counted for: - All SQL level temporary files. This includes files for filesort, transaction temporary space, analyze, binlog_stmt_cache etc. It does not include engine internal temporary files used for repair, alter table, index pre sorting etc. - All internal on disk temporary tables created as part of resolving a SELECT, multi-source update etc. Special cases: - When doing a commit, the last flush of the binlog_stmt_cache will not cause an error even if the temporary space limit is exceeded. This is to avoid giving errors on commit. This means that a user can temporary go over the limit with up to binlog_stmt_cache_size. Noteworthy issue: - One has to be careful when using small values for max_tmp_space_limit together with binary logging and with non transactional tables. If a the binary log entry for the query is bigger than binlog_stmt_cache_size and one hits the limit of max_tmp_space_limit when flushing the entry to disk, the query will abort and the binary log will not contain the last changes to the table. This will also stop the slave! This is also true for all Aria tables as Aria cannot do rollback (except in case of crashes)! One way to avoid it is to use @@binlog_format=statement for queries that updates a lot of rows. Implementation: - All writes to temporary files or internal temporary tables, that increases the file size, are routed through temp_file_size_cb_func() which updates and checks the temp space usage. - Most of the temporary file monitoring is done inside IO_CACHE. Temporary file monitoring is done inside the Aria engine. - MY_TRACK and MY_TRACK_WITH_LIMIT are new flags for ini_io_cache(). MY_TRACK means that we track the file usage. TRACK_WITH_LIMIT means that we track the file usage and we give an error if the limit is breached. This is used to not give an error on commit when binlog_stmp_cache is flushed. - global_tmp_space_used contains the total tmp space used so far. This is needed quickly check against max_total_tmp_space_usage. - Temporary space errors are using EE_LOCAL_TMP_SPACE_FULL and handler errors are using HA_ERR_LOCAL_TMP_SPACE_FULL. This is needed until we move general errors to it's own error space so that they cannot conflict with system error numbers. - Return value of my_chsize() and mysql_file_chsize() has changed so that -1 is returned in the case my_chsize() could not decrease the file size (very unlikely and will not happen on modern systems). All calls to _chsize() are updated to check for > 0 as the error condition. - At the destruction of THD we check that THD::tmp_file_space == 0 - At server end we check that global_tmp_space_used == 0 - As a precaution against errors in the tmp_space_used code, one can set max_tmp_space_usage and max_total_tmp_space_usage to 0 to disable the tmp space quota errors. - truncate_io_cache() function added. - Aria tables using static or dynamic row length are registered in 8K increments to avoid some calls to update_tmp_file_size(). Other things: - Ensure that all handler errors are registered. Before, some engine errors could be printed as "Unknown error". - Fixed bug in filesort() that causes a assert if there was an error when writing to the temporay file. - Fixed that compute_window_func() now takes into account write errors. - In case of parallel replication, rpl_group_info::cleanup_context() could call trans_rollback() with thd->error set, which would cause an assert. Fixed by resetting the error before calling trans_rollback(). - Fixed bug in subselect3.inc which caused following test to use heap tables with low value for max_heap_table_size - Fixed bug in sql_expression_cache where it did not overflow heap table to Aria table. - Added Max_tmp_disk_space_used to slow query log. - Fixed some bugs in log_slow_innodb.test --- include/my_base.h | 5 +- include/my_handler_errors.h | 5 +- include/my_sys.h | 23 +- include/mysys_err.h | 4 +- mysql-test/include/log_grep.inc | 13 +- mysql-test/main/create.result | 6 +- mysql-test/main/log_slow_innodb.result | 63 ++++-- mysql-test/main/log_slow_innodb.test | 40 +++- mysql-test/main/mysqld--help.result | 9 + mysql-test/main/status.result | 13 +- mysql-test/main/status.test | 6 +- mysql-test/main/subselect3.inc | 3 +- mysql-test/main/subselect3.result | 2 +- mysql-test/main/subselect3_jcl6.result | 2 +- mysql-test/main/tmp_space_usage-master.opt | 4 + mysql-test/main/tmp_space_usage.result | 153 +++++++++++++ mysql-test/main/tmp_space_usage.test | 203 ++++++++++++++++++ .../funcs_1/datadict/processlist_val.inc | 14 +- .../suite/funcs_1/r/is_columns_is.result | 2 + .../funcs_1/r/is_columns_is_embedded.result | 2 + .../funcs_1/r/processlist_priv_no_prot.result | 142 ++++++------ .../funcs_1/r/processlist_priv_ps.result | 142 ++++++------ .../funcs_1/r/processlist_val_no_prot.result | 47 ++-- .../suite/funcs_1/r/processlist_val_ps.result | 47 ++-- mysql-test/suite/rpl/r/tmp_space_usage.result | 25 +++ mysql-test/suite/rpl/t/tmp_space_usage.test | 38 ++++ .../sys_vars/r/sysvars_server_embedded.result | 20 ++ .../r/sysvars_server_notembedded.result | 20 ++ mysys/errors.c | 11 +- mysys/mf_cache.c | 8 +- mysys/mf_iocache.c | 110 ++++++++-- mysys/my_chsize.c | 7 +- plugin/feedback/sender_thread.cc | 3 +- sql/filesort.cc | 19 +- sql/handler.cc | 13 +- sql/json_table.cc | 3 +- sql/log.cc | 76 ++++--- sql/log_cache.h | 11 +- sql/log_event_client.cc | 8 +- sql/mf_iocache_encr.cc | 3 +- sql/mysqld.cc | 85 +++++++- sql/mysqld.h | 2 + sql/rpl_rli.cc | 16 +- sql/sql_class.cc | 40 +++- sql/sql_class.h | 30 ++- sql/sql_expression_cache.cc | 5 +- sql/sql_parse.cc | 3 +- sql/sql_select.cc | 12 +- sql/sql_show.cc | 3 + sql/sql_statistics.cc | 9 +- sql/sql_update.cc | 3 +- sql/sql_window.cc | 7 +- sql/sys_vars.cc | 18 ++ sql/uniques.cc | 4 +- storage/csv/ha_tina.cc | 2 +- storage/maria/ha_maria.cc | 24 +++ storage/maria/ma_bitmap.c | 6 +- storage/maria/ma_blockrec.c | 13 +- storage/maria/ma_check.c | 10 +- storage/maria/ma_close.c | 6 +- storage/maria/ma_create.c | 5 +- storage/maria/ma_delete_all.c | 11 +- storage/maria/ma_dynrec.c | 10 + storage/maria/ma_info.c | 21 ++ storage/maria/ma_loghandler.c | 2 +- storage/maria/ma_open.c | 1 + storage/maria/ma_page.c | 7 + storage/maria/ma_recovery.c | 2 +- storage/maria/ma_state.c | 18 +- storage/maria/ma_state.h | 2 +- storage/maria/ma_statrec.c | 7 + storage/maria/ma_write.c | 6 + storage/maria/maria_def.h | 18 ++ storage/myisam/mi_check.c | 10 +- storage/myisam/mi_create.c | 4 +- storage/myisam/mi_delete_all.c | 4 +- storage/rocksdb/rdb_index_merge.cc | 2 +- 77 files changed, 1373 insertions(+), 380 deletions(-) create mode 100644 mysql-test/main/tmp_space_usage-master.opt create mode 100644 mysql-test/main/tmp_space_usage.result create mode 100644 mysql-test/main/tmp_space_usage.test create mode 100644 mysql-test/suite/rpl/r/tmp_space_usage.result create mode 100644 mysql-test/suite/rpl/t/tmp_space_usage.test diff --git a/include/my_base.h b/include/my_base.h index b214ad045f3..8f562896bea 100644 --- a/include/my_base.h +++ b/include/my_base.h @@ -50,6 +50,7 @@ #define HA_OPEN_FOR_CREATE 4096U #define HA_OPEN_FOR_DROP (1U << 13) /* Open part of drop */ #define HA_OPEN_GLOBAL_TMP_TABLE (1U << 14) /* TMP table used by repliction */ +#define HA_OPEN_SIZE_TRACKING (1U << 15) /* Allow opening even if table is incompatible as this is for ALTER TABLE which @@ -543,7 +544,9 @@ enum ha_base_keytype { #define HA_ERR_COMMIT_ERROR 197 #define HA_ERR_PARTITION_LIST 198 #define HA_ERR_NO_ENCRYPTION 199 -#define HA_ERR_LAST 199 /* Copy of last error nr * */ +#define HA_ERR_LOCAL_TMP_SPACE_FULL 200 +#define HA_ERR_GLOBAL_TMP_SPACE_FULL 201 +#define HA_ERR_LAST 201 /* Copy of last error nr * */ /* Number of different errors */ #define HA_ERR_ERRORS (HA_ERR_LAST - HA_ERR_FIRST + 1) diff --git a/include/my_handler_errors.h b/include/my_handler_errors.h index 2f8efe11017..4a9520883a8 100644 --- a/include/my_handler_errors.h +++ b/include/my_handler_errors.h @@ -110,7 +110,10 @@ static const char *handler_error_messages[]= "Sequence values are conflicting", "Error during commit", "Cannot select partitions", - "Cannot initialize encryption. Check that all encryption parameters have been set" + "Cannot initialize encryption. Check that all encryption parameters have been set", + /* 200 */ + "Local temporary space limit reached", + "Global temporary space limit reached" }; #endif /* MYSYS_MY_HANDLER_ERRORS_INCLUDED */ diff --git a/include/my_sys.h b/include/my_sys.h index d5e8ebf6c6a..f347d4797e6 100644 --- a/include/my_sys.h +++ b/include/my_sys.h @@ -94,6 +94,8 @@ C_MODE_START #define MY_ROOT_USE_MPROTECT 0x20000U /* init_alloc_root: read only segments */ /* Tree that should delete things automatically */ #define MY_TREE_WITH_DELETE 0x40000U +#define MY_TRACK 0x80000U /* Track tmp usage */ +#define MY_TRACK_WITH_LIMIT 0x100000U /* Give error if over tmp_file_usage */ #define MY_CHECK_ERROR 1U /* Params to my_end; Check open-close */ #define MY_GIVE_INFO 2U /* Give time info about process*/ @@ -178,6 +180,17 @@ uchar *my_large_malloc(size_t *size, myf my_flags); void my_large_free(void *ptr, size_t size); void my_large_page_truncate(size_t *size); +/* Tracking tmp file usage */ + +struct tmp_file_tracking +{ + ulonglong previous_file_size; + ulonglong file_size; +}; + +typedef int (*TMPFILE_SIZE_CB)(struct tmp_file_tracking *track, int no_error); +extern TMPFILE_SIZE_CB update_tmp_file_size; + #ifdef _WIN32 extern BOOL my_obtain_privilege(LPCSTR lpPrivilege); #endif @@ -431,6 +444,8 @@ typedef struct st_io_cache /* Used when caching files */ */ IO_CACHE_SHARE *share; + /* Track tmpfile usage. Done if (myflags & MY_TRACK) is true */ + struct tmp_file_tracking tracking; /* A caller will use my_b_read() macro to read from the cache if the data is already in cache, it will be simply copied with @@ -498,6 +513,8 @@ extern PSI_file_key key_file_io_cache; /* inline functions for mf_iocache */ extern int my_b_flush_io_cache(IO_CACHE *info, int need_append_buffer_lock); +extern void end_tracking_io_cache(IO_CACHE *info); +extern void truncate_io_cache(IO_CACHE *info); extern int _my_b_get(IO_CACHE *info); extern int _my_b_read(IO_CACHE *info,uchar *Buffer,size_t Count); extern int _my_b_write(IO_CACHE *info,const uchar *Buffer,size_t Count); @@ -831,8 +848,10 @@ extern my_bool open_cached_file(IO_CACHE *cache,const char *dir, myf cache_myflags); extern my_bool real_open_cached_file(IO_CACHE *cache); extern void close_cached_file(IO_CACHE *cache); -File create_temp_file(char *to, const char *dir, const char *pfx, - int mode, myf MyFlags); +extern File create_temp_file(char *to, const char *dir, const char *pfx, + int mode, myf MyFlags); +extern my_bool io_cache_tmp_file_track(IO_CACHE *info, ulonglong file_size); + #define my_init_dynamic_array(A,B,C,D,E,F) init_dynamic_array2(A,B,C,NULL,D,E,F) #define my_init_dynamic_array2(A,B,C,D,E,F,G) init_dynamic_array2(A,B,C,D,E,F,G) extern my_bool init_dynamic_array2(PSI_memory_key psi_key, DYNAMIC_ARRAY *array, diff --git a/include/mysys_err.h b/include/mysys_err.h index d115b5ddb70..3ace837a84b 100644 --- a/include/mysys_err.h +++ b/include/mysys_err.h @@ -74,7 +74,9 @@ extern const char *globerrs[]; /* my_error_messages is here */ #define EE_MEMCNTL 38 #define EE_DUPLICATE_CHARSET 39 #define EE_NAME_DEPRECATED 40 -#define EE_ERROR_LAST 40 /* Copy last error nr */ +#define EE_LOCAL_TMP_SPACE_FULL 41 +#define EE_GLOBAL_TMP_SPACE_FULL 42 +#define EE_ERROR_LAST 42 /* Copy last error nr */ /* Add error numbers before EE_ERROR_LAST and change it accordingly. */ diff --git a/mysql-test/include/log_grep.inc b/mysql-test/include/log_grep.inc index 6be42214c95..d667a031261 100644 --- a/mysql-test/include/log_grep.inc +++ b/mysql-test/include/log_grep.inc @@ -54,7 +54,8 @@ perl; while () { print ; } - print $command_file "--let \$log_grep_failed= 1;\n"; + print $command_file "--let \$log_grep_failed= 1\n"; + print $command_file "--let \$log_grep_lines= $lines\n"; } else { print "[log_grep.inc] found expected match count: $log_expected_matches\n"; } @@ -69,6 +70,14 @@ EOF --remove_file $LOG_GREP_PERL_RESULT if ($log_grep_failed) { + --echo # + --echo # ERROR: report + --echo # + --echo log_file: $log_file + --echo grep_pattern: $grep_pattern + --echo Expected_matches: $log_expected_matches + --echo Found_matches: $log_grep_lines + SHOW SESSION STATUS LIKE 'Slow_queries'; SHOW GLOBAL VARIABLES LIKE 'log%'; SHOW GLOBAL VARIABLES LIKE 'long_query_time'; @@ -80,5 +89,5 @@ if ($log_grep_failed) SHOW SESSION VARIABLES LIKE 'min_examined_row_limit'; SHOW SESSION VARIABLES LIKE 'query_cache%'; SHOW SESSION VARIABLES LIKE 'slow_query%'; - --die Testcase failed! + --die Testcase failed! Error rapport above. } diff --git a/mysql-test/main/create.result b/mysql-test/main/create.result index d7f4eb274ea..ce5c66ca5fc 100644 --- a/mysql-test/main/create.result +++ b/mysql-test/main/create.result @@ -1089,7 +1089,8 @@ t1 CREATE TABLE `t1` ( `SENT_ROWS` bigint(10) NOT NULL, `QUERY_ID` bigint(10) NOT NULL, `INFO_BINARY` blob, - `TID` bigint(10) NOT NULL + `TID` bigint(10) NOT NULL, + `TMP_SPACE_USED` bigint(10) NOT NULL ) DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_general_ci drop table t1; create temporary table t1 like information_schema.processlist; @@ -1114,7 +1115,8 @@ t1 CREATE TEMPORARY TABLE `t1` ( `SENT_ROWS` bigint(10) NOT NULL, `QUERY_ID` bigint(10) NOT NULL, `INFO_BINARY` blob, - `TID` bigint(10) NOT NULL + `TID` bigint(10) NOT NULL, + `TMP_SPACE_USED` bigint(10) NOT NULL ) DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_general_ci drop table t1; create table t1 like information_schema.character_sets; diff --git a/mysql-test/main/log_slow_innodb.result b/mysql-test/main/log_slow_innodb.result index a7c2ac86eed..cd84aed0830 100644 --- a/mysql-test/main/log_slow_innodb.result +++ b/mysql-test/main/log_slow_innodb.result @@ -43,36 +43,53 @@ SELECT 1; [log_grep.inc] lines: 0 [log_grep.inc] file: log_slow_innodb-verbosity_2 pattern: ^# Tmp_tables: \d+ Tmp_disk_tables: \d+$ [log_grep.inc] lines: 0 +# +# Test 'query_plan' +# SET SESSION log_slow_verbosity='query_plan'; +[slow_log_start.inc] log_slow_innodb-verbosity_3 +INSERT INTO t1 VALUE(1001,1001); [log_slow_stop.inc] log_slow_innodb-verbosity_3 ---source include/log_slow_start.inc -INSERT INTO t1 VALUE(1000) -[log_grep.inc] file: log_slow_innodb-verbosity_3 ---source include/log_slow_start.inc -INSERT INTO t1 VALUE(1000) pattern: ^# Thread_id: .+ Schema: .+ QC_hit: (Yes|No)$ expected_matches: 2 +[log_grep.inc] file: log_slow_innodb-verbosity_3 pattern: ^# Thread_id: .+ Schema: .+ QC_hit: (Yes|No)$ expected_matches: 2 [log_grep.inc] found expected match count: 2 -[log_grep.inc] file: log_slow_innodb-verbosity_3 ---source include/log_slow_start.inc -INSERT INTO t1 VALUE(1000) pattern: ^# Query_time: \d+\.\d+ Lock_time: \d+\.\d+ Rows_sent: \d+ Rows_examined: \d+$ expected_matches: 2 +[log_grep.inc] file: log_slow_innodb-verbosity_3 pattern: ^# Query_time: \d+\.\d+ Lock_time: \d+\.\d+ Rows_sent: \d+ Rows_examined: \d+$ expected_matches: 2 [log_grep.inc] found expected match count: 2 -[log_grep.inc] file: log_slow_innodb-verbosity_3 ---source include/log_slow_start.inc -INSERT INTO t1 VALUE(1000) pattern: ^# Rows_affected: \d+ Bytes_sent: \d+$ expected_matches: 2 +[log_grep.inc] file: log_slow_innodb-verbosity_3 pattern: ^# Rows_affected: \d+ Bytes_sent: \d+$ expected_matches: 2 [log_grep.inc] found expected match count: 2 -[log_grep.inc] file: log_slow_innodb-verbosity_3 ---source include/log_slow_start.inc -INSERT INTO t1 VALUE(1000) pattern: ^# Full_scan: (Yes|No) Full_join: (Yes|No) Tmp_table: (Yes|No) Tmp_table_on_disk: (Yes|No)$ +[log_grep.inc] file: log_slow_innodb-verbosity_3 pattern: ^# Full_scan: (Yes|No) Full_join: (Yes|No) Tmp_table: (Yes|No) Tmp_table_on_disk: (Yes|No)$ [log_grep.inc] lines: 0 -[log_grep.inc] file: log_slow_innodb-verbosity_3 ---source include/log_slow_start.inc -INSERT INTO t1 VALUE(1000) pattern: ^# Filesort: (Yes|No) Filesort_on_disk: (Yes|No) Merge_passes: \d+\ Priority_queue: (Yes|No)$ +[log_grep.inc] file: log_slow_innodb-verbosity_3 pattern: ^# Filesort: (Yes|No) Filesort_on_disk: (Yes|No) Merge_passes: \d+\ Priority_queue: (Yes|No)$ [log_grep.inc] lines: 0 -[log_grep.inc] file: log_slow_innodb-verbosity_3 ---source include/log_slow_start.inc -INSERT INTO t1 VALUE(1000) pattern: ^# Filesort: (Yes|No) Filesort_on_disk: (Yes|No) Merge_passes: \d+\ Priority_queue: (Yes|No)$ +[log_grep.inc] file: log_slow_innodb-verbosity_3 pattern: ^# Filesort: (Yes|No) Filesort_on_disk: (Yes|No) Merge_passes: \d+\ Priority_queue: (Yes|No)$ [log_grep.inc] lines: 0 -[log_grep.inc] file: log_slow_innodb-verbosity_3 ---source include/log_slow_start.inc -INSERT INTO t1 VALUE(1000) pattern: ^# Tmp_tables: \d+ Tmp_disk_tables: \d+$ +[log_grep.inc] file: log_slow_innodb-verbosity_3 pattern: ^# Tmp_tables: \d+ Tmp_disk_tables: \d+$ [log_grep.inc] lines: 0 DROP TABLE t1; +# +# Test max_tmp_disk_space_used +# +CREATE TABLE t1(a INT primary key, c varchar(1024)) ENGINE=InnoDB; +insert into t1 select seq, concat(repeat(char(65+mod(seq,61)),32),repeat(char(65+mod(seq,59)),32)) from seq_1_to_1000; +SET SESSION log_slow_verbosity='query_plan'; +SET SESSION sort_buffer_size=16384; +SET SESSION long_query_time=0; +[slow_log_start.inc] log_slow_innodb-verbosity_4 +SELECT c, count(*) from t1 group by c order by 2 limit 700,10; +c count(*) +CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM 2 +DDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN 2 +EEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOO 2 +FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPPP 2 +GGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQ 2 +HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRR 2 +IIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIISSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSS 2 +JJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJJTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTT 2 +KKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKKUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUUU 2 +LLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV 2 +[log_slow_stop.inc] log_slow_innodb-verbosity_4 +[log_grep.inc] file: log_slow_innodb-verbosity_4 pattern: ^# Max_tmp_disk_space_used.* expected_matches: 1 +[log_grep.inc] found expected match count: 1 +drop table t1; +# +# Cleanup +# diff --git a/mysql-test/main/log_slow_innodb.test b/mysql-test/main/log_slow_innodb.test index 31eab5f414f..30fc8b941df 100644 --- a/mysql-test/main/log_slow_innodb.test +++ b/mysql-test/main/log_slow_innodb.test @@ -61,19 +61,49 @@ SELECT 1; --let log_slow_verbosity_innodb_expected_matches= 0 --source include/log_slow_grep.inc -# -# Test 'query_plan' -# +--echo # +--echo # Test 'query_plan' +--echo # SET SESSION log_slow_verbosity='query_plan'; -let log_file=$log_slow_prefix-verbosity_3 +--let log_file=$log_slow_prefix-verbosity_3 --source include/log_slow_start.inc -INSERT INTO t1 VALUE(1000); +INSERT INTO t1 VALUE(1001,1001); --source include/log_slow_stop.inc --let log_slow_verbosity_innodb_expected_matches= 1 --source include/log_slow_grep.inc DROP TABLE t1; + +--echo # +--echo # Test max_tmp_disk_space_used +--echo # + +CREATE TABLE t1(a INT primary key, c varchar(1024)) ENGINE=InnoDB; +insert into t1 select seq, concat(repeat(char(65+mod(seq,61)),32),repeat(char(65+mod(seq,59)),32)) from seq_1_to_1000; + +SET SESSION log_slow_verbosity='query_plan'; +SET SESSION sort_buffer_size=16384; +SET SESSION long_query_time=0; +--let log_file=$log_slow_prefix-verbosity_4 + +--disable_ps_protocol +--source include/log_slow_start.inc +SELECT c, count(*) from t1 group by c order by 2 limit 700,10; +--source include/log_slow_stop.inc +--enable_ps_protocol + +--let grep_pattern = ^# Max_tmp_disk_space_used.* + +--let log_expected_matches= 1 +--source include/log_grep.inc + +drop table t1; + +--echo # +--echo # Cleanup +--echo # + --source include/log_slow_cleanup.inc diff --git a/mysql-test/main/mysqld--help.result b/mysql-test/main/mysqld--help.result index 1546735b8fe..adb641babb8 100644 --- a/mysql-test/main/mysqld--help.result +++ b/mysql-test/main/mysqld--help.result @@ -670,6 +670,13 @@ The following specify which files/extra groups are read (specified before remain seconds will be aborted. The argument will be treated as a decimal value with microsecond precision. A value of 0 (default) means no timeout + --max-tmp-session-space-usage=# + The maximum total size of temporary file and temporary + table usage. A value of 0 disables this feature + --max-tmp-total-space-usage=# + The maximum total size of all temporary file and + temporary table usage over all connections. A value of 0 + disables this feature --max-user-connections=# The maximum number of active connections for a single user (0 = no limit) @@ -1766,6 +1773,8 @@ max-session-mem-used 9223372036854775807 max-sort-length 1024 max-sp-recursion-depth 0 max-statement-time 0 +max-tmp-session-space-usage 1099511627776 +max-tmp-total-space-usage 1099511627776 max-user-connections 0 max-write-lock-count 18446744073709551615 memlock FALSE diff --git a/mysql-test/main/status.result b/mysql-test/main/status.result index ebee9e3c79f..eda63c4f05b 100644 --- a/mysql-test/main/status.result +++ b/mysql-test/main/status.result @@ -183,7 +183,7 @@ show status like 'hand%write%'; Variable_name Value Handler_tmp_write 0 Handler_write 0 -show status like '%tmp%'; +show status like '%\_tmp%'; Variable_name Value Created_tmp_disk_tables 0 Created_tmp_files 0 @@ -191,12 +191,13 @@ Created_tmp_tables 0 Handler_tmp_delete 0 Handler_tmp_update 0 Handler_tmp_write 0 +Max_tmp_space_used 0 Rows_tmp_read 5 show status like 'hand%write%'; Variable_name Value Handler_tmp_write 0 Handler_write 0 -show status like '%tmp%'; +show status like '%\_tmp%'; Variable_name Value Created_tmp_disk_tables 0 Created_tmp_files 0 @@ -204,12 +205,13 @@ Created_tmp_tables 0 Handler_tmp_delete 0 Handler_tmp_update 0 Handler_tmp_write 0 -Rows_tmp_read 14 +Max_tmp_space_used 0 +Rows_tmp_read 15 show status like 'com_show_status'; Variable_name Value Com_show_status 8 rnd_diff tmp_table_diff -30 8 +32 8 disconnect con1; connection default; flush status; @@ -340,7 +342,7 @@ Handler_tmp_update 2 Handler_tmp_write 7 Handler_update 0 Handler_write 4 -show status like '%tmp%'; +show status like '%_tmp%'; Variable_name Value Created_tmp_disk_tables 1 Created_tmp_files 0 @@ -348,6 +350,7 @@ Created_tmp_tables 2 Handler_tmp_delete 0 Handler_tmp_update 2 Handler_tmp_write 7 +Max_tmp_space_used 32768 Rows_tmp_read 44 drop table t1; CREATE TABLE t1 (i int(11) DEFAULT NULL, KEY i (i) ) ENGINE=MyISAM; diff --git a/mysql-test/main/status.test b/mysql-test/main/status.test index a1cc028b98a..1d6a7b4a898 100644 --- a/mysql-test/main/status.test +++ b/mysql-test/main/status.test @@ -244,9 +244,9 @@ let $rnd_next = `show global status like 'handler_read_rnd_next'`; let $tmp_table = `show global status like 'Created_tmp_tables'`; show status like 'com_show_status'; show status like 'hand%write%'; -show status like '%tmp%'; +show status like '%\_tmp%'; show status like 'hand%write%'; -show status like '%tmp%'; +show status like '%\_tmp%'; show status like 'com_show_status'; let $rnd_next2 = `show global status like 'handler_read_rnd_next'`; let $tmp_table2 = `show global status like 'Created_tmp_tables'`; @@ -367,7 +367,7 @@ select b, count(*) from t1 group by b; select g, count(*) from t1 group by g; show status like 'Row%'; show status like 'Handler%'; -show status like '%tmp%'; +show status like '%_tmp%'; drop table t1; # diff --git a/mysql-test/main/subselect3.inc b/mysql-test/main/subselect3.inc index 8bb84eb24d9..8a02c0f835e 100644 --- a/mysql-test/main/subselect3.inc +++ b/mysql-test/main/subselect3.inc @@ -985,7 +985,7 @@ flush status; select count(*) from t0 A, t0 B, t0 C, t0 D where D.a in (select a from t1 E where a+1 < 10000 + A.a + B.a +C.a+D.a); --enable_ps2_protocol show status like 'Created_tmp_disk_tables'; -set @save_max_heap_table_size=@@max_heap_table_size; +set @@max_heap_table_size=@save_max_heap_table_size; set @@optimizer_switch=@save_optimizer_switch; drop table t0, t1; @@ -1137,6 +1137,7 @@ set @@optimizer_switch=@save_optimizer_switch; --echo --echo BUG#37842: Assertion in DsMrr_impl::dsmrr_init, at handler.cc:4307 --echo + CREATE TABLE t1 ( `pk` int(11) NOT NULL AUTO_INCREMENT, `int_key` int(11) DEFAULT NULL, diff --git a/mysql-test/main/subselect3.result b/mysql-test/main/subselect3.result index 4c654cb736f..550a9d12901 100644 --- a/mysql-test/main/subselect3.result +++ b/mysql-test/main/subselect3.result @@ -1179,7 +1179,7 @@ count(*) show status like 'Created_tmp_disk_tables'; Variable_name Value Created_tmp_disk_tables 1 -set @save_max_heap_table_size=@@max_heap_table_size; +set @@max_heap_table_size=@save_max_heap_table_size; set @@optimizer_switch=@save_optimizer_switch; drop table t0, t1; create table t0 (a int); diff --git a/mysql-test/main/subselect3_jcl6.result b/mysql-test/main/subselect3_jcl6.result index b606f0a5127..d74c04ce589 100644 --- a/mysql-test/main/subselect3_jcl6.result +++ b/mysql-test/main/subselect3_jcl6.result @@ -1182,7 +1182,7 @@ count(*) show status like 'Created_tmp_disk_tables'; Variable_name Value Created_tmp_disk_tables 1 -set @save_max_heap_table_size=@@max_heap_table_size; +set @@max_heap_table_size=@save_max_heap_table_size; set @@optimizer_switch=@save_optimizer_switch; drop table t0, t1; create table t0 (a int); diff --git a/mysql-test/main/tmp_space_usage-master.opt b/mysql-test/main/tmp_space_usage-master.opt new file mode 100644 index 00000000000..c6d6297bde1 --- /dev/null +++ b/mysql-test/main/tmp_space_usage-master.opt @@ -0,0 +1,4 @@ +--log-bin +--binlog-format=row +--max_tmp_session_space_usage=512K +--max_tmp_total_space_usage=768K diff --git a/mysql-test/main/tmp_space_usage.result b/mysql-test/main/tmp_space_usage.result new file mode 100644 index 00000000000..8436ba9d10e --- /dev/null +++ b/mysql-test/main/tmp_space_usage.result @@ -0,0 +1,153 @@ +call mtr.add_suppression("Write to binary log failed: .* temporary space limit reached. An incident event is written to binary log.*"); +flush status; +flush global status; +show session status like "max_tmp_space_used"; +Variable_name Value +Max_tmp_space_used 0 +show global status like "max_tmp_space_used"; +Variable_name Value +Max_tmp_space_used 0 +# +# MDEV-9101 Limit size of total size of created disk temporary files +# and tables. +show session status like "tmp_space_used"; +Variable_name Value +Tmp_space_used 0 +show global status like "tmp_space_used"; +Variable_name Value +Tmp_space_used 0 +select @@global.max_tmp_session_space_usage, @@global.max_tmp_total_space_usage; +@@global.max_tmp_session_space_usage @@global.max_tmp_total_space_usage +524288 786432 +select @@binlog_stmt_cache_size,@@binlog_format; +@@binlog_stmt_cache_size @@binlog_format +32768 ROW +create table t1 (a int primary key, v varchar(256), c int default(0)) engine=innodb; +insert into t1 (a,v) select seq, repeat(char(64+mod(seq,32)),mod(seq,254)+1) from seq_1_to_65536; +ERROR HY000: Local temporary space limit reached +set @@max_tmp_session_space_usage=1024*1024*1024; +insert into t1 (a,v) select seq, repeat(char(64+mod(seq,32)),mod(seq,254)+1) from seq_1_to_65536; +ERROR HY000: Global temporary space limit reached +set @@max_tmp_session_space_usage=default; +set @@binlog_format="statement"; +insert into t1 (a,v) select seq, repeat(char(64+mod(seq,32)),mod(seq,254)+1) from seq_1_to_65536; +insert into t1 (a,v) select seq, repeat(char(64+mod(seq,32)),mod(seq,254)+1) from seq_65537_to_131072; +select count(*) from t1; +count(*) +131072 +create table t2 (a int, b int) engine=innodb select seq as a, 0 as b from seq_1_to_131072; +set @@binlog_format="row"; +set @@tmp_memory_table_size=32*1024; +# The following queries should fail because of tmp_space_usage +select * from t1 order by a,v; +ERROR HY000: Local temporary space limit reached +select v,count(*) from t1 group by v limit 2; +ERROR HY000: Local temporary space limit reached +update t1 set v=right(v,2); +ERROR HY000: Local temporary space limit reached +set @@binlog_format="statement"; +set @@max_tmp_session_space_usage=65536; +set @@tmp_memory_table_size=0; +update t1,t2 set t1.c=t2.a, t2.b=1 where t1.a=t2.a; +ERROR HY000: Local temporary space limit reached +set @@binlog_format="row"; +set @@max_tmp_session_space_usage=default; +drop table t1,t2; +# +# Check max_tmp_total_space_usage & processlist +# +set @@tmp_memory_table_size=1024*1024; +show session status like "tmp_space_used"; +Variable_name Value +Tmp_space_used 0 +set @@tmp_memory_table_size=0; +flush status; +# session.max_tmp_session_space_usage == global.max_tmp_session_space_usage +connect c1, localhost, root,,; +connection default; +create table t1 (a int primary key, v varchar(256), c int default(0)) engine=innodb; +create table t2 (a int primary key, v varchar(256), c int default(0)) engine=innodb; +begin; +insert into t1 (a,v) select seq, repeat(char(64+mod(seq,32)),mod(seq,254)+1) from seq_1_to_3000; +connection c1; +# information_schema.process_list.tmp_space_used == status.tmp_space_used +insert into t2 (a,v) select seq, repeat(char(64+mod(seq,32)),mod(seq,254)+1) from seq_1_to_3000; +ERROR HY000: Global temporary space limit reached +# Test setting tmp_space_usage to 0 to disable quotas +set @save_max_tmp_total_space_usage=@@global.max_tmp_total_space_usage; +set @@global.max_tmp_total_space_usage=0; +set @@max_tmp_session_space_usage=0; +insert into t2 (a,v) select seq, repeat(char(64+mod(seq,32)),mod(seq,254)+1) from seq_1_to_3000; +set @@global.max_tmp_total_space_usage=@save_max_tmp_total_space_usage; +set @@max_tmp_session_space_usage=0; +connection default; +insert into t1 (a,v) values(9999990,0); +commit; +select count(*) from t1; +count(*) +3001 +disconnect c1; +drop table t1,t2; +# +# Test case from Elena +# +SET @@max_tmp_session_space_usage= 64*1024; +set @@binlog_format="statement"; +CREATE OR REPLACE TABLE t1 (a INT, b INT); +select benchmark(1,1); +benchmark(1,1) +0 +INSERT INTO t1 SELECT seq, seq FROM seq_1_to_100000; +ALTER TABLE t1 ORDER BY a, b; +ERROR HY000: Local temporary space limit reached +DROP TABLE t1; +# +# Show that setting max tmp space too low value can stop binary logging +# if non transactional tables are used. +# +set @save_max_tmp_total_space_usage=@@global.max_tmp_total_space_usage; +SET @@global.max_tmp_total_space_usage=64*1024; +set @@binlog_format="row"; +create table t1 (a int primary key, v varchar(256)) engine=myisam; +insert into t1 (a,v) select seq, repeat(char(64+mod(seq,32)),mod(seq,254)+1) from seq_1_to_65536; +Got one of the listed errors +show warnings; +Level Code Message +Error 42 Global temporary space limit reached +Error 1534 Writing one row to the row-based binary log failed +Warning 1196 Some non-transactional changed tables couldn't be rolled back +select count(*) <> 0 from t1; +count(*) <> 0 +1 +truncate table t1; +alter table t1 engine=innodb; +insert into t1 (a,v) select seq, repeat(char(64+mod(seq,32)),mod(seq,254)+1) from seq_1_to_65536; +Got one of the listed errors +show warnings; +Level Code Message +Error 42 Global temporary space limit reached +Error 1534 Writing one row to the row-based binary log failed +select count(*) <> 0 from t1; +count(*) <> 0 +0 +drop table t1; +set @@global.max_tmp_total_space_usage=@save_max_tmp_total_space_usage; +# +# Check updating non transactional table +# +SET max_tmp_session_space_usage= 64*1024; +CREATE TABLE t1 ( +a varchar(1024), b varchar(1024), c varchar(1024), d varchar(1024), e varchar(1024), f varchar(1024), g varchar(1024) +) ENGINE=MyISAM; +INSERT INTO t1 VALUES +(REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024)), +(REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024)), +(REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024)), +(REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024)), +(REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024)), +(REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024)), +('x','x','x','x','x','x','x'); +UPDATE t1 SET a = '' LIMIT 100; +ERROR HY000: Local temporary space limit reached +DROP TABLE t1; +# End of 11.5 tests diff --git a/mysql-test/main/tmp_space_usage.test b/mysql-test/main/tmp_space_usage.test new file mode 100644 index 00000000000..467dbd4c6f2 --- /dev/null +++ b/mysql-test/main/tmp_space_usage.test @@ -0,0 +1,203 @@ +--source include/have_sequence.inc +--source include/have_innodb.inc +--source include/big_test.inc +--source include/not_valgrind.inc +--source include/have_log_bin.inc + +call mtr.add_suppression("Write to binary log failed: .* temporary space limit reached. An incident event is written to binary log.*"); + +# +# Test tmp_space_usage +# + +# flush global status is needed because of max_tmp_space_used +flush status; +flush global status; +show session status like "max_tmp_space_used"; +show global status like "max_tmp_space_used"; + +--echo # +--echo # MDEV-9101 Limit size of total size of created disk temporary files +--echo # and tables. + +# Print variables that can affect the test result +show session status like "tmp_space_used"; +show global status like "tmp_space_used"; +select @@global.max_tmp_session_space_usage, @@global.max_tmp_total_space_usage; +select @@binlog_stmt_cache_size,@@binlog_format; + +create table t1 (a int primary key, v varchar(256), c int default(0)) engine=innodb; +--error 41 +insert into t1 (a,v) select seq, repeat(char(64+mod(seq,32)),mod(seq,254)+1) from seq_1_to_65536; +set @@max_tmp_session_space_usage=1024*1024*1024; +--error 42 +insert into t1 (a,v) select seq, repeat(char(64+mod(seq,32)),mod(seq,254)+1) from seq_1_to_65536; +set @@max_tmp_session_space_usage=default; + +set @@binlog_format="statement"; +insert into t1 (a,v) select seq, repeat(char(64+mod(seq,32)),mod(seq,254)+1) from seq_1_to_65536; +insert into t1 (a,v) select seq, repeat(char(64+mod(seq,32)),mod(seq,254)+1) from seq_65537_to_131072; +select count(*) from t1; + +create table t2 (a int, b int) engine=innodb select seq as a, 0 as b from seq_1_to_131072; + +# Force usage of on disk tmp tables +set @@binlog_format="row"; +set @@tmp_memory_table_size=32*1024; + +--echo # The following queries should fail because of tmp_space_usage +--error 41 +select * from t1 order by a,v; +--error 200 +select v,count(*) from t1 group by v limit 2; +--error 41 +update t1 set v=right(v,2); + +set @@binlog_format="statement"; +set @@max_tmp_session_space_usage=65536; +set @@tmp_memory_table_size=0; +--error 200 +update t1,t2 set t1.c=t2.a, t2.b=1 where t1.a=t2.a; +set @@binlog_format="row"; +set @@max_tmp_session_space_usage=default; + +drop table t1,t2; + +--echo # +--echo # Check max_tmp_total_space_usage & processlist +--echo # + +# We have to set tmp_memory_table_size to ensure we do not use disk for +# the following two show commands. +set @@tmp_memory_table_size=1024*1024; +show session status like "tmp_space_used"; +# The following is disabled until we can do "show status" without using +# temporary files +# show global status like "tmp_space_used"; +set @@tmp_memory_table_size=0; + +let $tmp_usage1=`select variable_value from information_schema.session_status where variable_name="max_tmp_space_used"`; +flush status; +let $tmp_usage2=`select variable_value from information_schema.global_status where variable_name="max_tmp_space_used"`; + +--disable_query_log +if ($tmp_usage1 == $tmp_usage2) +{ +--echo # session.max_tmp_session_space_usage == global.max_tmp_session_space_usage +} +if ($tmp_usage1 != $tmp_usage2) +{ +--echo session.max_tmp_session_space_usage ($tmp_usage1) != global.max_tmp_session_space_usage ($tmp_usage2) +} +--enable_query_log + + +connect(c1, localhost, root,,); +connection default; + +create table t1 (a int primary key, v varchar(256), c int default(0)) engine=innodb; +create table t2 (a int primary key, v varchar(256), c int default(0)) engine=innodb; + +begin; +insert into t1 (a,v) select seq, repeat(char(64+mod(seq,32)),mod(seq,254)+1) from seq_1_to_3000; + +let $id=`select connection_id()`; +let $tmp_usage1=`select variable_value from information_schema.session_status where variable_name="tmp_space_used"`; + +connection c1; +--disable_query_log +let $tmp_usage2=`select tmp_space_used from information_schema.processlist where id=$id`; +if ($tmp_usage1 == $tmp_usage2) +{ +--echo # information_schema.process_list.tmp_space_used == status.tmp_space_used +} +if ($tmp_usage1 != $tmp_usage2) +{ +--echo tmp_space_used difference: $tmp_usage1 != $tmp_usage2 +} +--enable_query_log + +--error 42 +insert into t2 (a,v) select seq, repeat(char(64+mod(seq,32)),mod(seq,254)+1) from seq_1_to_3000; +--echo # Test setting tmp_space_usage to 0 to disable quotas +set @save_max_tmp_total_space_usage=@@global.max_tmp_total_space_usage; +set @@global.max_tmp_total_space_usage=0; +set @@max_tmp_session_space_usage=0; +insert into t2 (a,v) select seq, repeat(char(64+mod(seq,32)),mod(seq,254)+1) from seq_1_to_3000; +set @@global.max_tmp_total_space_usage=@save_max_tmp_total_space_usage; +set @@max_tmp_session_space_usage=0; +connection default; +insert into t1 (a,v) values(9999990,0); +commit; +select count(*) from t1; + +disconnect c1; +drop table t1,t2; + +--echo # +--echo # Test case from Elena +--echo # + +SET @@max_tmp_session_space_usage= 64*1024; +set @@binlog_format="statement"; + +CREATE OR REPLACE TABLE t1 (a INT, b INT); +select benchmark(1,1); +INSERT INTO t1 SELECT seq, seq FROM seq_1_to_100000; +--error 41 +ALTER TABLE t1 ORDER BY a, b; + +# Cleanup +DROP TABLE t1; + +--echo # +--echo # Show that setting max tmp space too low value can stop binary logging +--echo # if non transactional tables are used. +--echo # + +set @save_max_tmp_total_space_usage=@@global.max_tmp_total_space_usage; +SET @@global.max_tmp_total_space_usage=64*1024; +set @@binlog_format="row"; + +create table t1 (a int primary key, v varchar(256)) engine=myisam; +--error 41,42 +insert into t1 (a,v) select seq, repeat(char(64+mod(seq,32)),mod(seq,254)+1) from seq_1_to_65536; +show warnings; +select count(*) <> 0 from t1; + +# Shhow that this problem does not exists with transactional tables +truncate table t1; +alter table t1 engine=innodb; +--error 41,42 +insert into t1 (a,v) select seq, repeat(char(64+mod(seq,32)),mod(seq,254)+1) from seq_1_to_65536; +show warnings; +select count(*) <> 0 from t1; +drop table t1; + +set @@global.max_tmp_total_space_usage=@save_max_tmp_total_space_usage; + +--echo # +--echo # Check updating non transactional table +--echo # + +SET max_tmp_session_space_usage= 64*1024; + +CREATE TABLE t1 ( + a varchar(1024), b varchar(1024), c varchar(1024), d varchar(1024), e varchar(1024), f varchar(1024), g varchar(1024) +) ENGINE=MyISAM; + +INSERT INTO t1 VALUES +(REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024)), +(REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024)), +(REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024)), +(REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024)), +(REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024)), +(REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024),REPEAT('x',1024)), +('x','x','x','x','x','x','x'); + +--error 41 +UPDATE t1 SET a = '' LIMIT 100; + +DROP TABLE t1; + +--echo # End of 11.5 tests diff --git a/mysql-test/suite/funcs_1/datadict/processlist_val.inc b/mysql-test/suite/funcs_1/datadict/processlist_val.inc index ab2482393d5..079411d127b 100644 --- a/mysql-test/suite/funcs_1/datadict/processlist_val.inc +++ b/mysql-test/suite/funcs_1/datadict/processlist_val.inc @@ -92,7 +92,7 @@ echo # - INFO must contain the corresponding SHOW/SELECT PROCESSLIST # # 1. Just dump what we get ---replace_column 1 3 6