database/mariadb
1
0
Fork 0
mirror of https://github.com/MariaDB/server.git synced 2025-08-08 11:22:35 +03:00
Code Activity

MDEV-34705: Binlog-in-engine: Implement PURGE BINARY LOGS

Browse Source 
Still ToDo: is to restrict auto-purge so that it does not purge any binlog
file with out-of-band data that might still be needed by a connected slave.

Signed-off-by: Kristian Nielsen <knielsen@knielsen-hq.org>
This commit is contained in:
Kristian Nielsen
2025-02-04 21:22:12 +00:00
parent d4b37fcc85
commit 0671add213
15 changed files with 1040 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files

77
mysql-test/include/wait_for_engine_binlog.inc Normal file
View File

@@ -0,0 +1,77 @@
# include/wait_for_engine_binlog.inc
#
# SUMMARY
#
# Waits until a specific (engine-implemented) binlog file is seen with
# the specified size in SHOW BINARY LOGS.
# Used to avoid sporadic failures due to races with the binlog
# pre-allocation thread.
#
# USAGE
#
# --let $binlog_name= binlog-000002.ibb
# --let $binlog_size= 262144
# --source include/wait_for_engine_binlog.inc
#
# OPTIONALLY:
#
# let $wait_timeout= 60; # Override default 30 seconds with 60.
# let $wait_notfound= 1; # Wait until specified binlog _not_ found.
#
# EXAMPLE
# binlog_in_engine.binlog_flush_purge.test
#
--disable_query_log
let $_wait_counter= 300;
if ($wait_timeout)
{
let $_wait_counter= `SELECT $wait_timeout * 10`;
}
# Reset $wait_timeout so that its value won't be used on subsequent
# calls, and default will be used instead.
let $wait_timeout= 0;
--let $_expect= 1
--let $_message= exist
if ($wait_notfound) {
--let $_expect= 0
--let $_message= no longer exist
}
let $wait_notfound= 0;
--let $_done= 0
--let $_i= 0
while (!$_done) {
--let $_j= 1
--let $_end= 0
--let $_found= 0
while (!$_end) {
--let $_x= query_get_value(SHOW BINARY LOGS, Log_name, $_j)
if ($_x == No such row) {
--let $_end= 1
}
if ($_x == $binlog_name) {
--let $_y= query_get_value(SHOW BINARY LOGS, File_size, $_j)
if ($_y == $binlog_size) {
--let $_found= 1
--let $_end= 1
}
}
inc $_j;
}
if ($_found == $_expect) {
--let $_done= 1
}
if (!$_done) {
sleep 0.1;
inc $_i;
if ($_i >= $_wait_counter) {
SHOW BINARY LOGS;
--die Timeout waiting for binlog '$binlog_name' to $_message
}
}
}
--enable_query_log

132
mysql-test/suite/binlog_in_engine/binlog_flush_purge.result Normal file
View File

@@ -0,0 +1,132 @@
RESET MASTER;
CREATE TABLE t1 (a INT PRIMARY KEY) ENGINE=InnoDB;
CREATE TABLE t2 (a INT PRIMARY KEY, b VARCHAR(2048)) ENGINE=InnoDB;
INSERT INTO t1 VALUES (1);
BEGIN;
INSERT INTO t1 VALUES (2);
INSERT INTO t1 VALUES (3);
COMMIT;
INSERT INTO t2 VALUES (0, REPEAT("x", 2048));
INSERT INTO t2 SELECT a+1, b FROM t2;
INSERT INTO t2 SELECT a+2, b FROM t2;
INSERT INTO t2 SELECT a+4, b FROM t2;
INSERT INTO t2 SELECT a+8, b FROM t2;
SHOW BINARY LOGS;
Log_name File_size
binlog-000000.ibb 262144
binlog-000001.ibb 262144
FLUSH BINARY LOGS;
SHOW BINARY LOGS;
Log_name File_size
binlog-000000.ibb 49152
binlog-000001.ibb 262144
binlog-000002.ibb 262144
RESET MASTER;
SHOW BINARY LOGS;
Log_name File_size
binlog-000000.ibb 262144
binlog-000001.ibb 262144
INSERT INTO t1 VALUES (100);
INSERT INTO t2 VALUES (100, 'xyzzy');
DROP TABLE t1, t2;
CREATE TABLE t1 (a INT PRIMARY KEY, b VARCHAR(2048)) ENGINE=InnoDB;
SET @old_min_slaves= @@GLOBAL.slave_connections_needed_for_purge;
SET GLOBAL slave_connections_needed_for_purge= 1;
PURGE BINARY LOGS TO 'binlog-000001.ibb';
ERROR HY000: A purgeable log is in use, will not purge
SHOW WARNINGS;
Level Code Message
Note 1375 Binary log 'binlog-000000.ibb' is not purged because less than 'slave_connections_needed_for_purge' slaves have processed it
Error 1378 A purgeable log is in use, will not purge
SET GLOBAL slave_connections_needed_for_purge= 0;
PURGE BINARY LOGS TO 'binlog-000001.ibb';
ERROR HY000: A purgeable log is in use, will not purge
SHOW WARNINGS;
Level Code Message
Note 1375 Binary log 'binlog-000000.ibb' is not purged because the binlog file is in active use
Error 1378 A purgeable log is in use, will not purge
SET @old_max_total= @@GLOBAL.max_binlog_total_size;
SET GLOBAL max_binlog_total_size= 4*@@GLOBAL.max_binlog_size;
SET SESSION binlog_format= ROW;
*** Do 1500 transactions ...
SHOW BINARY LOGS;
Log_name File_size
binlog-000011.ibb 262144
binlog-000012.ibb 262144
binlog-000013.ibb 262144
binlog-000014.ibb 262144
*** Test purge by date.
SET GLOBAL max_binlog_total_size= 0;
SET @old_expire= @@GLOBAL.binlog_expire_logs_seconds;
SET GLOBAL binlog_expire_logs_seconds= 1;
*** Do 187 inserts ...
SET GLOBAL binlog_expire_logs_seconds= 0;
*** Do 1000 transactions ...
SHOW BINARY LOGS;
Log_name File_size
binlog-000013.ibb 262144
binlog-000014.ibb 262144
binlog-000015.ibb 262144
binlog-000016.ibb 262144
binlog-000017.ibb 262144
binlog-000018.ibb 262144
binlog-000019.ibb 262144
binlog-000020.ibb 262144
binlog-000021.ibb 262144
binlog-000022.ibb 262144
binlog-000023.ibb 262144
binlog-000024.ibb 262144
SET @now= NOW();
*** Do 187 inserts ...
PURGE BINARY LOGS BEFORE @now;
SHOW BINARY LOGS;
Log_name File_size
binlog-000023.ibb 262144
binlog-000024.ibb 262144
binlog-000025.ibb 262144
*** Test PURGE BINARY LOGS TO
PURGE BINARY LOGS TO 'binlog-000025.ibb';
ERROR HY000: A purgeable log is in use, will not purge
SHOW WARNINGS;
Level Code Message
Note 1375 Binary log 'binlog-000024.ibb' is not purged because the binlog file is in active use
Error 1378 A purgeable log is in use, will not purge
SHOW BINARY LOGS;
Log_name File_size
binlog-000024.ibb 262144
binlog-000025.ibb 262144
*** Do 436 inserts ...
SHOW BINARY LOGS;
Log_name File_size
binlog-000024.ibb 262144
binlog-000025.ibb 262144
binlog-000026.ibb 262144
binlog-000027.ibb 262144
binlog-000028.ibb 262144
binlog-000029.ibb 262144
PURGE BINARY LOGS TO 'binlog-000025.ibb';
SHOW BINARY LOGS;
Log_name File_size
binlog-000025.ibb 262144
binlog-000026.ibb 262144
binlog-000027.ibb 262144
binlog-000028.ibb 262144
binlog-000029.ibb 262144
PURGE BINARY LOGS TO 'binlog-999999.ibb';
ERROR HY000: Target log not found in binlog index
SHOW WARNINGS;
Level Code Message
Error 1373 Target log not found in binlog index
*** Test purging logs when setting the maximum size.
SET GLOBAL max_binlog_total_size= ceil(1.5*@@GLOBAL.max_binlog_size);
Warnings:
Note 1375 Binary log 'binlog-000028.ibb' is not purged because the binlog file is in active use
SHOW BINARY LOGS;
Log_name File_size
binlog-000028.ibb 262144
binlog-000029.ibb 262144
SET SESSION binlog_format= MIXED;
DROP TABLE t1;
SET GLOBAL max_binlog_total_size= @old_max_total;
SET GLOBAL slave_connections_needed_for_purge= @old_min_slaves;
SET GLOBAL binlog_expire_logs_seconds= @old_expire;

164
mysql-test/suite/binlog_in_engine/binlog_flush_purge.test
View File

@@ -16,15 +16,20 @@ INSERT INTO t2 SELECT a+2, b FROM t2;
INSERT INTO t2 SELECT a+4, b FROM t2; INSERT INTO t2 SELECT a+4, b FROM t2;
INSERT INTO t2 SELECT a+8, b FROM t2; INSERT INTO t2 SELECT a+8, b FROM t2;
--let $binlog_name= binlog-000001.ibb
--let $binlog_size= 262144
--source include/wait_for_engine_binlog.inc
SHOW BINARY LOGS; SHOW BINARY LOGS;
FLUSH BINARY LOGS; FLUSH BINARY LOGS;
--let $binlog_name= binlog-000002.ibb
--let $binlog_size= 262144
--source include/wait_for_engine_binlog.inc
SHOW BINARY LOGS; SHOW BINARY LOGS;
RESET MASTER; RESET MASTER;
# ToDo: This will be racy, the second binlog file binlog-000001.ibb is --let $binlog_name= binlog-000001.ibb
# pre-allocated in the background, so will be visible or not depending on exact --let $binlog_size= 262144
# timing. So just omit this SHOW BINARY LOGS or wait for both to be created --source include/wait_for_engine_binlog.inc
# or something.
SHOW BINARY LOGS; SHOW BINARY LOGS;
INSERT INTO t1 VALUES (100); INSERT INTO t1 VALUES (100);
@@ -32,4 +37,153 @@ INSERT INTO t2 VALUES (100, 'xyzzy');
DROP TABLE t1, t2; DROP TABLE t1, t2;
--exec $MYSQL_BINLOG --read-from-remote-server --user=root --host=127.0.0.1 --port=$MASTER_MYPORT master-bin.000001 --start-position=0-1-1 > $MYSQLTEST_VARDIR/tmp/mysqlbinlog.txt # Test purge by size
CREATE TABLE t1 (a INT PRIMARY KEY, b VARCHAR(2048)) ENGINE=InnoDB;
SET @old_min_slaves= @@GLOBAL.slave_connections_needed_for_purge;
SET GLOBAL slave_connections_needed_for_purge= 1;
--error ER_LOG_IN_USE
PURGE BINARY LOGS TO 'binlog-000001.ibb';
SHOW WARNINGS;
SET GLOBAL slave_connections_needed_for_purge= 0;
--error ER_LOG_IN_USE
PURGE BINARY LOGS TO 'binlog-000001.ibb';
SHOW WARNINGS;
SET @old_max_total= @@GLOBAL.max_binlog_total_size;
SET GLOBAL max_binlog_total_size= 4*@@GLOBAL.max_binlog_size;
SET SESSION binlog_format= ROW;
--let $num_trx= 1500
--echo *** Do $num_trx transactions ...
--disable_query_log
--let $i= 0
while ($i < $num_trx) {
eval INSERT INTO t1 VALUES ($i+100000, REPEAT("x", 2048));
inc $i;
}
--enable_query_log
# The precise point at which we move to the next binlog file depends on the
# exact size of binlogged transactions, which might change as server code is
# developed, and then this test will fail with a different set of binlog files
# appearing in SHOW BINARY LOGS.
#
# In this case, just check that the general structure of the present binlogs
# is similar, and then update the $binlog_name waited for and the .result file.
--let $binlog_name= binlog-000010.ibb
--let $binlog_size= 262144
--let $wait_notfound= 1
--source include/wait_for_engine_binlog.inc
--let $binlog_name= binlog-000014.ibb
--let $binlog_size= 262144
--source include/wait_for_engine_binlog.inc
SHOW BINARY LOGS;
--echo *** Test purge by date.
SET GLOBAL max_binlog_total_size= 0;
SET @old_expire= @@GLOBAL.binlog_expire_logs_seconds;
SET GLOBAL binlog_expire_logs_seconds= 1;
--sleep 2
--let $num_insert= `SELECT floor(256*1.5*1024/2100)`
--echo *** Do $num_insert inserts ...
--disable_query_log
BEGIN;
--let $i= 0
while ($i < $num_insert) {
eval INSERT INTO t1 VALUES ($i+200000, REPEAT("x", 2048));
inc $i;
}
COMMIT;
--enable_query_log
--let $binlog_name= binlog-000012.ibb
--let $binlog_size= 262144
--let $wait_notfound= 1
--source include/wait_for_engine_binlog.inc
# SHOW BINARY LOGS here will not be stable.
# We can wait for the log before the --sleep 2 to expire.
# But the next log might also expire, if there is a random delay sufficiently
# long before the automatic purge runs.
#SHOW BINARY LOGS;
SET GLOBAL binlog_expire_logs_seconds= 0;
--let $num_trx= 1000
--echo *** Do $num_trx transactions ...
--disable_query_log
--let $i= 0
while ($i < $num_trx) {
eval INSERT INTO t1 VALUES ($i+300000, REPEAT("x", 2048));
inc $i;
}
--enable_query_log
--let $binlog_name= binlog-000024.ibb
--let $binlog_size= 262144
--source include/wait_for_engine_binlog.inc
SHOW BINARY LOGS;
--sleep 1
SET @now= NOW();
--sleep 1
--let $num_insert= `SELECT floor(256*1.5*1024/2100)`
--echo *** Do $num_insert inserts ...
--disable_query_log
BEGIN;
--let $i= 0
while ($i < $num_insert) {
eval INSERT INTO t1 VALUES ($i+400000, REPEAT("x", 2048));
inc $i;
}
COMMIT;
--enable_query_log
PURGE BINARY LOGS BEFORE @now;
--let $binlog_name= binlog-000022.ibb
--let $binlog_size= 262144
--let $wait_notfound= 1
--source include/wait_for_engine_binlog.inc
--let $binlog_name= binlog-000025.ibb
--let $binlog_size= 262144
--source include/wait_for_engine_binlog.inc
SHOW BINARY LOGS;
--echo *** Test PURGE BINARY LOGS TO
--let $current= query_get_value(SHOW BINARY LOGS, Log_name, 3)
--error ER_LOG_IN_USE
eval PURGE BINARY LOGS TO '$current';
SHOW WARNINGS;
SHOW BINARY LOGS;
--let $num_insert= `SELECT floor(256*3.5*1024/2100)`
--echo *** Do $num_insert inserts ...
--disable_query_log
BEGIN;
--let $i= 0
while ($i < $num_insert) {
eval INSERT INTO t1 VALUES ($i+500000, REPEAT("x", 2048));
inc $i;
}
COMMIT;
--enable_query_log
--source include/wait_for_engine_binlog.inc
--let $binlog_name= binlog-000029.ibb
--let $binlog_size= 262144
--source include/wait_for_engine_binlog.inc
SHOW BINARY LOGS;
eval PURGE BINARY LOGS TO '$current';
--let $binlog_name= binlog-000024.ibb
--let $binlog_size= 262144
--let $wait_notfound= 1
--source include/wait_for_engine_binlog.inc
SHOW BINARY LOGS;
--error ER_UNKNOWN_TARGET_BINLOG
PURGE BINARY LOGS TO 'binlog-999999.ibb';
SHOW WARNINGS;
--echo *** Test purging logs when setting the maximum size.
SET GLOBAL max_binlog_total_size= ceil(1.5*@@GLOBAL.max_binlog_size);
SHOW BINARY LOGS;
SET SESSION binlog_format= MIXED;
DROP TABLE t1;
SET GLOBAL max_binlog_total_size= @old_max_total;
SET GLOBAL slave_connections_needed_for_purge= @old_min_slaves;
SET GLOBAL binlog_expire_logs_seconds= @old_expire;

64
sql/handler.h
View File

@@ -64,6 +64,7 @@ struct rpl_gtid;
struct slave_connection_state; struct slave_connection_state;
struct rpl_binlog_state_base; struct rpl_binlog_state_base;
struct handler_binlog_event_group_info; struct handler_binlog_event_group_info;
struct handler_binlog_purge_info;
// the following is for checking tables // the following is for checking tables
@@ -1572,6 +1573,13 @@ struct handlerton
bool (*binlog_flush)(); bool (*binlog_flush)();
/* Engine implementation of RESET MASTER. */ /* Engine implementation of RESET MASTER. */
bool (*reset_binlogs)(); bool (*reset_binlogs)();
/*
Engine implementation of PURGE BINARY LOGS.
Return 0 for ok or one of LOG_INFO_* errors.
See also ha_binlog_purge_info() for auto-purge.
*/
int (*binlog_purge)(handler_binlog_purge_info *purge_info);
/* /*
Optional clauses in the CREATE/ALTER TABLE Optional clauses in the CREATE/ALTER TABLE
@@ -5865,6 +5873,16 @@ struct handler_binlog_event_group_info {
Class for reading a binlog implemented in an engine. Class for reading a binlog implemented in an engine.
*/ */
class handler_binlog_reader { class handler_binlog_reader {
public:
/*
Approximate current position (from which next call to read_binlog_data()
will need to read). Updated by the engine. Used to know which binlog files
the active dump threads are currently reading from, to avoid purging
actively used binlogs.
*/
uint64_t cur_file_no;
uint64_t cur_file_pos;
private: private:
/* Position and length of any remaining data in buf[]. */ /* Position and length of any remaining data in buf[]. */
uint32_t buf_data_pos; uint32_t buf_data_pos;
@@ -5874,15 +5892,59 @@ private:
public: public:
handler_binlog_reader() handler_binlog_reader()
: buf_data_pos(0), buf_data_remain(0) : cur_file_no(~(uint64_t)0), cur_file_pos(~(uint64_t)0),
buf_data_pos(0), buf_data_remain(0)
{ } { }
virtual ~handler_binlog_reader() { }; virtual ~handler_binlog_reader() { };
virtual int read_binlog_data(uchar *buf, uint32_t len) = 0; virtual int read_binlog_data(uchar *buf, uint32_t len) = 0;
virtual bool data_available()= 0; virtual bool data_available()= 0;
/*
This initializes the current read position to the point of the slave GTID
position passed in as POS. It is permissible to start at a position a bit
earlier in the binlog, only cost is the extra read cost of reading not
needed event data.
If position is found, must return the corresponding binlog state in the
STATE output parameter and initialize cur_file_no and cur_file_pos members.
Returns:
-1 Error
0 The requested GTID position not found, needed binlogs have been purged
1 Ok, position found and returned.
*/
virtual int init_gtid_pos(slave_connection_state *pos, virtual int init_gtid_pos(slave_connection_state *pos,
rpl_binlog_state_base *state) = 0; rpl_binlog_state_base *state) = 0;
int read_log_event(String *packet, uint32_t ev_offset, size_t max_allowed); int read_log_event(String *packet, uint32_t ev_offset, size_t max_allowed);
}; };
/* Structure returned by ha_binlog_purge_info(). */
struct handler_binlog_purge_info {
/* The earliest binlog file that is in use by a dump thread. */
uint64_t limit_file_no;
/*
Set by engine to give a reason why a requested purge could not be done.
If set, then nonpurge_filename should be set to the filename.
Also set by ha_binlog_purge_info() when it returns false, to the reason
why no purge is possible. In this case, the nonpurge_filename is set
to the empty string.
*/
const char *nonpurge_reason;
/* The user-configured maximum total size of the binlog. */
ulonglong limit_size;
/* Binlog name, for PURGE BINARY LOGS TO. */
const char *limit_name;
/* The earliest file date (unix timestamp) that should not be purged. */
time_t limit_date;
/* Whether purge by date and/or by size and/or name is requested. */
bool purge_by_date, purge_by_size, purge_by_name;
/*
The name of the file that could not be purged, when nonpurge_reason
is given.
*/
char nonpurge_filename[FN_REFLEN];
};
#endif /* HANDLER_INCLUDED */ #endif /* HANDLER_INCLUDED */

78
sql/log.cc
View File

@@ -5326,8 +5326,8 @@ int MYSQL_BIN_LOG::purge_index_entry(THD *thd, ulonglong *reclaimed_space,
} }
else else
{ {
if (unlikely((error= find_log_pos(&check_log_info, if (likely((error= find_log_pos(&check_log_info,
log_info.log_file_name, need_mutex)))) log_info.log_file_name, need_mutex))))
{ {
if (error != LOG_INFO_EOF) if (error != LOG_INFO_EOF)
{ {
@@ -5600,6 +5600,40 @@ err:
DBUG_RETURN(error); DBUG_RETURN(error);
} }
void
MYSQL_BIN_LOG::engine_purge_logs_by_size(ulonglong max_total_size)
{
DBUG_ASSERT(opt_binlog_engine_hton);
if (!is_open())
return;
handler_binlog_purge_info purge_info;
auto p= engine_binlog_in_use();
purge_info.limit_file_no= p.first;
uint num_dump_threads= p.second;
if (num_dump_threads < slave_connections_needed_for_purge)
{
purge_info.limit_file_no= 0;
purge_info.nonpurge_reason= "less than "
"'slave_connections_needed_for_purge' slaves have processed it";
}
else
purge_info.nonpurge_reason= nullptr;
purge_info.nonpurge_filename[0]= '\0';
purge_info.purge_by_date= false;
purge_info.limit_date= my_time(0);
purge_info.purge_by_size= true;
purge_info.limit_size= max_total_size;
purge_info.purge_by_name= false;
purge_info.limit_name= nullptr;
int res= (*opt_binlog_engine_hton->binlog_purge)(&purge_info);
if (res && purge_info.nonpurge_reason)
give_purge_note(purge_info.nonpurge_reason,
purge_info.nonpurge_filename, true);
}
/* /*
@param log_file_name_arg Name of log file to check @param log_file_name_arg Name of log file to check
@param interactive True if called by a PURGE BINLOG command. @param interactive True if called by a PURGE BINLOG command.
@@ -5630,6 +5664,7 @@ MYSQL_BIN_LOG::can_purge_log(const char *log_file_name_arg,
int res; int res;
const char *reason; const char *reason;
DBUG_ASSERT(is_relay_log || !opt_binlog_engine_hton);
if (is_active(log_file_name_arg) || if (is_active(log_file_name_arg) ||
(!is_relay_log && waiting_for_slave_to_change_binlog && (!is_relay_log && waiting_for_slave_to_change_binlog &&
purge_sending_new_binlog_file == sending_new_binlog_file && purge_sending_new_binlog_file == sending_new_binlog_file &&
@@ -5693,23 +5728,39 @@ error:
/* purge_warning_given is reset after next sucessful purge */ /* purge_warning_given is reset after next sucessful purge */
purge_warning_given= 1; purge_warning_given= 1;
if (interactive) give_purge_note(reason, log_file_name_arg, interactive);
{
my_printf_error(ER_BINLOG_PURGE_PROHIBITED,
"Binary log '%s' is not purged because %s",
MYF(ME_NOTE), log_file_name_arg, reason);
}
else
{
sql_print_information("Binary log '%s' is not purged because %s",
log_file_name_arg, reason);
}
} }
return 0; return 0;
} }
#endif /* HAVE_REPLICATION */ #endif /* HAVE_REPLICATION */
void
give_purge_note(const char *reason, const char *file_name, bool interactive)
{
if (interactive)
{
if (file_name && file_name[0])
my_printf_error(ER_BINLOG_PURGE_PROHIBITED,
"Binary log '%s' is not purged because %s",
MYF(ME_NOTE), file_name, reason);
else
my_printf_error(ER_BINLOG_PURGE_PROHIBITED,
"Binary log purge is prevented because %s",
MYF(ME_NOTE), reason);
}
else
{
if (file_name && file_name[0])
sql_print_information("Binary log '%s' is not purged because %s",
file_name, reason);
else
sql_print_information("Binary log purge is prevented because %s",
reason);
}
}
/** /**
Count a total size of binary logs (except the active one) to the variable Count a total size of binary logs (except the active one) to the variable
binlog_space_total. binlog_space_total.
@@ -5727,6 +5778,7 @@ int MYSQL_BIN_LOG::count_binlog_space()
LOG_INFO log_info; LOG_INFO log_info;
DBUG_ENTER("count_binlog_space"); DBUG_ENTER("count_binlog_space");
DBUG_ASSERT(!opt_binlog_engine_hton);
binlog_space_total = 0; binlog_space_total = 0;
if ((error= find_log_pos(&log_info, NullS, false /*need_lock_index=false*/))) if ((error= find_log_pos(&log_info, NullS, false /*need_lock_index=false*/)))
goto done; goto done;

11
sql/log.h
View File

@@ -28,6 +28,8 @@ class Gtid_log_event;
bool reopen_fstreams(const char *filename, FILE *outstream, FILE *errstream); bool reopen_fstreams(const char *filename, FILE *outstream, FILE *errstream);
void setup_log_handling(); void setup_log_handling();
void give_purge_note(const char *reason, const char *file_name,
bool interactive);
bool trans_has_updated_trans_table(const THD* thd); bool trans_has_updated_trans_table(const THD* thd);
bool stmt_has_updated_trans_table(const THD *thd); bool stmt_has_updated_trans_table(const THD *thd);
bool use_trans_cache(const THD* thd, bool is_transactional); bool use_trans_cache(const THD* thd, bool is_transactional);
@@ -266,12 +268,14 @@ class Relay_log_info;
*/ */
typedef struct st_log_info typedef struct st_log_info
{ {
/* file_no only used when --binlog-storage-engine set. */
std::atomic<uint64_t> file_no;
/* log_file_name and *_offset only used when --binlog-storage-engine unset. */
char log_file_name[FN_REFLEN]; char log_file_name[FN_REFLEN];
my_off_t index_file_offset, index_file_start_offset; my_off_t index_file_offset, index_file_start_offset;
my_off_t pos; my_off_t pos;
bool fatal; // if the purge happens to give us a negative offset st_log_info() : file_no(~(uint64_t)0), index_file_offset(0),
st_log_info() : index_file_offset(0), index_file_start_offset(0), index_file_start_offset(0), pos(0)
pos(0), fatal(0)
{ {
DBUG_ENTER("LOG_INFO"); DBUG_ENTER("LOG_INFO");
log_file_name[0] = '\0'; log_file_name[0] = '\0';
@@ -1084,6 +1088,7 @@ public:
return 0; return 0;
return real_purge_logs_by_size(binlog_pos); return real_purge_logs_by_size(binlog_pos);
} }
void engine_purge_logs_by_size(ulonglong max_total_size);
int set_purge_index_file_name(const char *base_file_name); int set_purge_index_file_name(const char *base_file_name);
int open_purge_index_file(bool destroy); int open_purge_index_file(bool destroy);
bool truncate_and_remove_binlogs(const char *truncate_file, bool truncate_and_remove_binlogs(const char *truncate_file,

188
sql/sql_repl.cc
View File

@@ -612,9 +612,7 @@ static my_bool adjust_callback(THD *thd, my_off_t *purge_offset)
we just started reading the index file. In that case we just started reading the index file. In that case
we have nothing to adjust we have nothing to adjust
*/ */
if (linfo->index_file_offset < *purge_offset) if (linfo->index_file_offset >= *purge_offset)
linfo->fatal= (linfo->index_file_offset != 0);
else
linfo->index_file_offset-= *purge_offset; linfo->index_file_offset-= *purge_offset;
} }
mysql_mutex_unlock(&thd->LOCK_thd_data); mysql_mutex_unlock(&thd->LOCK_thd_data);
@@ -654,7 +652,7 @@ static my_bool log_in_use_callback(THD *thd, st_log_in_use *arg)
/* /*
Check if a log is in use. Check if a log is in use (legacy binlog).
@return 0 Not used @return 0 Not used
@return 1 A slave is reading from the log @return 1 A slave is reading from the log
@@ -676,6 +674,94 @@ int log_in_use(const char* log_name, uint min_connected)
} }
struct st_engine_binlog_in_use {
uint64_t min_file_no;
uint count;
};
my_bool
engine_binlog_in_use_callback(THD *thd, st_engine_binlog_in_use *arg)
{
if (thd->current_linfo)
{
mysql_mutex_lock(&thd->LOCK_thd_data);
if (LOG_INFO *linfo= thd->current_linfo)
{
uint64_t file_no= linfo->file_no.load(std::memory_order_relaxed);
if (file_no < arg->min_file_no)
arg->min_file_no= file_no;
if (file_no != ~(uint64_t)0)
++arg->count;
}
mysql_mutex_unlock(&thd->LOCK_thd_data);
}
return FALSE;
}
/*
Find earliest binlog file in use (--binlog-storage-engine).
Returns a pair of the earliest file_no binlog in use by a dump thread,
and the number of actively running dump threads.
*/
std::pair<uint64_t, uint>
engine_binlog_in_use()
{
DBUG_ASSERT(opt_binlog_engine_hton);
st_engine_binlog_in_use arg{~(uint64_t)0, 0};
server_threads.iterate(engine_binlog_in_use_callback, &arg);
return {arg.min_file_no, arg.count};
}
/*
Inform engine about server state relevant for automatic binlog purge.
Used by engines that implement --binlog-storage-engine.
Returns true if automatic purge should proceed with supplied information,
false if automatic purge is disabled due to
--slave-connections-needed-for-purge.
*/
bool
ha_binlog_purge_info(handler_binlog_purge_info *out_info)
{
auto p= engine_binlog_in_use();
out_info->limit_file_no= p.first;
uint num_dump_threads= p.second;
out_info->purge_by_name= false;
out_info->limit_name= nullptr;
if (binlog_expire_logs_seconds)
{
out_info->purge_by_date= true;
out_info->limit_date= my_time(0) - binlog_expire_logs_seconds;
}
else
out_info->purge_by_date= false;
if (binlog_space_limit)
{
out_info->purge_by_size= true;
out_info->limit_size= binlog_space_limit;
}
else
out_info->purge_by_size= false;
out_info->nonpurge_filename[0]= '\0';
if (num_dump_threads >= slave_connections_needed_for_purge)
{
out_info->nonpurge_reason= nullptr;
return true;
}
else
{
out_info->nonpurge_reason= "less than 'slave_connections_needed_for_purge' "
"slaves have processed it";
return false;
}
}
bool purge_error_message(THD* thd, int res) bool purge_error_message(THD* thd, int res)
{ {
uint errcode; uint errcode;
@@ -703,17 +789,53 @@ bool purge_error_message(THD* thd, int res)
*/ */
bool purge_master_logs(THD* thd, const char* to_log) bool purge_master_logs(THD* thd, const char* to_log)
{ {
char search_file_name[FN_REFLEN];
if (!mysql_bin_log.is_open()) if (!mysql_bin_log.is_open())
{ {
my_ok(thd); my_ok(thd);
return FALSE; return FALSE;
} }
mysql_bin_log.make_log_name(search_file_name, to_log); int res;
return purge_error_message(thd, if (!opt_binlog_engine_hton)
mysql_bin_log.purge_logs(thd, search_file_name, {
0, 1, 1, 1, NULL)); char search_file_name[FN_REFLEN];
mysql_bin_log.make_log_name(search_file_name, to_log);
res= mysql_bin_log.purge_logs(thd, search_file_name, 0, 1, 1, 1, NULL);
}
else
{
handler_binlog_purge_info purge_info;
auto p= engine_binlog_in_use();
purge_info.limit_file_no= p.first;
uint num_dump_threads= p.second;
if (num_dump_threads < slave_connections_needed_for_purge)
{
/*
Prevent purging any file.
We need to do it this way, since we have to call into the engine to let
it check if there are any files to potentially purge. If there are, we
want to give an error that purge was not possible. But if there were no
files to purge in any case, we do not want to give any error.
*/
purge_info.limit_file_no= 0;
purge_info.nonpurge_reason= "less than "
"'slave_connections_needed_for_purge' slaves have processed it";
}
else
purge_info.nonpurge_reason= nullptr;
purge_info.nonpurge_filename[0]= '\0';
purge_info.purge_by_date= false;
purge_info.limit_date= (time_t)0;
purge_info.purge_by_size= false;
purge_info.limit_size= 0;
purge_info.purge_by_name= true;
purge_info.limit_name= to_log;
res= (*opt_binlog_engine_hton->binlog_purge)(&purge_info);
if (res && purge_info.nonpurge_reason)
give_purge_note(purge_info.nonpurge_reason,
purge_info.nonpurge_filename, true);
}
return purge_error_message(thd, res);
} }
@@ -735,10 +857,36 @@ bool purge_master_logs_before_date(THD* thd, time_t purge_time)
my_ok(thd); my_ok(thd);
return 0; return 0;
} }
return purge_error_message(thd, int res;
mysql_bin_log.purge_logs_before_date(thd, if (!opt_binlog_engine_hton)
purge_time, res= mysql_bin_log.purge_logs_before_date(thd, purge_time, 1);
1)); else
{
handler_binlog_purge_info purge_info;
auto p= engine_binlog_in_use();
purge_info.limit_file_no= p.first;
uint num_dump_threads= p.second;
if (num_dump_threads < slave_connections_needed_for_purge)
{
purge_info.limit_file_no= 0;
purge_info.nonpurge_reason= "less than "
"'slave_connections_needed_for_purge' slaves have processed it";
}
else
purge_info.nonpurge_reason= nullptr;
purge_info.nonpurge_filename[0]= '\0';
purge_info.purge_by_date= true;
purge_info.limit_date= purge_time;
purge_info.purge_by_size= false;
purge_info.limit_size= 0;
purge_info.purge_by_name= false;
purge_info.limit_name= nullptr;
res= (*opt_binlog_engine_hton->binlog_purge)(&purge_info);
if (res && purge_info.nonpurge_reason)
give_purge_note(purge_info.nonpurge_reason,
purge_info.nonpurge_filename, true);
}
return purge_error_message(thd, res);
} }
void set_read_error(binlog_send_info *info, int error) void set_read_error(binlog_send_info *info, int error)
@@ -3169,7 +3317,7 @@ static int send_events(binlog_send_info *info, IO_CACHE* log, LOG_INFO* linfo,
* return 0 - OK * return 0 - OK
* else NOK * else NOK
*/ */
static int send_engine_events(binlog_send_info *info) static int send_engine_events(binlog_send_info *info, LOG_INFO* linfo)
{ {
int error; int error;
ulong ev_offset; ulong ev_offset;
@@ -3191,6 +3339,10 @@ static int send_engine_events(binlog_send_info *info)
set_read_error(info, error); set_read_error(info, error);
return 1; return 1;
} }
linfo->file_no.store(reader->cur_file_no, std::memory_order_relaxed);
linfo->pos= (my_off_t) reader->cur_file_pos;
if (error == LOG_READ_EOF) if (error == LOG_READ_EOF)
{ {
PSI_stage_info old_stage; PSI_stage_info old_stage;
@@ -3280,7 +3432,7 @@ static int send_one_binlog_file(binlog_send_info *info,
if (opt_binlog_engine_hton) if (opt_binlog_engine_hton)
{ {
info->dirlen= 0; info->dirlen= 0;
if (send_engine_events(info)) if (send_engine_events(info, linfo))
return 1; return 1;
} }
else else
@@ -5105,7 +5257,8 @@ retry:
cur_link->name.str+= dir_len; cur_link->name.str+= dir_len;
cur_link->name.length-= dir_len; cur_link->name.length-= dir_len;
if (mysql_bin_log.get_reset_master_count() > expected_reset_masters) if (!opt_binlog_engine_hton &&
mysql_bin_log.get_reset_master_count() > expected_reset_masters)
{ {
/* /*
Reset master was called after we cached filenames. Reset master was called after we cached filenames.
@@ -5115,7 +5268,8 @@ retry:
goto retry; goto retry;
} }
if (!(strncmp(fname+dir_len, cur.log_file_name+cur_dir_len, length))) if (!opt_binlog_engine_hton &&
!(strncmp(fname+dir_len, cur.log_file_name+cur_dir_len, length)))
cur_link->size= cur.pos; /* The active log, use the active position */ cur_link->size= cur.pos; /* The active log, use the active position */
else else
{ {

1
sql/sql_repl.h
View File

@@ -39,6 +39,7 @@ int reset_master(THD* thd, rpl_gtid *init_state, uint32 init_state_len,
bool purge_master_logs(THD* thd, const char* to_log); bool purge_master_logs(THD* thd, const char* to_log);
bool purge_master_logs_before_date(THD* thd, time_t purge_time); bool purge_master_logs_before_date(THD* thd, time_t purge_time);
int log_in_use(const char* log_name, uint min_connections); int log_in_use(const char* log_name, uint min_connections);
std::pair<uint64_t, uint> engine_binlog_in_use();
void adjust_linfo_offsets(my_off_t purge_offset); void adjust_linfo_offsets(my_off_t purge_offset);
void show_binlogs_get_fields(THD *thd, List<Item> *field_list); void show_binlogs_get_fields(THD *thd, List<Item> *field_list);
bool show_binlogs(THD* thd); bool show_binlogs(THD* thd);

24
sql/sys_vars.cc
View File

@@ -1266,14 +1266,22 @@ static bool update_binlog_space_limit(sys_var *, THD *,
if (opt_bin_log) if (opt_bin_log)
{ {
if (binlog_space_limit) if (opt_binlog_engine_hton)
mysql_bin_log.count_binlog_space(); {
/* Inform can_purge_log() that it should do a recheck of log_in_use() */ if (loc_binlog_space_limit)
sending_new_binlog_file++; mysql_bin_log.engine_purge_logs_by_size(loc_binlog_space_limit);
mysql_bin_log.unlock_index(); }
mysql_bin_log.purge(1); else
mysql_mutex_lock(&LOCK_global_system_variables); {
return 0; if (loc_binlog_space_limit)
mysql_bin_log.count_binlog_space();
/* Inform can_purge_log() that it should do a recheck of log_in_use() */
sending_new_binlog_file++;
mysql_bin_log.unlock_index();
mysql_bin_log.purge(1);
mysql_mutex_lock(&LOCK_global_system_variables);
return 0;
}
} }
mysql_bin_log.unlock_index(); mysql_bin_log.unlock_index();
mysql_mutex_lock(&LOCK_global_system_variables); mysql_mutex_lock(&LOCK_global_system_variables);

20
storage/innobase/fsp/fsp_binlog.cc
View File

@@ -265,13 +265,13 @@ fsp_binlog_open(const char *file_name, pfs_os_file_t fh,
@param[in] file_no Index of the binlog tablespace @param[in] file_no Index of the binlog tablespace
@param[out] new_space The newly created tablespace @param[out] new_space The newly created tablespace
@return DB_SUCCESS or error code */ @return DB_SUCCESS or error code */
dberr_t fsp_binlog_tablespace_create(uint64_t file_no, fil_space_t **new_space) dberr_t fsp_binlog_tablespace_create(uint64_t file_no, uint32_t size_in_pages,
fil_space_t **new_space)
{ {
pfs_os_file_t fh; pfs_os_file_t fh;
bool ret; bool ret;
*new_space= nullptr; *new_space= nullptr;
uint32_t size= innodb_binlog_size_in_pages;
if(srv_read_only_mode) if(srv_read_only_mode)
return DB_ERROR; return DB_ERROR;
@@ -296,7 +296,8 @@ dberr_t fsp_binlog_tablespace_create(uint64_t file_no, fil_space_t **new_space)
/* We created the binlog file and now write it full of zeros */ /* We created the binlog file and now write it full of zeros */
if (!os_file_set_size(name, fh, if (!os_file_set_size(name, fh,
os_offset_t{size} << srv_page_size_shift)) { os_offset_t{size_in_pages} << srv_page_size_shift)
) {
sql_print_error("Unable to allocate file %s", name); sql_print_error("Unable to allocate file %s", name);
os_file_close(fh); os_file_close(fh);
os_file_delete(innodb_data_file_key, name); os_file_delete(innodb_data_file_key, name);
@@ -317,7 +318,8 @@ dberr_t fsp_binlog_tablespace_create(uint64_t file_no, fil_space_t **new_space)
return DB_ERROR; return DB_ERROR;
} }
fil_node_t* node = (*new_space)->add(name, fh, size, false, true); fil_node_t* node = (*new_space)->add(name, fh, size_in_pages,
false, true);
node->find_metadata(); node->find_metadata();
mysql_mutex_unlock(&fil_system.mutex); mysql_mutex_unlock(&fil_system.mutex);
@@ -575,6 +577,8 @@ fsp_binlog_flush()
chunk_data_flush dummy_data; chunk_data_flush dummy_data;
mtr_t mtr; mtr_t mtr;
mysql_mutex_lock(&purge_binlog_mutex);
mtr.start(); mtr.start();
mtr.x_lock_space(space); mtr.x_lock_space(space);
/* /*
@@ -588,6 +592,12 @@ fsp_binlog_flush()
{ {
mtr.trim_pages(page_id_t(space_id, page_no + 1)); mtr.trim_pages(page_id_t(space_id, page_no + 1));
mtr.commit_shrink(*space, page_no + 1); mtr.commit_shrink(*space, page_no + 1);
size_t reclaimed= (space->size - (page_no + 1)) << srv_page_size_shift;
if (UNIV_LIKELY(total_binlog_used_size >= reclaimed))
total_binlog_used_size-= reclaimed;
else
ut_ad(0);
} }
else else
mtr.commit(); mtr.commit();
@@ -596,6 +606,8 @@ fsp_binlog_flush()
while (buf_flush_list_space(space)) while (buf_flush_list_space(space))
; ;
mysql_mutex_unlock(&purge_binlog_mutex);
/* /*
Now get a new GTID state record written to the next binlog tablespace. Now get a new GTID state record written to the next binlog tablespace.
This ensures that the new state (in case of DELETE_DOMAIN_ID) will be This ensures that the new state (in case of DELETE_DOMAIN_ID) will be

2
storage/innobase/handler/ha_innodb.cc
View File

@@ -549,6 +549,7 @@ mysql_pfs_key_t trx_sys_mutex_key;
mysql_pfs_key_t srv_threads_mutex_key; mysql_pfs_key_t srv_threads_mutex_key;
mysql_pfs_key_t tpool_cache_mutex_key; mysql_pfs_key_t tpool_cache_mutex_key;
mysql_pfs_key_t fsp_active_binlog_mutex_key; mysql_pfs_key_t fsp_active_binlog_mutex_key;
mysql_pfs_key_t fsp_purge_binlog_mutex_key;
/* all_innodb_mutexes array contains mutexes that are /* all_innodb_mutexes array contains mutexes that are
performance schema instrumented if "UNIV_PFS_MUTEX" performance schema instrumented if "UNIV_PFS_MUTEX"
@@ -4135,6 +4136,7 @@ static int innodb_init(void* p)
innobase_hton->get_binlog_file_list= innodb_get_binlog_file_list; innobase_hton->get_binlog_file_list= innodb_get_binlog_file_list;
innobase_hton->binlog_flush= innodb_binlog_flush; innobase_hton->binlog_flush= innodb_binlog_flush;
innobase_hton->reset_binlogs= innodb_reset_binlogs; innobase_hton->reset_binlogs= innodb_reset_binlogs;
innobase_hton->binlog_purge= innodb_binlog_purge;
innodb_remember_check_sysvar_funcs(); innodb_remember_check_sysvar_funcs();

350
storage/innobase/handler/innodb_binlog.cc
View File

@@ -57,6 +57,32 @@ rpl_binlog_state_base binlog_diff_state;
static std::thread binlog_prealloc_thr_obj; static std::thread binlog_prealloc_thr_obj;
static bool prealloc_thread_end= false; static bool prealloc_thread_end= false;
/*
Mutex around purge operations, including earliest_binlog_file_no and
total_binlog_used_size.
*/
mysql_mutex_t purge_binlog_mutex;
/* The earliest binlog tablespace file. Used in binlog purge. */
static uint64_t earliest_binlog_file_no;
/*
The total space in use by binlog tablespace files. Maintained in-memory to
not have to stat(2) every file for every new binlog tablespace allocated in
case of --max-binlog-total-size.
Initialized at server startup (and in RESET MASTER), and updated as binlog
files are pre-allocated and purged.
*/
size_t total_binlog_used_size;
static bool purge_warning_given= false;
#ifdef UNIV_PFS_THREAD
mysql_pfs_key_t binlog_prealloc_thread_key;
#endif
/* Structure holding context for out-of-band chunks of binlogged event group. */ /* Structure holding context for out-of-band chunks of binlogged event group. */
struct binlog_oob_context { struct binlog_oob_context {
@@ -375,8 +401,8 @@ private:
struct found_binlogs { struct found_binlogs {
uint64_t last_file_no, prev_file_no; uint64_t last_file_no, prev_file_no, earliest_file_no;
size_t last_size, prev_size; size_t last_size, prev_size, total_size;
int found_binlogs; int found_binlogs;
}; };
@@ -384,6 +410,7 @@ struct found_binlogs {
static void innodb_binlog_prealloc_thread(); static void innodb_binlog_prealloc_thread();
static int innodb_binlog_discover(); static int innodb_binlog_discover();
static bool binlog_state_recover(); static bool binlog_state_recover();
static void innodb_binlog_autopurge(uint64_t first_open_file_no);
/* /*
@@ -420,6 +447,7 @@ void
innodb_binlog_startup_init() innodb_binlog_startup_init()
{ {
fsp_binlog_init(); fsp_binlog_init();
mysql_mutex_init(fsp_purge_binlog_mutex_key, &purge_binlog_mutex, nullptr);
binlog_diff_state.init(); binlog_diff_state.init();
innodb_binlog_inited= 1; innodb_binlog_inited= 1;
} }
@@ -432,6 +460,8 @@ innodb_binlog_init_state()
binlog_cur_end_offset[0].store(~(uint64_t)0, std::memory_order_relaxed); binlog_cur_end_offset[0].store(~(uint64_t)0, std::memory_order_relaxed);
binlog_cur_end_offset[1].store(~(uint64_t)0, std::memory_order_relaxed); binlog_cur_end_offset[1].store(~(uint64_t)0, std::memory_order_relaxed);
last_created_binlog_file_no= ~(uint64_t)0; last_created_binlog_file_no= ~(uint64_t)0;
earliest_binlog_file_no= ~(uint64_t)0;
total_binlog_used_size= 0;
active_binlog_file_no.store(~(uint64_t)0, std::memory_order_release); active_binlog_file_no.store(~(uint64_t)0, std::memory_order_release);
active_binlog_space= nullptr; active_binlog_space= nullptr;
binlog_cur_page_no= 0; binlog_cur_page_no= 0;
@@ -531,6 +561,18 @@ process_binlog_name(found_binlogs *bls, uint64_t idx, size_t size)
bls->prev_file_no= idx; bls->prev_file_no= idx;
bls->prev_size= size; bls->prev_size= size;
} }
if (bls->found_binlogs == 0)
{
bls->earliest_file_no= idx;
bls->total_size= size;
}
else
{
if (idx < bls->earliest_file_no)
bls->earliest_file_no= idx;
bls->total_size+= size;
}
} }
@@ -702,6 +744,9 @@ innodb_binlog_discover()
if (!page_buf) if (!page_buf)
return -1; return -1;
if (binlog_files.found_binlogs >= 1) { if (binlog_files.found_binlogs >= 1) {
earliest_binlog_file_no= binlog_files.earliest_file_no;
total_binlog_used_size= binlog_files.total_size;
int res= find_pos_in_binlog(binlog_files.last_file_no, int res= find_pos_in_binlog(binlog_files.last_file_no,
binlog_files.last_size, binlog_files.last_size,
page_buf.get(), page_buf.get(),
@@ -771,6 +816,8 @@ innodb_binlog_discover()
/* No binlog files found, start from scratch. */ /* No binlog files found, start from scratch. */
file_no= 0; file_no= 0;
earliest_binlog_file_no= 0;
total_binlog_used_size= 0;
ib::info() << "Starting a new binlog from file number " << file_no << "."; ib::info() << "Starting a new binlog from file number " << file_no << ".";
return 0; return 0;
} }
@@ -802,6 +849,7 @@ void innodb_binlog_close(bool shutdown)
if (shutdown && innodb_binlog_inited >= 1) if (shutdown && innodb_binlog_inited >= 1)
{ {
binlog_diff_state.free(); binlog_diff_state.free();
mysql_mutex_destroy(&purge_binlog_mutex);
fsp_binlog_shutdown(); fsp_binlog_shutdown();
} }
} }
@@ -813,6 +861,10 @@ void innodb_binlog_close(bool shutdown)
static void static void
innodb_binlog_prealloc_thread() innodb_binlog_prealloc_thread()
{ {
my_thread_init();
#ifdef UNIV_PFS_THREAD
pfs_register_thread(binlog_prealloc_thread_key);
#endif
mysql_mutex_lock(&active_binlog_mutex); mysql_mutex_lock(&active_binlog_mutex);
while (1) while (1)
@@ -832,7 +884,18 @@ innodb_binlog_prealloc_thread()
*/ */
++last_created; ++last_created;
mysql_mutex_unlock(&active_binlog_mutex); mysql_mutex_unlock(&active_binlog_mutex);
dberr_t res2= fsp_binlog_tablespace_create(last_created, &new_space);
mysql_mutex_lock(&purge_binlog_mutex);
uint32_t size_in_pages= innodb_binlog_size_in_pages;
dberr_t res2= fsp_binlog_tablespace_create(last_created, size_in_pages,
&new_space);
if (earliest_binlog_file_no == ~(uint64_t)0)
earliest_binlog_file_no= last_created;
total_binlog_used_size+= (size_in_pages << srv_page_size_shift);
innodb_binlog_autopurge(first_open);
mysql_mutex_unlock(&purge_binlog_mutex);
mysql_mutex_lock(&active_binlog_mutex); mysql_mutex_lock(&active_binlog_mutex);
ut_a(res2 == DB_SUCCESS /* ToDo: Error handling. */); ut_a(res2 == DB_SUCCESS /* ToDo: Error handling. */);
ut_a(new_space); ut_a(new_space);
@@ -874,6 +937,12 @@ innodb_binlog_prealloc_thread()
} }
mysql_mutex_unlock(&active_binlog_mutex); mysql_mutex_unlock(&active_binlog_mutex);
my_thread_end();
#ifdef UNIV_PFS_THREAD
pfs_delete_thread();
#endif
} }
@@ -1650,6 +1719,8 @@ int ha_innodb_binlog_reader::read_binlog_data(uchar *buf, uint32_t len)
{ {
int res= read_data(buf, len); int res= read_data(buf, len);
chunk_rd.release(res == 0); chunk_rd.release(res == 0);
cur_file_no= chunk_rd.current_file_no();
cur_file_pos= chunk_rd.current_pos();
return res; return res;
} }
@@ -2074,6 +2145,8 @@ ha_innodb_binlog_reader::init_gtid_pos(slave_connection_state *pos,
{ {
chunk_rd.seek(file_no, offset); chunk_rd.seek(file_no, offset);
chunk_rd.skip_partial(true); chunk_rd.skip_partial(true);
cur_file_no= chunk_rd.current_file_no();
cur_file_pos= chunk_rd.current_pos();
} }
return res; return res;
} }
@@ -2121,43 +2194,17 @@ innobase_binlog_write_direct(IO_CACHE *cache,
bool bool
innodb_find_binlogs(uint64_t *out_first, uint64_t *out_last) innodb_find_binlogs(uint64_t *out_first, uint64_t *out_last)
{ {
MY_DIR *dir= my_dir(innodb_binlog_directory, MYF(0)); mysql_mutex_lock(&active_binlog_mutex);
if (!dir) *out_last= last_created_binlog_file_no;
mysql_mutex_unlock(&active_binlog_mutex);
mysql_mutex_lock(&purge_binlog_mutex);
*out_first= earliest_binlog_file_no;
mysql_mutex_unlock(&purge_binlog_mutex);
if (*out_first == ~(uint64_t)0 || *out_last == ~(uint64_t)0)
{ {
sql_print_error("Could not read the binlog directory '%s', error code %d", ut_ad(0 /* Impossible, we wait at startup for binlog to be created. */);
innodb_binlog_directory, my_errno);
return true; return true;
} }
size_t num_entries= dir->number_of_files;
fileinfo *entries= dir->dir_entry;
uint64_t first_file_no, last_file_no;
uint64_t num_file_no= 0;
for (size_t i= 0; i < num_entries; ++i) {
const char *name= entries[i].name;
uint64_t file_no;
if (!is_binlog_name(name, &file_no))
continue;
if (num_file_no == 0 || file_no < first_file_no)
first_file_no= file_no;
if (num_file_no == 0 || file_no > last_file_no)
last_file_no= file_no;
++num_file_no;
}
my_dirend(dir);
if (num_file_no == 0)
{
sql_print_error("No binlog files found (deleted externally?)");
return true;
}
if (num_file_no != last_file_no - first_file_no + 1)
{
sql_print_error("Missing binlog files (deleted externally?)");
return true;
}
*out_first= first_file_no;
*out_last= last_file_no;
return false; return false;
} }
@@ -2207,3 +2254,232 @@ innodb_reset_binlogs()
return err; return err;
} }
/*
The low-level function handling binlog purge.
How much to purge is determined by:
1. Lowest file_no that should not be purged. This is determined as the
minimum of:
1a. active_binlog_file_no
1b. first_open_binlog_file_no
1c. Any file_no in use by an active dump thread
1d. Any file_no containing oob data referenced by file_no from (1c)
1e. User specified file_no (from PURGE BINARY LOGS TO, if any).
1f. (ToDo): Any file_no that was still active at the last checkpoint.
2. Unix timestamp specifying the minimal value that should not be purged,
optional (used by PURGE BINARY LOGS BEFORE and --binlog-expire-log-seconds).
3. Maximum total size of binlogs, optional (from --max-binlog-total-size).
Sets out_file_no to the earliest binlog file not purged.
Additionally returns:
0 Purged all files as requested.
1 Some files were not purged due to being currently in-use (by binlog
writing or active dump threads).
*/
static int
innodb_binlog_purge_low(uint64_t limit_file_no,
bool by_date, time_t limit_date,
bool by_size, ulonglong limit_size,
bool by_name, uint64_t limit_name_file_no,
uint64_t *out_file_no)
{
ut_ad(by_date || by_size || by_name);
ut_a(limit_file_no <= active_binlog_file_no.load(std::memory_order_relaxed));
ut_a(limit_file_no <= first_open_binlog_file_no);
mysql_mutex_assert_owner(&purge_binlog_mutex);
size_t loc_total_size= total_binlog_used_size;
uint64_t file_no;
bool want_purge;
for (file_no= earliest_binlog_file_no; ; ++file_no)
{
want_purge= false;
char filename[OS_FILE_MAX_PATH];
binlog_name_make(filename, file_no);
MY_STAT stat_buf;
if (!my_stat(filename, &stat_buf, MYF(0)))
{
if (my_errno == ENOENT)
sql_print_information("InnoDB: File already gone when purging binlog "
"file '%s'", filename);
else
sql_print_warning("InnoDB: Failed to stat() when trying to purge "
"binlog file '%' (errno: %d)", filename, my_errno);
continue;
}
if (by_date && stat_buf.st_mtime < limit_date)
want_purge= true;
if (by_size && loc_total_size > limit_size)
want_purge= true;
if (by_name && file_no < limit_name_file_no)
want_purge= true;
if (file_no >= limit_file_no || !want_purge)
break;
earliest_binlog_file_no= file_no + 1;
if (loc_total_size < (size_t)stat_buf.st_size)
{
/*
Somehow we miscounted size, files changed from outside server or
possibly bug. We will handle not underflowing the total. If this
assertion becomes a problem for testing, it can just be removed.
*/
ut_ad(0);
}
else
loc_total_size-= stat_buf.st_size;
if (my_delete(filename, MYF(0)))
{
if (my_errno == ENOENT)
{
/*
File already gone, just ignore the error.
(This should be somewhat unusual to happen as stat() succeeded).
*/
}
else
{
sql_print_warning("InnoDB: Delete failed while trying to purge binlog "
"file '%s' (errno: %d)", filename, my_error);
continue;
}
}
}
total_binlog_used_size= loc_total_size;
*out_file_no= file_no;
return (want_purge ? 1 : 0);
}
static void
innodb_binlog_autopurge(uint64_t first_open_file_no)
{
handler_binlog_purge_info UNINIT_VAR(purge_info);
#ifdef HAVE_REPLICATION
extern bool ha_binlog_purge_info(handler_binlog_purge_info *out_info);
bool can_purge= ha_binlog_purge_info(&purge_info);
#else
bool can_purge= false;
#endif
if (!can_purge ||
!(purge_info.purge_by_size || purge_info.purge_by_date))
return;
/*
ToDo: Here, we need to move back the purge_info.limit_file_no to the
earliest file containing any oob data referenced from the supplied
purge_info.limit_file_no.
*/
/* Don't purge any actively open tablespace files. */
uint64_t limit_file_no= purge_info.limit_file_no;
if (limit_file_no == ~(uint64_t)0 || limit_file_no > first_open_file_no)
limit_file_no= first_open_file_no;
uint64_t active= active_binlog_file_no.load(std::memory_order_relaxed);
if (limit_file_no > active)
limit_file_no= active;
uint64_t file_no;
int res=
innodb_binlog_purge_low(limit_file_no,
purge_info.purge_by_date, purge_info.limit_date,
purge_info.purge_by_size, purge_info.limit_size,
false, 0,
&file_no);
if (res)
{
if (!purge_warning_given)
{
char filename[BINLOG_NAME_MAX_LEN];
binlog_name_make_short(filename, file_no);
if (purge_info.nonpurge_reason)
sql_print_information("InnoDB: Binlog file %s could not be purged "
"because %s",
filename, purge_info.nonpurge_reason);
else if (purge_info.limit_file_no == file_no)
sql_print_information("InnoDB: Binlog file %s could not be purged "
"because it is in use by a binlog dump thread "
"(connected slave)", filename);
else if (limit_file_no == file_no)
sql_print_information("InnoDB: Binlog file %s could not be purged "
"because it is in active use", filename);
else
sql_print_information("InnoDB: Binlog file %s could not be purged "
"because it might still be needed", filename);
purge_warning_given= true;
}
}
else
purge_warning_given= false;
}
int
innodb_binlog_purge(handler_binlog_purge_info *purge_info)
{
/*
Let us check that we do not get an attempt to purge by file, date, and/or
size at the same time.
(If we do, it is not necesarily a problem, but this cannot happen in
current server code).
*/
ut_ad(1 == (!!purge_info->purge_by_name +
!!purge_info->purge_by_date +
!!purge_info->purge_by_size));
if (!purge_info->purge_by_name && !purge_info->purge_by_date &&
!purge_info->purge_by_size)
return 0;
mysql_mutex_lock(&active_binlog_mutex);
uint64_t limit_file_no=
std::min(active_binlog_file_no.load(std::memory_order_relaxed),
first_open_binlog_file_no);
uint64_t last_created= last_created_binlog_file_no;
mysql_mutex_unlock(&active_binlog_mutex);
uint64_t to_file_no= ~(uint64_t)0;
if (purge_info->purge_by_name)
{
if (!is_binlog_name(purge_info->limit_name, &to_file_no) ||
to_file_no > last_created)
return LOG_INFO_EOF;
}
mysql_mutex_lock(&purge_binlog_mutex);
uint64_t file_no;
int res= innodb_binlog_purge_low(
std::min(purge_info->limit_file_no, limit_file_no),
purge_info->purge_by_date, purge_info->limit_date,
purge_info->purge_by_size, purge_info->limit_size,
purge_info->purge_by_name, to_file_no,
&file_no);
mysql_mutex_unlock(&purge_binlog_mutex);
if (res == 1)
{
static_assert(sizeof(purge_info->nonpurge_filename) >= BINLOG_NAME_MAX_LEN,
"No room to return filename");
binlog_name_make_short(purge_info->nonpurge_filename, file_no);
if (!purge_info->nonpurge_reason)
{
if (limit_file_no == file_no)
purge_info->nonpurge_reason= "the binlog file is in active use";
else if (purge_info->limit_file_no == file_no)
purge_info->nonpurge_reason= "it is in use by a binlog dump thread "
"(connected slave)";
}
res= LOG_INFO_IN_USE;
}
else
purge_warning_given= false;
return res;
}

5
storage/innobase/include/fsp_binlog.h
View File

@@ -189,6 +189,10 @@ public:
/* Release any buffer pool page latch. */ /* Release any buffer pool page latch. */
void release(bool release_file_page= false); void release(bool release_file_page= false);
bool data_available(); bool data_available();
uint64_t current_file_no() { return s.file_no; }
uint64_t current_pos() {
return (s.page_no << srv_page_size_shift) + s.in_page_offset;
}
}; };
@@ -210,6 +214,7 @@ extern fil_space_t *fsp_binlog_open(const char *file_name, pfs_os_file_t fh,
uint64_t file_no, size_t file_size, uint64_t file_no, size_t file_size,
bool open_empty); bool open_empty);
extern dberr_t fsp_binlog_tablespace_create(uint64_t file_no, extern dberr_t fsp_binlog_tablespace_create(uint64_t file_no,
uint32_t size_in_pages,
fil_space_t **new_space); fil_space_t **new_space);
extern std::pair<uint64_t, uint64_t> fsp_binlog_write_rec( extern std::pair<uint64_t, uint64_t> fsp_binlog_write_rec(
struct chunk_data_base *chunk_data, mtr_t *mtr, byte chunk_type); struct chunk_data_base *chunk_data, mtr_t *mtr, byte chunk_type);

11
storage/innobase/include/innodb_binlog.h
View File

@@ -35,6 +35,7 @@ struct rpl_binlog_state_base;
struct rpl_gtid; struct rpl_gtid;
struct handler_binlog_event_group_info; struct handler_binlog_event_group_info;
class handler_binlog_reader; class handler_binlog_reader;
struct handler_binlog_purge_info;
/* /*
@@ -72,6 +73,8 @@ extern uint32_t binlog_cur_page_no;
extern uint32_t binlog_cur_page_offset; extern uint32_t binlog_cur_page_offset;
extern ulonglong innodb_binlog_state_interval; extern ulonglong innodb_binlog_state_interval;
extern rpl_binlog_state_base binlog_diff_state; extern rpl_binlog_state_base binlog_diff_state;
extern mysql_mutex_t purge_binlog_mutex;
extern size_t total_binlog_used_size;
static inline void static inline void
@@ -83,6 +86,13 @@ binlog_name_make(char name_buf[OS_FILE_MAX_PATH], uint64_t file_no)
} }
static inline void
binlog_name_make_short(char *name_buf, uint64_t file_no)
{
sprintf(name_buf, BINLOG_NAME_BASE "%06" PRIu64 BINLOG_NAME_EXT, file_no);
}
extern void innodb_binlog_startup_init(); extern void innodb_binlog_startup_init();
extern bool innodb_binlog_init(size_t binlog_size, const char *directory); extern bool innodb_binlog_init(size_t binlog_size, const char *directory);
extern void innodb_binlog_close(bool shutdown); extern void innodb_binlog_close(bool shutdown);
@@ -99,5 +109,6 @@ extern bool innobase_binlog_write_direct
const rpl_gtid *gtid); const rpl_gtid *gtid);
extern bool innodb_find_binlogs(uint64_t *out_first, uint64_t *out_last); extern bool innodb_find_binlogs(uint64_t *out_first, uint64_t *out_last);
extern bool innodb_reset_binlogs(); extern bool innodb_reset_binlogs();
extern int innodb_binlog_purge(handler_binlog_purge_info *purge_info);
#endif /* innodb_binlog_h */ #endif /* innodb_binlog_h */

1
storage/innobase/include/univ.i
View File

@@ -482,6 +482,7 @@ extern mysql_pfs_key_t trx_pool_manager_mutex_key;
extern mysql_pfs_key_t lock_wait_mutex_key; extern mysql_pfs_key_t lock_wait_mutex_key;
extern mysql_pfs_key_t srv_threads_mutex_key; extern mysql_pfs_key_t srv_threads_mutex_key;
extern mysql_pfs_key_t fsp_active_binlog_mutex_key; extern mysql_pfs_key_t fsp_active_binlog_mutex_key;
extern mysql_pfs_key_t fsp_purge_binlog_mutex_key;
# endif /* UNIV_PFS_MUTEX */ # endif /* UNIV_PFS_MUTEX */
# ifdef UNIV_PFS_RWLOCK # ifdef UNIV_PFS_RWLOCK