database/mariadb
1
0
Fork 0
mirror of https://github.com/MariaDB/server.git synced 2025-08-08 11:22:35 +03:00
Code Activity

MDEV-4991: GTID binlog indexing

Browse Source 
Improve the performance of slave connect using B+-Tree indexes on each binlog
file. The index allows fast lookup of a GTID position to the corresponding
offset in the binlog file, as well as lookup of a position to find the
corresponding GTID position.

This eliminates a costly sequential scan of the starting binlog file
to find the GTID starting position when a slave connects. This is
especially costly if the binlog file is not cached in memory (IO
cost), or if it is encrypted or a lot of slaves connect simultaneously
(CPU cost).

The size of the index files is generally less than 1% of the binlog data, so
not expected to be an issue.

Most of the work writing the index is done as a background task, in
the binlog background thread. This minimises the performance impact on
transaction commit. A simple global mutex is used to protect index
reads and (background) index writes; this is fine as slave connect is
a relatively infrequent operation.

Here are the user-visible options and status variables. The feature is on by
default and is expected to need no tuning or configuration for most users.

binlog_gtid_index
  On by default. Can be used to disable the indexes for testing purposes.

binlog_gtid_index_page_size (default 4096)
  Page size to use for the binlog GTID index. This is the size of the nodes
  in the B+-tree used internally in the index. A very small page-size (64 is
  the minimum) will be less efficient, but can be used to stress the
  BTree-code during testing.

binlog_gtid_index_span_min (default 65536)
  Control sparseness of the binlog GTID index. If set to N, at most one
  index record will be added for every N bytes of binlog file written.
  This can be used to reduce the number of records in the index, at
  the cost only of having to scan a few more events in the binlog file
  before finding the target position

Two status variables are available to monitor the use of the GTID indexes:

  Binlog_gtid_index_hit
  Binlog_gtid_index_miss

The "hit" status increments for each successful lookup in a GTID index.
The "miss" increments when a lookup is not possible. This indicates that the
index file is missing (eg. binlog written by old server version
without GTID index support), or corrupt.

Signed-off-by: Kristian Nielsen <knielsen@knielsen-hq.org>
This commit is contained in:
Kristian Nielsen
2023-09-08 13:12:49 +02:00
parent 20741b9237
commit d039346a7a
32 changed files with 4315 additions and 256 deletions
Download Patch File Download Diff File Expand all files Collapse all files

135
mysql-test/suite/binlog/r/binlog_gtid_index.result Normal file
View File

@@ -0,0 +1,135 @@
SET GLOBAL binlog_gtid_index= 0;
SET GLOBAL binlog_gtid_index= 1;
SET @gtid1= @@gtid_binlog_pos;
CREATE TABLE t1 (a INT PRIMARY KEY);
SET @gtid2= @@gtid_binlog_pos;
INSERT INTO t1 VALUES (1);
SET @gtid3= @@gtid_binlog_pos;
INSERT INTO t1 VALUES (2);
INSERT INTO t1 VALUES (3);
INSERT INTO t1 VALUES (4);
SET @gtid4= @@gtid_binlog_pos;
INSERT INTO t1 VALUES (5);
SET @gtid5= @@gtid_binlog_pos;
SET @gtid6= @@gtid_binlog_pos;
INSERT INTO t1 VALUES (106);
INSERT INTO t1 VALUES (107);
Ok
1
Ok
1
Ok
1
Ok
1
Ok
1
Ok
1
FLUSH BINARY LOGS;
Ok
1
Ok
1
Ok
1
Ok
1
Ok
1
Ok
1
*** Test that purge deletes the gtid index files. ***
FLUSH BINARY LOGS;
INSERT INTO t1 VALUES (200);
FLUSH BINARY LOGS;
INSERT INTO t1 VALUES (201);
FLUSH BINARY LOGS;
INSERT INTO t1 VALUES (202);
PURGE BINARY LOGS TO 'FILE';
*** Test missed index lookup due to missing or corrupt index file.
FLUSH NO_WRITE_TO_BINLOG BINARY LOGS;
INSERT INTO t1 VALUES (301);
INSERT INTO t1 VALUES (302);
INSERT INTO t1 VALUES (303);
SET @gtid_pos= @@GLOBAL.gtid_binlog_pos;
INSERT INTO t1 VALUES (304);
INSERT INTO t1 VALUES (305);
FLUSH NO_WRITE_TO_BINLOG STATUS;
+++ Initial status:
SHOW STATUS LIKE 'binlog_gtid_index_%';
Variable_name Value
Binlog_gtid_index_hit 0
Binlog_gtid_index_miss 0
+++ GTID Lookup in good index.
Gtid_Lookup_Ok
1
SHOW STATUS LIKE 'binlog_gtid_index_%';
Variable_name Value
Binlog_gtid_index_hit 1
Binlog_gtid_index_miss 0
+++ GTID Lookup, index file is missing.
Gtid_Lookup_Ok
1
SHOW STATUS LIKE 'binlog_gtid_index_%';
Variable_name Value
Binlog_gtid_index_hit 1
Binlog_gtid_index_miss 1
FLUSH NO_WRITE_TO_BINLOG BINARY LOGS;
INSERT INTO t1 VALUES (306);
SET @gtid_pos= @@GLOBAL.gtid_binlog_pos;
INSERT INTO t1 VALUES (307);
INSERT INTO t1 VALUES (308);
FLUSH NO_WRITE_TO_BINLOG BINARY LOGS;
+++ GTID Lookup, first page of index is corrupt.
Gtid_Lookup_Ok
1
SHOW STATUS LIKE 'binlog_gtid_index_%';
Variable_name Value
Binlog_gtid_index_hit 1
Binlog_gtid_index_miss 2
SET @old_page_size= @@GLOBAL.binlog_gtid_index_page_size;
SET @old_span_min= @@GLOBAL.binlog_gtid_index_span_min;
SET GLOBAL binlog_gtid_index_page_size= 64;
SET GLOBAL binlog_gtid_index_span_min= 1;
FLUSH NO_WRITE_TO_BINLOG BINARY LOGS;
INSERT INTO t1 VALUES (310);
INSERT INTO t1 VALUES (311);
INSERT INTO t1 VALUES (312);
SET @gtid_pos= @@GLOBAL.gtid_binlog_pos;
INSERT INTO t1 VALUES (313);
INSERT INTO t1 VALUES (314);
INSERT INTO t1 VALUES (315);
INSERT INTO t1 VALUES (316);
FLUSH NO_WRITE_TO_BINLOG BINARY LOGS;
SET GLOBAL binlog_gtid_index_page_size= @old_page_size;
SET GLOBAL binlog_gtid_index_span_min= @old_span_min;
+++ GTID Lookup, root page of index is corrupt.
Gtid_Lookup_Ok
1
SHOW STATUS LIKE 'binlog_gtid_index_%';
Variable_name Value
Binlog_gtid_index_hit 1
Binlog_gtid_index_miss 3
*** Test BINLOG_GTID_POS() with too-large offset.
FLUSH NO_WRITE_TO_BINLOG BINARY LOGS;
INSERT INTO t1 VALUES (401);
INSERT INTO t1 VALUES (402);
+++ Test the hot index.
SELECT BINLOG_GTID_POS('FILE', 100000000);
BINLOG_GTID_POS('FILE', 100000000)
NULL
SHOW STATUS LIKE 'binlog_gtid_index_%';
Variable_name Value
Binlog_gtid_index_hit 2
Binlog_gtid_index_miss 3
FLUSH NO_WRITE_TO_BINLOG BINARY LOGS;
+++ Test the cold index.
SELECT BINLOG_GTID_POS('FILE', 100000000);
BINLOG_GTID_POS('FILE', 100000000)
NULL
SHOW STATUS LIKE 'binlog_gtid_index_%';
Variable_name Value
Binlog_gtid_index_hit 3
Binlog_gtid_index_miss 3
DROP TABLE t1;

28
mysql-test/suite/binlog/r/binlog_gtid_index_crash.result Normal file
View File

@@ -0,0 +1,28 @@
*** Test that binlog GTID index is recovered after a crash.
CREATE TABLE t1 (a INT PRIMARY KEY, b INT) ENGINE=InnoDB;
Ok
1
Ok
1
Ok
1
SHOW STATUS LIKE 'binlog_gtid_index_%';
Variable_name Value
Binlog_gtid_index_hit 3
Binlog_gtid_index_miss 0
*** Crash the server, check that GTID index can be used after restart.
SET debug_dbug="d,crash_shutdown";
shutdown;
ERROR HY000: Lost connection to server during query
FLUSH NO_WRITE_TO_BINLOG STATUS;
Ok
1
Ok
1
Ok
1
SHOW STATUS LIKE 'binlog_gtid_index_%';
Variable_name Value
Binlog_gtid_index_hit 3
Binlog_gtid_index_miss 0
DROP TABLE t1;