Skip to content

Commit bc2c16a

Browse files
abhinav04sharmainikep
abhinav04sharma
authored and
inikep
committed
Adding a flag to delay auto binlog rotation when a raft purge is active
Summary: Purging logs in raft is very expensive because we read logs to find the last opid for every to-be-purged binlog. Ideally we should be storing this info in the index file like the prev gtid set but that is a big work item. Purge can cause stalls in the commit pipeline when auto rotation is triggered (current binlog goes beyond max_binlog_size). To quick fix added an atomic to indicate if a raft purge is active, if yes, we delay to auto rotate operation in group commit, avoiding stalls at the cost of slighly exceeding max_binlog_size. Reviewed By: anirbanr-fb Differential Revision: D40743288 fbshipit-source-id: f51a654 --------------------------------------------------------------------- Fix purge log duration logging Summary: Incorrect type %f was used in the format string Reviewed By: li-chi Differential Revision: D42725127 fbshipit-source-id: abd2ecc
1 parent 8568057 commit bc2c16a

10 files changed

+192
-4
lines changed

‎mysql-test/r/mysqld--help-notwin.result

+6
Original file line numberDiff line numberDiff line change
@@ -393,6 +393,11 @@ The following options may be given as the first argument:
393393
tables
394394
--default-week-format=#
395395
The default week format used by WEEK() functions
396+
--delay-auto-rotation-on-raft-log-purge
397+
On the leader, when raft logs are being purged delay
398+
automatic log rotation until purge is done even if we go
399+
a beyond the max binlog size.
400+
(Defaults to on; use --skip-delay-auto-rotation-on-raft-log-purge to disable.)
396401
--delay-key-write[=name]
397402
Type of DELAY_KEY_WRITE
398403
--delayed-insert-limit=#
@@ -3151,6 +3156,7 @@ default-table-encryption FALSE
31513156
default-time-zone (No default value)
31523157
default-tmp-storage-engine InnoDB
31533158
default-week-format 0
3159+
delay-auto-rotation-on-raft-log-purge TRUE
31543160
delay-key-write ON
31553161
delayed-insert-limit 100
31563162
delayed-insert-timeout 300

‎mysql-test/suite/rpl_raft/r/rpl_raft_delay_rotation_on_raft_log_purge.result

+52
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
include/raft_3_node.inc
2+
Warnings:
3+
Note #### Sending passwords in plain text without SSL/TLS is extremely insecure.
4+
Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information.
5+
Warnings:
6+
Note #### Sending passwords in plain text without SSL/TLS is extremely insecure.
7+
Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information.
8+
[connection master]
9+
include/rpl_connect.inc [creating server_4]
10+
include/rpl_connect.inc [creating server_5]
11+
show status like 'rpl_raft_role';
12+
Variable_name Value
13+
Rpl_raft_role LEADER
14+
show status like 'rpl_raft_role';
15+
Variable_name Value
16+
Rpl_raft_role FOLLOWER
17+
reset master;
18+
show status like 'rpl_raft_role';
19+
Variable_name Value
20+
Rpl_raft_role FOLLOWER
21+
reset master;
22+
FLUSH BINARY LOGS;
23+
FLUSH BINARY LOGS;
24+
FLUSH BINARY LOGS;
25+
FLUSH BINARY LOGS;
26+
SELECT SLEEP(1);
27+
SLEEP(1)
28+
0
29+
SET @@GLOBAL.max_binlog_size = 4096;
30+
CREATE TABLE t1(a INT PRIMARY KEY, b TEXT);
31+
SET debug_sync='raft_purge_flag_set SIGNAL purge_reached WAIT_FOR purge_continue';
32+
PURGE RAFT LOGS BEFORE NOW();
33+
SET debug_sync='now WAIT_FOR purge_reached';
34+
INSERT INTO t1 VALUES(1, REPEAT('a', 5000));
35+
SET debug_sync='now SIGNAL purge_continue';
36+
Warnings:
37+
Warning 1868 file ./binary-logs-13000.000006 was not purged because it is the active log file.
38+
"There should be only one raft log:"
39+
SHOW RAFT LOGS;
40+
Log_name File_size Encrypted
41+
# # No
42+
INSERT INTO t1 VALUES(2, "");
43+
"Now we should have two raft logs:"
44+
SHOW RAFT LOGS;
45+
Log_name File_size Encrypted
46+
# # No
47+
# # No
48+
SET @@GLOBAL.max_binlog_size = default;
49+
DROP TABLE t1;
50+
include/sync_slave_sql_with_master.inc
51+
include/sync_slave_sql_with_master.inc
52+
include/rpl_end.inc

‎mysql-test/suite/rpl_raft/t/rpl_raft_delay_rotation_on_raft_log_purge.test

+43
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
source include/have_debug_sync.inc;
2+
source ../include/raft_3_node.inc;
3+
4+
connection server_1;
5+
FLUSH BINARY LOGS;
6+
FLUSH BINARY LOGS;
7+
FLUSH BINARY LOGS;
8+
FLUSH BINARY LOGS;
9+
SELECT SLEEP(1);
10+
11+
SET @@GLOBAL.max_binlog_size = 4096;
12+
CREATE TABLE t1(a INT PRIMARY KEY, b TEXT);
13+
14+
SET debug_sync='raft_purge_flag_set SIGNAL purge_reached WAIT_FOR purge_continue';
15+
send PURGE RAFT LOGS BEFORE NOW();
16+
17+
connection server_1_1;
18+
SET debug_sync='now WAIT_FOR purge_reached';
19+
INSERT INTO t1 VALUES(1, REPEAT('a', 5000));
20+
SET debug_sync='now SIGNAL purge_continue';
21+
22+
connection server_1;
23+
reap;
24+
25+
echo "There should be only one raft log:";
26+
replace_column 1 # 2 #;
27+
SHOW RAFT LOGS;
28+
INSERT INTO t1 VALUES(2, "");
29+
echo "Now we should have two raft logs:";
30+
replace_column 1 # 2 #;
31+
SHOW RAFT LOGS;
32+
33+
# Cleanup
34+
connection server_1;
35+
SET @@GLOBAL.max_binlog_size = default;
36+
DROP TABLE t1;
37+
38+
let $sync_slave_connection= server_2;
39+
source include/sync_slave_sql_with_master.inc;
40+
let $sync_slave_connection= server_3;
41+
source include/sync_slave_sql_with_master.inc;
42+
43+
source include/rpl_end.inc;

‎mysql-test/suite/sys_vars/r/delay_auto_rotation_on_raft_log_purge_basic.result

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
Default value of delay_auto_rotation_on_raft_log_purge is 1
2+
SELECT @@global.delay_auto_rotation_on_raft_log_purge;
3+
@@global.delay_auto_rotation_on_raft_log_purge
4+
1
5+
SELECT @@session.delay_auto_rotation_on_raft_log_purge;
6+
ERROR HY000: Variable 'delay_auto_rotation_on_raft_log_purge' is a GLOBAL variable
7+
Expected error 'Variable is a GLOBAL variable'
8+
delay_auto_rotation_on_raft_log_purge is a dynamic variable
9+
SET @@global.delay_auto_rotation_on_raft_log_purge = 1;
10+
SELECT @@global.delay_auto_rotation_on_raft_log_purge;
11+
@@global.delay_auto_rotation_on_raft_log_purge
12+
1
13+
SET @@global.delay_auto_rotation_on_raft_log_purge = 0;
14+
SELECT @@global.delay_auto_rotation_on_raft_log_purge;
15+
@@global.delay_auto_rotation_on_raft_log_purge
16+
0
17+
SET @@global.delay_auto_rotation_on_raft_log_purge = default;
18+
SELECT @@global.delay_auto_rotation_on_raft_log_purge;
19+
@@global.delay_auto_rotation_on_raft_log_purge
20+
1

‎mysql-test/suite/sys_vars/t/delay_auto_rotation_on_raft_log_purge_basic.test

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
-- source include/load_sysvars.inc
2+
3+
####
4+
# Verify default value 1
5+
####
6+
--echo Default value of delay_auto_rotation_on_raft_log_purge is 1
7+
SELECT @@global.delay_auto_rotation_on_raft_log_purge;
8+
9+
####
10+
# Verify that this is not a session variable #
11+
####
12+
--Error ER_INCORRECT_GLOBAL_LOCAL_VAR
13+
SELECT @@session.delay_auto_rotation_on_raft_log_purge;
14+
--echo Expected error 'Variable is a GLOBAL variable'
15+
16+
####
17+
## Verify that the variable is dynamic
18+
####
19+
--echo delay_auto_rotation_on_raft_log_purge is a dynamic variable
20+
SET @@global.delay_auto_rotation_on_raft_log_purge = 1;
21+
SELECT @@global.delay_auto_rotation_on_raft_log_purge;
22+
23+
SET @@global.delay_auto_rotation_on_raft_log_purge = 0;
24+
SELECT @@global.delay_auto_rotation_on_raft_log_purge;
25+
26+
SET @@global.delay_auto_rotation_on_raft_log_purge = default;
27+
SELECT @@global.delay_auto_rotation_on_raft_log_purge;

‎sql/binlog.cc

+32-4
Original file line numberDiff line numberDiff line change
@@ -3995,7 +3995,21 @@ bool purge_raft_logs(THD *thd, const char *to_log) {
39953995

39963996
// If mysql_bin_log is not an apply log, then it represents the 'raft logs' on
39973997
// leader. Call purge_source_logs_to_file() to handle the purge correctly
3998-
if (!mysql_bin_log.is_apply_log) return purge_source_logs_to_file(thd, to_log);
3998+
if (!mysql_bin_log.is_apply_log) {
3999+
mysql_bin_log.is_raft_log_purge_active = true;
4000+
const auto start = std::chrono::system_clock::now();
4001+
DEBUG_SYNC(thd, "raft_purge_flag_set");
4002+
bool retval = purge_source_logs_to_file(thd, to_log);
4003+
mysql_bin_log.is_raft_log_purge_active = false;
4004+
const auto end = std::chrono::system_clock::now();
4005+
sql_print_information(
4006+
"Raft log purging was active for %s usecs",
4007+
std::to_string(
4008+
std::chrono::duration_cast<std::chrono::microseconds>(end - start)
4009+
.count())
4010+
.c_str());
4011+
return retval;
4012+
}
39994013

40004014
Master_info *active_mi;
40014015
if (!get_and_lock_master_info(&active_mi)) {
@@ -4055,8 +4069,21 @@ bool purge_raft_logs_before_date(THD *thd, time_t purge_time) {
40554069
// If mysql_bin_log is not an apply log, then it represents the 'raft logs' on
40564070
// leader. Call purge_source_logs_before_date() to handle the purge
40574071
// correctly
4058-
if (!mysql_bin_log.is_apply_log)
4059-
return purge_source_logs_before_date(thd, purge_time);
4072+
if (!mysql_bin_log.is_apply_log) {
4073+
mysql_bin_log.is_raft_log_purge_active = true;
4074+
const auto start = std::chrono::system_clock::now();
4075+
DEBUG_SYNC(thd, "raft_purge_flag_set");
4076+
bool retval = purge_source_logs_before_date(thd, purge_time);
4077+
mysql_bin_log.is_raft_log_purge_active = false;
4078+
const auto end = std::chrono::system_clock::now();
4079+
sql_print_information(
4080+
"Raft log purging was active for %s usecs",
4081+
std::to_string(
4082+
std::chrono::duration_cast<std::chrono::microseconds>(end - start)
4083+
.count())
4084+
.c_str());
4085+
return retval;
4086+
}
40604087

40614088
Master_info *active_mi;
40624089
if (!get_and_lock_master_info(&active_mi)) {
@@ -12024,7 +12051,8 @@ int MYSQL_BIN_LOG::ordered_commit(THD *thd, bool all, bool skip_commit) {
1202412051
*/
1202512052
if (DBUG_EVALUATE_IF("force_rotate", 1, 0) ||
1202612053
(do_rotate && thd->commit_error == THD::CE_NONE &&
12027-
!is_rotating_caused_by_incident)) {
12054+
!is_rotating_caused_by_incident &&
12055+
(!delay_auto_rotation_on_raft_log_purge || !is_raft_log_purge_active))) {
1202812056
/*
1202912057
Do not force the rotate as several consecutive groups may
1203012058
request unnecessary rotations.

‎sql/binlog.h

+2
Original file line numberDiff line numberDiff line change
@@ -611,6 +611,8 @@ class MYSQL_BIN_LOG : public TC_LOG {
611611

612612
uint64_t get_ttl_compaction_ts() const { return ttl_compaction_ts.load(); }
613613

614+
std::atomic<bool> is_raft_log_purge_active{false};
615+
614616
private:
615617
/**
616618
Checks binlog error action to identify if the server needs to abort on

‎sql/mysqld.cc

+1
Original file line numberDiff line numberDiff line change
@@ -1252,6 +1252,7 @@ ulong opt_commit_consensus_error_action = 0;
12521252
bool enable_raft_plugin = 0;
12531253
bool disallow_raft = 1; // raft is not allowed by default
12541254
bool override_enable_raft_check = false;
1255+
bool delay_auto_rotation_on_raft_log_purge = true;
12551256
bool abort_on_raft_purge_error = false;
12561257
ulonglong apply_log_retention_num = 0;
12571258
ulonglong apply_log_retention_duration = 0;

‎sql/mysqld.h

+1
Original file line numberDiff line numberDiff line change
@@ -472,6 +472,7 @@ extern bool disable_raft_log_repointing;
472472
extern ulong opt_raft_signal_async_dump_threads;
473473
extern bool disallow_raft;
474474
extern bool override_enable_raft_check;
475+
extern bool delay_auto_rotation_on_raft_log_purge;
475476
extern bool abort_on_raft_purge_error;
476477
extern ulonglong apply_log_retention_num;
477478
extern ulonglong apply_log_retention_duration;

‎sql/sys_vars.cc

+8
Original file line numberDiff line numberDiff line change
@@ -9243,6 +9243,14 @@ static Sys_var_bool Sys_enable_raft_plugin(
92439243
NO_MUTEX_GUARD, NOT_IN_BINLOG, ON_CHECK(validate_enable_raft),
92449244
ON_UPDATE(update_enable_raft_change));
92459245

9246+
static Sys_var_bool Sys_delay_auto_rotation_on_raft_log_purge(
9247+
"delay_auto_rotation_on_raft_log_purge",
9248+
"On the leader, when raft logs are being purged delay automatic log "
9249+
"rotation until purge is done even if we go a beyond the max binlog size.",
9250+
GLOBAL_VAR(delay_auto_rotation_on_raft_log_purge), CMD_LINE(OPT_ARG),
9251+
DEFAULT(true), NO_MUTEX_GUARD, NOT_IN_BINLOG, ON_CHECK(nullptr),
9252+
ON_UPDATE(nullptr));
9253+
92469254
static Sys_var_bool Sys_abort_on_raft_purge_error(
92479255
"abort_on_raft_purge_error",
92489256
"Any error in raft plugin to purge files will abort the server",

0 commit comments

Comments
 (0)