database/mariadb
1
0
Fork 0
mirror of https://github.com/MariaDB/server.git synced 2025-12-24 11:21:21 +03:00
Code Activity

MDEV-12548 Initial implementation of Mariabackup for MariaDB 10.2

Browse Source 
InnoDB I/O and buffer pool interfaces and the redo log format
have been changed between MariaDB 10.1 and 10.2, and the backup
code has to be adjusted accordingly.

The code has been simplified, and many memory leaks have been fixed.
Instead of the file name xtrabackup_logfile, the file name ib_logfile0
is being used for the copy of the redo log. Unnecessary InnoDB startup and
shutdown and some unnecessary threads have been removed.

Some help was provided by Vladislav Vaintroub.

Parameters have been cleaned up and aligned with those of MariaDB 10.2.

The --dbug option has been added, so that in debug builds,
--dbug=d,ib_log can be specified to enable diagnostic messages
for processing redo log entries.

By default, innodb_doublewrite=OFF, so that --prepare works faster.
If more crash-safety for --prepare is needed, double buffering
can be enabled.

The parameter innodb_log_checksums=OFF can be used to ignore redo log
checksums in --backup.

Some messages have been cleaned up.
Unless --export is specified, Mariabackup will not deal with undo log.
The InnoDB mini-transaction redo log is not only about user-level
transactions; it is actually about mini-transactions. To avoid confusion,
call it the redo log, not transaction log.

We disable any undo log processing in --prepare.

Because MariaDB 10.2 supports indexed virtual columns, the
undo log processing would need to be able to evaluate virtual column
expressions. To reduce the amount of code dependencies, we will not
process any undo log in prepare.

This means that the --export option must be disabled for now.

This also means that the following options are redundant
and have been removed:
	xtrabackup --apply-log-only
	innobackupex --redo-only

In addition to disabling any undo log processing, we will disable any
further changes to data pages during --prepare, including the change
buffer merge. This means that restoring incremental backups should
reliably work even when change buffering is being used on the server.
Because of this, preparing a backup will not generate any further
redo log, and the redo log file can be safely deleted. (If the
--export option is enabled in the future, it must generate redo log
when processing undo logs and buffered changes.)

In --prepare, we cannot easily know if a partial backup was used,
especially when restoring a series of incremental backups. So, we
simply warn about any missing files, and ignore the redo log for them.

FIXME: Enable the --export option.

FIXME: Improve the handling of the MLOG_INDEX_LOAD record, and write
a test that initiates a backup while an ALGORITHM=INPLACE operation
is creating indexes or rebuilding a table. An error should be detected
when preparing the backup.

FIXME: In --incremental --prepare, xtrabackup_apply_delta() should
ensure that if FSP_SIZE is modified, the file size will be adjusted
accordingly.
This commit is contained in:
Marko Mäkelä
2017-06-30 10:49:37 +03:00
parent dc722559cc
commit 8c71c6aa8b
52 changed files with 1551 additions and 2189 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
extra/mariabackup/CMakeLists.txt
View File

@@ -30,7 +30,7 @@ ENDIF()
INCLUDE_DIRECTORIES(
${CMAKE_SOURCE_DIR}/include
${CMAKE_SOURCE_DIR}/storage/xtradb/include
${CMAKE_SOURCE_DIR}/storage/innodb/include
${CMAKE_SOURCE_DIR}/sql
${CMAKE_CURRENT_SOURCE_DIR}/quicklz
${CMAKE_CURRENT_SOURCE_DIR}
@@ -74,9 +74,9 @@ MYSQL_ADD_EXECUTABLE(mariabackup
backup_mysql.cc
backup_copy.cc
encryption_plugin.cc
${PROJECT_SOURCE_DIR}/libmysql/libmysql.c
${PROJECT_SOURCE_DIR}/sql/net_serv.cc
${NT_SERVICE_SOURCE}
${PROJECT_SOURCE_DIR}/libmysqld/libmysql.c
COMPONENT backup
)

210
extra/mariabackup/backup_copy.cc
View File

@@ -89,7 +89,7 @@ struct datadir_iter_t {
ulint filepath_len;
char *filepath_rel;
ulint filepath_rel_len;
os_ib_mutex_t mutex;
pthread_mutex_t mutex;
os_file_dir_t dir;
os_file_dir_t dbdir;
os_file_stat_t dbinfo;
@@ -107,7 +107,7 @@ struct datadir_thread_ctxt_t {
datadir_iter_t *it;
uint n_thread;
uint *count;
os_ib_mutex_t count_mutex;
pthread_mutex_t count_mutex;
os_thread_id_t id;
bool ret;
};
@@ -134,12 +134,12 @@ datadir_node_fill(datadir_node_t *node, datadir_iter_t *it)
{
if (node->filepath_len < it->filepath_len) {
free(node->filepath);
node->filepath = (char*)(ut_malloc(it->filepath_len));
node->filepath = (char*)(malloc(it->filepath_len));
node->filepath_len = it->filepath_len;
}
if (node->filepath_rel_len < it->filepath_rel_len) {
free(node->filepath_rel);
node->filepath_rel = (char*)(ut_malloc(it->filepath_rel_len));
node->filepath_rel = (char*)(malloc(it->filepath_rel_len));
node->filepath_rel_len = it->filepath_rel_len;
}
@@ -153,8 +153,8 @@ static
void
datadir_node_free(datadir_node_t *node)
{
ut_free(node->filepath);
ut_free(node->filepath_rel);
free(node->filepath);
free(node->filepath_rel);
memset(node, 0, sizeof(datadir_node_t));
}
@@ -178,10 +178,10 @@ datadir_iter_new(const char *path, bool skip_first_level = true)
{
datadir_iter_t *it;
it = static_cast<datadir_iter_t *>(ut_malloc(sizeof(datadir_iter_t)));
it = static_cast<datadir_iter_t *>(malloc(sizeof(datadir_iter_t)));
memset(it, 0, sizeof(datadir_iter_t));
it->mutex = os_mutex_create();
pthread_mutex_init(&it->mutex, NULL);
it->datadir_path = strdup(path);
it->dir = os_file_opendir(it->datadir_path, TRUE);
@@ -194,20 +194,20 @@ datadir_iter_new(const char *path, bool skip_first_level = true)
it->err = DB_SUCCESS;
it->dbpath_len = FN_REFLEN;
it->dbpath = static_cast<char*>(ut_malloc(it->dbpath_len));
it->dbpath = static_cast<char*>(malloc(it->dbpath_len));
it->filepath_len = FN_REFLEN;
it->filepath = static_cast<char*>(ut_malloc(it->filepath_len));
it->filepath = static_cast<char*>(malloc(it->filepath_len));
it->filepath_rel_len = FN_REFLEN;
it->filepath_rel = static_cast<char*>(ut_malloc(it->filepath_rel_len));
it->filepath_rel = static_cast<char*>(malloc(it->filepath_rel_len));
it->skip_first_level = skip_first_level;
return(it);
error:
ut_free(it);
free(it);
return(NULL);
}
@@ -246,19 +246,15 @@ datadir_iter_next_database(datadir_iter_t *it)
+ strlen (it->dbinfo.name) + 2;
if (len > it->dbpath_len) {
it->dbpath_len = len;
free(it->dbpath);
if (it->dbpath) {
ut_free(it->dbpath);
}
it->dbpath = static_cast<char*>
(ut_malloc(it->dbpath_len));
it->dbpath = static_cast<char*>(
malloc(it->dbpath_len));
}
ut_snprintf(it->dbpath, it->dbpath_len,
"%s/%s", it->datadir_path,
it->dbinfo.name);
srv_normalize_path_for_win(it->dbpath);
os_normalize_path(it->dbpath);
if (it->dbinfo.type == OS_FILE_TYPE_FILE) {
it->is_file = true;
@@ -306,8 +302,8 @@ make_path_n(int n, char **path, ulint *path_len, ...)
va_end(vl);
if (len_needed < *path_len) {
ut_free(*path);
*path = static_cast<char*>(ut_malloc(len_needed));
free(*path);
*path = static_cast<char*>(malloc(len_needed));
}
va_start(vl, path_len);
@@ -378,7 +374,7 @@ datadir_iter_next(datadir_iter_t *it, datadir_node_t *node)
{
bool ret = true;
os_mutex_enter(it->mutex);
pthread_mutex_lock(&it->mutex);
if (datadir_iter_next_file(it)) {
@@ -413,7 +409,7 @@ datadir_iter_next(datadir_iter_t *it, datadir_node_t *node)
ret = false;
done:
os_mutex_exit(it->mutex);
pthread_mutex_unlock(&it->mutex);
return(ret);
}
@@ -427,7 +423,7 @@ static
void
datadir_iter_free(datadir_iter_t *it)
{
os_mutex_free(it->mutex);
pthread_mutex_destroy(&it->mutex);
if (it->dbdir) {
@@ -439,11 +435,11 @@ datadir_iter_free(datadir_iter_t *it)
os_file_closedir(it->dir);
}
ut_free(it->dbpath);
ut_free(it->filepath);
ut_free(it->filepath_rel);
free(it->dbpath);
free(it->filepath);
free(it->filepath_rel);
free(it->datadir_path);
ut_free(it);
free(it);
}
@@ -466,17 +462,17 @@ static
void
datafile_close(datafile_cur_t *cursor)
{
if (cursor->file != 0) {
if (cursor->file != OS_FILE_CLOSED) {
os_file_close(cursor->file);
}
ut_free(cursor->buf);
free(cursor->buf);
}
static
bool
datafile_open(const char *file, datafile_cur_t *cursor, uint thread_n)
{
ulint success;
bool success;
memset(cursor, 0, sizeof(datafile_cur_t));
@@ -490,11 +486,9 @@ datafile_open(const char *file, datafile_cur_t *cursor, uint thread_n)
xb_get_relative_path(cursor->abs_path, FALSE),
sizeof(cursor->rel_path));
cursor->file = os_file_create_simple_no_error_handling(0,
cursor->abs_path,
OS_FILE_OPEN,
OS_FILE_READ_ONLY,
&success, 0);
cursor->file = os_file_create_simple_no_error_handling(
0, cursor->abs_path,
OS_FILE_OPEN, OS_FILE_READ_ALLOW_DELETE, true, &success);
if (!success) {
/* The following call prints an error message */
os_file_get_last_error(TRUE);
@@ -518,7 +512,7 @@ datafile_open(const char *file, datafile_cur_t *cursor, uint thread_n)
posix_fadvise(cursor->file, 0, 0, POSIX_FADV_SEQUENTIAL);
cursor->buf_size = 10 * 1024 * 1024;
cursor->buf = static_cast<byte *>(ut_malloc((ulint)cursor->buf_size));
cursor->buf = static_cast<byte *>(malloc((ulint)cursor->buf_size));
return(true);
}
@@ -528,7 +522,6 @@ static
xb_fil_cur_result_t
datafile_read(datafile_cur_t *cursor)
{
ulint success;
ulint to_read;
xtrabackup_io_throttling();
@@ -540,14 +533,14 @@ datafile_read(datafile_cur_t *cursor)
return(XB_FIL_CUR_EOF);
}
success = os_file_read(cursor->file, cursor->buf, cursor->buf_offset,
to_read);
if (!success) {
if (!os_file_read(IORequestRead,
cursor->file, cursor->buf, cursor->buf_offset,
to_read)) {
return(XB_FIL_CUR_ERROR);
}
posix_fadvise(cursor->file, cursor->buf_offset, to_read,
POSIX_FADV_DONTNEED);
POSIX_FADV_DONTNEED);
cursor->buf_read = to_read;
cursor->buf_offset += to_read;
@@ -917,13 +910,13 @@ run_data_threads(datadir_iter_t *it, os_thread_func_t func, uint n)
{
datadir_thread_ctxt_t *data_threads;
uint i, count;
os_ib_mutex_t count_mutex;
pthread_mutex_t count_mutex;
bool ret;
data_threads = (datadir_thread_ctxt_t*)
(ut_malloc(sizeof(datadir_thread_ctxt_t) * n));
malloc(sizeof(datadir_thread_ctxt_t) * n);
count_mutex = os_mutex_create();
pthread_mutex_init(&count_mutex, NULL);
count = n;
for (i = 0; i < n; i++) {
@@ -937,15 +930,15 @@ run_data_threads(datadir_iter_t *it, os_thread_func_t func, uint n)
/* Wait for threads to exit */
while (1) {
os_thread_sleep(100000);
os_mutex_enter(count_mutex);
pthread_mutex_lock(&count_mutex);
if (count == 0) {
os_mutex_exit(count_mutex);
pthread_mutex_unlock(&count_mutex);
break;
}
os_mutex_exit(count_mutex);
pthread_mutex_unlock(&count_mutex);
}
os_mutex_free(count_mutex);
pthread_mutex_destroy(&count_mutex);
ret = true;
for (i = 0; i < n; i++) {
@@ -955,7 +948,7 @@ run_data_threads(datadir_iter_t *it, os_thread_func_t func, uint n)
}
}
ut_free(data_threads);
free(data_threads);
return(ret);
}
@@ -974,7 +967,6 @@ copy_file(ds_ctxt_t *datasink,
ds_file_t *dstfile = NULL;
datafile_cur_t cursor;
xb_fil_cur_result_t res;
const char *action;
if (!datafile_open(src_file_path, &cursor, thread_n)) {
goto error_close;
@@ -991,9 +983,8 @@ copy_file(ds_ctxt_t *datasink,
goto error;
}
action = xb_get_copy_action();
msg_ts("[%02u] %s %s to %s\n",
thread_n, action, src_file_path, dstfile->path);
thread_n, xb_get_copy_action(), src_file_path, dstfile->path);
/* The main copy loop */
while ((res = datafile_read(&cursor)) == XB_FIL_CUR_SUCCESS) {
@@ -1111,7 +1102,7 @@ read_link_file(const char *ibd_filepath, const char *link_filepath)
while (lastch > 4 && filepath[lastch] <= 0x20) {
filepath[lastch--] = 0x00;
}
srv_normalize_path_for_win(filepath);
os_normalize_path(filepath);
}
tablespace_locations[ibd_filepath] = filepath;
@@ -1629,14 +1620,9 @@ apply_log_finish()
return(true);
}
extern void
os_io_init_simple(void);
bool
copy_back()
{
char *innobase_data_file_path_copy;
ulint i;
bool ret;
datadir_iter_t *it = NULL;
datadir_node_t node;
@@ -1679,24 +1665,16 @@ copy_back()
if (!innobase_data_file_path) {
innobase_data_file_path = (char*) "ibdata1:10M:autoextend";
}
innobase_data_file_path_copy = strdup(innobase_data_file_path);
if (!(ret = srv_parse_data_file_paths_and_sizes(
innobase_data_file_path_copy))) {
srv_sys_space.set_path(".");
if (!srv_sys_space.parse_params(innobase_data_file_path, true)) {
msg("syntax error in innodb_data_file_path\n");
return(false);
}
srv_max_n_threads = 1000;
//os_sync_mutex = NULL;
ut_mem_init();
/* temporally dummy value to avoid crash */
srv_page_size_shift = 14;
srv_page_size = (1 << srv_page_size_shift);
os_sync_init();
sync_init();
os_io_init_simple();
mem_init(srv_mem_pool_size);
sync_check_init();
ut_crc32_init();
/* copy undo tablespaces */
@@ -1707,9 +1685,9 @@ copy_back()
ds_data = ds_create(dst_dir, DS_TYPE_LOCAL);
for (i = 1; i <= srv_undo_tablespaces; i++) {
for (ulong i = 1; i <= srv_undo_tablespaces; i++) {
char filename[20];
sprintf(filename, "undo%03u", (uint)i);
sprintf(filename, "undo%03lu", i);
if (!(ret = copy_or_move_file(filename, filename,
dst_dir, 1))) {
goto cleanup;
@@ -1720,29 +1698,30 @@ copy_back()
ds_data = NULL;
}
/* copy redo logs */
dst_dir = (srv_log_group_home_dir && *srv_log_group_home_dir)
? srv_log_group_home_dir : mysql_data_home;
? srv_log_group_home_dir : mysql_data_home;
/* --backup generates a single ib_logfile0, which we must copy
if it exists. */
ds_data = ds_create(dst_dir, DS_TYPE_LOCAL);
if (!file_exists("ib_logfile0")) {
/* After completed --prepare, redo log files are redundant.
We must delete any redo logs at the destination, so that
the database will not jump to a different log sequence number
(LSN). */
for (i = 0; i < (ulong)innobase_log_files_in_group; i++) {
char filename[20];
sprintf(filename, "ib_logfile%lu", i);
if (!file_exists(filename)) {
continue;
}
if (!(ret = copy_or_move_file(filename, filename,
dst_dir, 1))) {
goto cleanup;
for (uint i = 0; i <= SRV_N_LOG_FILES_MAX + 1; i++) {
char filename[FN_REFLEN];
snprintf(filename, sizeof filename, "%s/ib_logfile%u",
dst_dir, i);
unlink(filename);
}
} else if (!(ret = copy_or_move_file("ib_logfile0", "ib_logfile0",
dst_dir, 1))) {
goto cleanup;
}
ds_destroy(ds_data);
ds_data = NULL;
/* copy innodb system tablespace(s) */
@@ -1751,17 +1730,19 @@ copy_back()
ds_data = ds_create(dst_dir, DS_TYPE_LOCAL);
for (i = 0; i < srv_n_data_files; i++) {
const char *filename = base_name(srv_data_file_names[i]);
for (Tablespace::const_iterator iter(srv_sys_space.begin()),
end(srv_sys_space.end());
iter != end;
++iter) {
const char *filename = base_name(iter->name());
if (!(ret = copy_or_move_file(filename, srv_data_file_names[i],
if (!(ret = copy_or_move_file(filename, iter->name(),
dst_dir, 1))) {
goto cleanup;
}
}
ds_destroy(ds_data);
ds_data = NULL;
/* copy the rest of tablespaces */
ds_data = ds_create(mysql_data_home, DS_TYPE_LOCAL);
@@ -1771,7 +1752,7 @@ copy_back()
datadir_node_init(&node);
while (datadir_iter_next(it, &node)) {
const char *ext_list[] = {"backup-my.cnf", "xtrabackup_logfile",
const char *ext_list[] = {"backup-my.cnf",
"xtrabackup_binary", "xtrabackup_binlog_info",
"xtrabackup_checkpoints", ".qp", ".pmap", ".tmp",
NULL};
@@ -1817,21 +1798,18 @@ copy_back()
continue;
}
/* skip redo logs */
if (sscanf(filename, "ib_logfile%d%c", &i_tmp, &c_tmp) == 1) {
/* skip the redo log (it was already copied) */
if (!strcmp(filename, "ib_logfile0")) {
continue;
}
/* skip innodb data files */
is_ibdata_file = false;
for (i = 0; i < srv_n_data_files; i++) {
const char *ibfile;
ibfile = base_name(srv_data_file_names[i]);
if (strcmp(ibfile, filename) == 0) {
for (Tablespace::const_iterator iter(srv_sys_space.begin()),
end(srv_sys_space.end()); iter != end; ++iter) {
if (strcmp(iter->name(), filename) == 0) {
is_ibdata_file = true;
continue;
break;
}
}
if (is_ibdata_file) {
@@ -1873,20 +1851,13 @@ cleanup:
datadir_node_free(&node);
free(innobase_data_file_path_copy);
if (ds_data != NULL) {
ds_destroy(ds_data);
}
ds_data = NULL;
//os_sync_free();
mem_close();
//os_sync_mutex = NULL;
ut_free_all_mem();
sync_close();
sync_initialized = FALSE;
sync_check_close();
return(ret);
}
@@ -1965,9 +1936,9 @@ cleanup:
datadir_node_free(&node);
os_mutex_enter(ctxt->count_mutex);
pthread_mutex_lock(&ctxt->count_mutex);
--(*ctxt->count);
os_mutex_exit(ctxt->count_mutex);
pthread_mutex_unlock(&ctxt->count_mutex);
ctxt->ret = ret;
@@ -1982,10 +1953,7 @@ decrypt_decompress()
datadir_iter_t *it = NULL;
srv_max_n_threads = 1000;
//os_sync_mutex = NULL;
ut_mem_init();
os_sync_init();
sync_init();
sync_check_init();
/* cd to backup directory */
if (my_setwd(xtrabackup_target_dir, MYF(MY_WME)))
@@ -2014,11 +1982,7 @@ decrypt_decompress()
ds_data = NULL;
sync_close();
sync_initialized = FALSE;
//os_sync_free();
//os_sync_mutex = NULL;
ut_free_all_mem();
sync_check_close();
return(ret);
}

57
extra/mariabackup/backup_mysql.cc
View File

@@ -54,6 +54,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA
#include "mysqld.h"
#include "encryption_plugin.h"
#include <sstream>
#include <sql_error.h>
char *tool_name;
@@ -494,7 +495,7 @@ get_mysql_vars(MYSQL *connection)
&& innodb_log_files_in_group_var) {
char *endptr;
innobase_log_files_in_group = strtol(
srv_n_log_files = strtol(
innodb_log_files_in_group_var, &endptr, 10);
ut_ad(*endptr == 0);
}
@@ -503,7 +504,7 @@ get_mysql_vars(MYSQL *connection)
&& innodb_log_file_size_var) {
char *endptr;
innobase_log_file_size = strtoll(
srv_log_file_size = strtoll(
innodb_log_file_size_var, &endptr, 10);
ut_ad(*endptr == 0);
}
@@ -850,9 +851,9 @@ static
void
start_query_killer()
{
kill_query_thread_stop = os_event_create();
kill_query_thread_started = os_event_create();
kill_query_thread_stopped = os_event_create();
kill_query_thread_stop = os_event_create(0);
kill_query_thread_started = os_event_create(0);
kill_query_thread_stopped = os_event_create(0);
os_thread_create(kill_query_thread, NULL, &kill_query_thread_id);
@@ -1368,17 +1369,27 @@ cleanup:
return(result);
}
static string escape_and_quote(MYSQL *mysql,const char *str)
struct escape_and_quote
{
if (!str)
return "NULL";
size_t len = strlen(str);
char* escaped = (char *)alloca(2 * len + 3);
escaped[0] = '\'';
size_t new_len = mysql_real_escape_string(mysql, escaped+1, str, len);
escaped[new_len + 1] = '\'';
escaped[new_len + 2] = 0;
return string(escaped);
escape_and_quote(MYSQL *mysql, const char *str)
: mysql(mysql), str(str) {}
MYSQL * const mysql;
const char * const str;
};
static
std::ostream&
operator<<(std::ostream& s, const escape_and_quote& eq)
{
if (!eq.str)
return s << "NULL";
s << '\'';
size_t len = strlen(eq.str);
char* escaped = (char *)alloca(2 * len + 1);
len = mysql_real_escape_string(eq.mysql, escaped, eq.str, len);
s << std::string(escaped, len);
s << '\'';
return s;
}
/*********************************************************************//**
@@ -1395,12 +1406,9 @@ write_xtrabackup_info(MYSQL *connection)
char buf_start_time[100];
char buf_end_time[100];
tm tm;
ostringstream oss;
std::ostringstream oss;
const char *xb_stream_name[] = {"file", "tar", "xbstream"};
ut_ad(xtrabackup_stream_fmt < 3);
uuid = read_mysql_one_value(connection, "SELECT UUID()");
server_version = read_mysql_one_value(connection, "SELECT VERSION()");
localtime_r(&history_start_time, &tm);
@@ -1528,28 +1536,21 @@ bool write_backup_config_file()
"# The MySQL server\n"
"[mysqld]\n"
"innodb_checksum_algorithm=%s\n"
"innodb_log_checksum_algorithm=%s\n"
"innodb_data_file_path=%s\n"
"innodb_log_files_in_group=%lu\n"
"innodb_log_file_size=%lld\n"
"innodb_log_file_size=%llu\n"
"innodb_page_size=%lu\n"
"innodb_log_block_size=%lu\n"
"innodb_undo_directory=%s\n"
"innodb_undo_tablespaces=%lu\n"
"%s%s\n"
"%s%s\n"
"%s\n",
innodb_checksum_algorithm_names[srv_checksum_algorithm],
innodb_checksum_algorithm_names[srv_log_checksum_algorithm],
innobase_data_file_path,
srv_n_log_files,
innobase_log_file_size,
srv_log_file_size,
srv_page_size,
srv_log_block_size,
srv_undo_dir,
srv_undo_tablespaces,
innobase_doublewrite_file ? "innodb_doublewrite_file=" : "",
innobase_doublewrite_file ? innobase_doublewrite_file : "",
innobase_buffer_pool_filename ?
"innodb_buffer_pool_filename=" : "",
innobase_buffer_pool_filename ?

4
extra/mariabackup/wsrep.h → extra/mariabackup/backup_wsrep.h
View File

@@ -19,8 +19,8 @@ along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
*******************************************************/
#ifndef WSREP_H
#define WSREP_H
#ifndef BACKUP_WSREP_H
#define BACKUP_WSREP_H
/***********************************************************************
Store Galera checkpoint info in the 'xtrabackup_galera_info' file, if that

63
extra/mariabackup/changed_page_bitmap.cc
View File

@@ -193,9 +193,9 @@ log_online_read_bitmap_page(
ut_a(bitmap_file->offset
<= bitmap_file->size - MODIFIED_PAGE_BLOCK_SIZE);
ut_a(bitmap_file->offset % MODIFIED_PAGE_BLOCK_SIZE == 0);
success = os_file_read(bitmap_file->file, page, bitmap_file->offset,
MODIFIED_PAGE_BLOCK_SIZE);
success = os_file_read(IORequestRead,
bitmap_file->file, page, bitmap_file->offset,
MODIFIED_PAGE_BLOCK_SIZE);
if (UNIV_UNLIKELY(!success)) {
@@ -355,8 +355,7 @@ log_online_setup_bitmap_file_range(
bitmap_files->files =
static_cast<log_online_bitmap_file_range_t::files_t *>
(ut_malloc(bitmap_files->count
* sizeof(bitmap_files->files[0])));
(malloc(bitmap_files->count * sizeof(bitmap_files->files[0])));
memset(bitmap_files->files, 0,
bitmap_files->count * sizeof(bitmap_files->files[0]));
@@ -426,9 +425,9 @@ log_online_setup_bitmap_file_range(
/****************************************************************//**
Open a bitmap file for reading.
@return TRUE if opened successfully */
@return whether opened successfully */
static
ibool
bool
log_online_open_bitmap_file_read_only(
/*==================================*/
const char* name, /*!<in: bitmap file
@@ -438,23 +437,21 @@ log_online_open_bitmap_file_read_only(
log_online_bitmap_file_t* bitmap_file) /*!<out: opened bitmap
file */
{
ibool success = FALSE;
bool success = false;
xb_ad(name[0] != '\0');
ut_snprintf(bitmap_file->name, FN_REFLEN, "%s%s", srv_data_home, name);
bitmap_file->file
= os_file_create_simple_no_error_handling(0, bitmap_file->name,
OS_FILE_OPEN,
OS_FILE_READ_ONLY,
&success,0);
bitmap_file->file = os_file_create_simple_no_error_handling(
0, bitmap_file->name,
OS_FILE_OPEN, OS_FILE_READ_ONLY, true, &success);
if (UNIV_UNLIKELY(!success)) {
/* Here and below assume that bitmap file names do not
contain apostrophes, thus no need for ut_print_filename(). */
msg("InnoDB: Warning: error opening the changed page "
"bitmap \'%s\'\n", bitmap_file->name);
return FALSE;
return success;
}
bitmap_file->size = os_file_get_size(bitmap_file->file);
@@ -465,7 +462,7 @@ log_online_open_bitmap_file_read_only(
posix_fadvise(bitmap_file->file, 0, 0, POSIX_FADV_NOREUSE);
#endif
return TRUE;
return success;
}
/****************************************************************//**
@@ -909,6 +906,37 @@ xb_page_bitmap_setup_next_page(
return TRUE;
}
/** Find the node with the smallest key that greater than equal to search key.
@param[in] tree red-black tree
@param[in] key search key
@return node with the smallest greater-than-or-equal key
@retval NULL if none was found */
static
const ib_rbt_node_t*
rbt_lower_bound(const ib_rbt_t* tree, const void* key)
{
ut_ad(!tree->cmp_arg);
const ib_rbt_node_t* ge = NULL;
for (const ib_rbt_node_t *node = tree->root->left;
node != tree->nil; ) {
int result = tree->compare(node->value, key);
if (result < 0) {
node = node->right;
} else {
ge = node;
if (result == 0) {
break;
}
node = node->left;
}
}
return(ge);
}
/****************************************************************//**
Set up a new bitmap range iterator over a given space id changed
pages in a given bitmap.
@@ -922,8 +950,7 @@ xb_page_bitmap_range_init(
{
byte search_page[MODIFIED_PAGE_BLOCK_SIZE];
xb_page_bitmap_range *result
= static_cast<xb_page_bitmap_range *>
(ut_malloc(sizeof(*result)));
= static_cast<xb_page_bitmap_range *>(malloc(sizeof(*result)));
memset(result, 0, sizeof(*result));
result->bitmap = bitmap;
@@ -1014,5 +1041,5 @@ xb_page_bitmap_range_deinit(
/*========================*/
xb_page_bitmap_range* bitmap_range) /*! in/out: bitmap range */
{
ut_free(bitmap_range);
free(bitmap_range);
}

21
extra/mariabackup/encryption_plugin.cc
View File

@@ -8,6 +8,7 @@
#include <vector>
#include <common.h>
#include <backup_mysql.h>
#include <log0crypt.h>
extern struct st_maria_plugin *mysql_optional_plugins[];
@@ -18,14 +19,14 @@ extern char *xb_plugin_load;
extern char *xb_plugin_dir;
const int PLUGIN_MAX_ARGS = 1024;
vector<string> backup_plugins_args;
std::vector<std::string> backup_plugins_args;
const char *QUERY_PLUGIN =
"SELECT plugin_name, plugin_library, @@plugin_dir"
" FROM information_schema.plugins WHERE plugin_type='ENCRYPTION'"
" AND plugin_status='ACTIVE'";
string encryption_plugin_config;
std::string encryption_plugin_config;
static void add_to_plugin_load_list(const char *plugin_def)
{
@@ -38,7 +39,7 @@ void encryption_plugin_backup_init(MYSQL *mysql)
{
MYSQL_RES *result;
MYSQL_ROW row;
ostringstream oss;
std::ostringstream oss;
char *argv[PLUGIN_MAX_ARGS];
int argc;
@@ -59,17 +60,17 @@ void encryption_plugin_backup_init(MYSQL *mysql)
if (*p == '\\') *p = '/';
#endif
string plugin_load(name);
std::string plugin_load(name);
if (library)
plugin_load += string("=") + library;
plugin_load += std::string("=") + library;
oss << "plugin_load=" << plugin_load << endl;
oss << "plugin_load=" << plugin_load << std::endl;
/* Required to load the plugin later.*/
add_to_plugin_load_list(plugin_load.c_str());
strncpy(opt_plugin_dir, dir, FN_REFLEN);
oss << "plugin_dir=" << '"' << dir << '"' << endl;
oss << "plugin_dir=" << '"' << dir << '"' << std::endl;
/* Read plugin variables. */
@@ -80,12 +81,12 @@ void encryption_plugin_backup_init(MYSQL *mysql)
result = xb_mysql_query(mysql, query, true, true);
while ((row = mysql_fetch_row(result)))
{
string arg("--");
std::string arg("--");
arg += row[0];
arg += "=";
arg += row[1];
backup_plugins_args.push_back(arg);
oss << row[0] << "=" << row[1] << endl;
oss << row[0] << "=" << row[1] << std::endl;
}
mysql_free_result(result);
@@ -94,7 +95,7 @@ void encryption_plugin_backup_init(MYSQL *mysql)
result = xb_mysql_query(mysql, "select @@innodb_encrypt_log", true, true);
row = mysql_fetch_row(result);
srv_encrypt_log = (row != 0 && row[0][0] == '1');
oss << "innodb_encrypt_log=" << row[0] << endl;
oss << "innodb_encrypt_log=" << row[0] << std::endl;
mysql_free_result(result);

187
extra/mariabackup/fil_cur.cc
View File

@@ -24,8 +24,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
#include <my_base.h>
#include <univ.i>
#include <fil0fil.h>
#include <fsp0fsp.h>
#include <srv0start.h>
#include <trx0sys.h>
@@ -63,7 +63,7 @@ xb_get_relative_path(
prev = NULL;
cur = path;
while ((next = strchr(cur, SRV_PATH_SEPARATOR)) != NULL) {
while ((next = strchr(cur, OS_PATH_SEPARATOR)) != NULL) {
prev = cur;
cur = next + 1;
@@ -96,7 +96,7 @@ xb_fil_node_close_file(
ut_a(node->n_pending_flushes == 0);
ut_a(!node->being_extended);
if (!node->open) {
if (!node->is_open()) {
mutex_exit(&fil_system->mutex);
@@ -106,19 +106,19 @@ xb_fil_node_close_file(
ret = os_file_close(node->handle);
ut_a(ret);
node->open = FALSE;
node->handle = OS_FILE_CLOSED;
ut_a(fil_system->n_open > 0);
fil_system->n_open--;
fil_n_file_opened--;
if (node->space->purpose == FIL_TABLESPACE &&
if (node->space->purpose == FIL_TYPE_TABLESPACE &&
fil_is_user_tablespace_id(node->space->id)) {
ut_a(UT_LIST_GET_LEN(fil_system->LRU) > 0);
/* The node is in the LRU list, remove it */
UT_LIST_REMOVE(LRU, fil_system->LRU, node);
UT_LIST_REMOVE(fil_system->LRU, node);
}
mutex_exit(&fil_system->mutex);
@@ -137,10 +137,7 @@ xb_fil_cur_open(
fil_node_t* node, /*!< in: source tablespace node */
uint thread_n) /*!< thread number for diagnostics */
{
ulint page_size;
ulint page_size_shift;
ulint zip_size;
ibool success;
bool success;
/* Initialize these first so xb_fil_cur_close() handles them correctly
in case of error */
@@ -148,27 +145,26 @@ xb_fil_cur_open(
cursor->node = NULL;
cursor->space_id = node->space->id;
cursor->is_system = !fil_is_user_tablespace_id(node->space->id);
strncpy(cursor->abs_path, node->name, sizeof(cursor->abs_path));
/* Get the relative path for the destination tablespace name, i.e. the
one that can be appended to the backup root directory. Non-system
tablespaces may have absolute paths for remote tablespaces in MySQL
5.6+. We want to make "local" copies for the backup. */
tablespaces may have absolute paths for DATA DIRECTORY.
We want to make "local" copies for the backup. */
strncpy(cursor->rel_path,
xb_get_relative_path(cursor->abs_path, cursor->is_system),
xb_get_relative_path(cursor->abs_path, cursor->is_system()),
sizeof(cursor->rel_path));
/* In the backup mode we should already have a tablespace handle created
by fil_load_single_table_tablespace() unless it is a system
by fil_ibd_load() unless it is a system
tablespace. Otherwise we open the file here. */
if (cursor->is_system || !srv_backup_mode || srv_close_files) {
node->handle =
os_file_create_simple_no_error_handling(0, node->name,
OS_FILE_OPEN,
OS_FILE_READ_ONLY,
&success,0);
if (cursor->is_system() || srv_operation == SRV_OPERATION_RESTORE
|| xb_close_files) {
node->handle = os_file_create_simple_no_error_handling(
0, node->name,
OS_FILE_OPEN,
OS_FILE_READ_ALLOW_DELETE, true, &success);
if (!success) {
/* The following call prints an error message */
os_file_get_last_error(TRUE);
@@ -181,22 +177,20 @@ xb_fil_cur_open(
}
mutex_enter(&fil_system->mutex);
node->open = TRUE;
fil_system->n_open++;
fil_n_file_opened++;
if (node->space->purpose == FIL_TABLESPACE &&
if (node->space->purpose == FIL_TYPE_TABLESPACE &&
fil_is_user_tablespace_id(node->space->id)) {
/* Put the node to the LRU list */
UT_LIST_ADD_FIRST(LRU, fil_system->LRU, node);
UT_LIST_ADD_FIRST(fil_system->LRU, node);
}
mutex_exit(&fil_system->mutex);
}
ut_ad(node->open);
ut_ad(node->is_open());
cursor->node = node;
cursor->file = node->handle;
@@ -210,8 +204,8 @@ xb_fil_cur_open(
return(XB_FIL_CUR_ERROR);
}
if (srv_unix_file_flush_method == SRV_UNIX_O_DIRECT
|| srv_unix_file_flush_method == SRV_UNIX_O_DIRECT_NO_FSYNC) {
if (srv_file_flush_method == SRV_O_DIRECT
|| srv_file_flush_method == SRV_O_DIRECT_NO_FSYNC) {
os_file_set_nocache(cursor->file, node->name, "OPEN");
}
@@ -219,32 +213,29 @@ xb_fil_cur_open(
posix_fadvise(cursor->file, 0, 0, POSIX_FADV_SEQUENTIAL);
/* Determine the page size */
zip_size = xb_get_zip_size(cursor->file);
if (zip_size == ULINT_UNDEFINED) {
ulint flags = xb_get_space_flags(cursor->file);
if (flags == ULINT_UNDEFINED) {
xb_fil_cur_close(cursor);
return(XB_FIL_CUR_SKIP);
} else if (zip_size) {
page_size = zip_size;
page_size_shift = get_bit_shift(page_size);
msg("[%02u] %s is compressed with page size = "
"%lu bytes\n", thread_n, node->name, page_size);
if (page_size_shift < 10 || page_size_shift > 14) {
msg("[%02u] xtrabackup: Error: Invalid "
"page size: %lu.\n", thread_n, page_size);
ut_error;
}
} else {
page_size = UNIV_PAGE_SIZE;
page_size_shift = UNIV_PAGE_SIZE_SHIFT;
}
if (!fsp_flags_is_valid(flags)) {
ulint cflags = fsp_flags_convert_from_101(flags);
if (cflags == ULINT_UNDEFINED) {
msg("[%02u] xtrabackup: Error: Invalid "
"tablespace flags: %x.\n", thread_n, uint(flags));
return(XB_FIL_CUR_SKIP);
}
flags = cflags;
}
const page_size_t page_size(flags);
cursor->page_size = page_size;
cursor->page_size_shift = page_size_shift;
cursor->zip_size = zip_size;
/* Allocate read buffer */
cursor->buf_size = XB_FIL_CUR_PAGES * page_size;
cursor->buf_size = XB_FIL_CUR_PAGES * page_size.physical();
cursor->orig_buf = static_cast<byte *>
(ut_malloc(cursor->buf_size + UNIV_PAGE_SIZE));
(malloc(cursor->buf_size + UNIV_PAGE_SIZE));
cursor->buf = static_cast<byte *>
(ut_align(cursor->orig_buf, UNIV_PAGE_SIZE));
@@ -254,7 +245,8 @@ xb_fil_cur_open(
cursor->buf_page_no = 0;
cursor->thread_n = thread_n;
cursor->space_size = (ulint)(cursor->statinfo.st_size / page_size);
cursor->space_size = (ulint)(cursor->statinfo.st_size
/ page_size.physical());
cursor->read_filter = read_filter;
cursor->read_filter->init(&cursor->read_filter_ctxt, cursor,
@@ -282,6 +274,8 @@ xb_fil_cur_read(
xb_fil_cur_result_t ret;
ib_int64_t offset;
ib_int64_t to_read;
const ulint page_size = cursor->page_size.physical();
xb_ad(!cursor->is_system() || page_size == UNIV_PAGE_SIZE);
cursor->read_filter->get_next_batch(&cursor->read_filter_ctxt,
&offset, &to_read);
@@ -296,10 +290,10 @@ xb_fil_cur_read(
xb_a(to_read > 0 && to_read <= 0xFFFFFFFFLL);
if (to_read % cursor->page_size != 0 &&
offset + to_read == cursor->statinfo.st_size) {
if ((to_read & ~(page_size - 1))
&& offset + to_read == cursor->statinfo.st_size) {
if (to_read < (ib_int64_t) cursor->page_size) {
if (to_read < (ib_int64_t) page_size) {
msg("[%02u] xtrabackup: Warning: junk at the end of "
"%s:\n", cursor->thread_n, cursor->abs_path);
msg("[%02u] xtrabackup: Warning: offset = %llu, "
@@ -312,12 +306,12 @@ xb_fil_cur_read(
}
to_read = (ib_int64_t) (((ulint) to_read) &
~(cursor->page_size - 1));
~(page_size - 1));
}
xb_a(to_read % cursor->page_size == 0);
xb_a((to_read & (page_size - 1)) == 0);
npages = (ulint) (to_read >> cursor->page_size_shift);
npages = (ulint) (to_read / cursor->page_size.physical());
retry_count = 10;
ret = XB_FIL_CUR_SUCCESS;
@@ -328,59 +322,59 @@ read_retry:
cursor->buf_read = 0;
cursor->buf_npages = 0;
cursor->buf_offset = offset;
cursor->buf_page_no = (ulint)(offset >> cursor->page_size_shift);
cursor->buf_page_no = (ulint)(offset / cursor->page_size.physical());
success = os_file_read(cursor->file, cursor->buf, offset,
(ulint)to_read);
FilSpace space(cursor->space_id);
if (!space()) {
return(XB_FIL_CUR_ERROR);
}
success = os_file_read(IORequestRead,
cursor->file, cursor->buf, offset,
(ulint) to_read);
if (!success) {
return(XB_FIL_CUR_ERROR);
}
fil_system_enter();
fil_space_t *space = fil_space_get_by_id(cursor->space_id);
fil_system_exit();
/* check pages for corruption and re-read if necessary. i.e. in case of
partially written pages */
for (page = cursor->buf, i = 0; i < npages;
page += cursor->page_size, i++) {
ib_int64_t page_no = cursor->buf_page_no + i;
page += page_size, i++) {
ulint page_no = cursor->buf_page_no + i;
bool checksum_ok = fil_space_verify_crypt_checksum(page, cursor->zip_size,space, (ulint)page_no);
if (!checksum_ok &&
buf_page_is_corrupted(true, page, cursor->zip_size,space)) {
if (cursor->is_system &&
page_no >= (ib_int64_t)FSP_EXTENT_SIZE &&
page_no < (ib_int64_t) FSP_EXTENT_SIZE * 3) {
/* skip doublewrite buffer pages */
xb_a(cursor->page_size == UNIV_PAGE_SIZE);
if (cursor->space_id == TRX_SYS_SPACE &&
page_no >= FSP_EXTENT_SIZE &&
page_no < FSP_EXTENT_SIZE * 3) {
/* We ignore the doublewrite buffer pages */
} else if (!fil_space_verify_crypt_checksum(
page, cursor->page_size, space->id, page_no)
&& buf_page_is_corrupted(true, page,
cursor->page_size,
space)) {
retry_count--;
if (retry_count == 0) {
msg("[%02u] xtrabackup: "
"Page %lu is a doublewrite buffer page, "
"skipping.\n", cursor->thread_n, page_no);
} else {
retry_count--;
if (retry_count == 0) {
msg("[%02u] xtrabackup: "
"Error: failed to read page after "
"10 retries. File %s seems to be "
"corrupted.\n", cursor->thread_n,
cursor->abs_path);
ret = XB_FIL_CUR_ERROR;
break;
}
"Error: failed to read page after "
"10 retries. File %s seems to be "
"corrupted.\n", cursor->thread_n,
cursor->abs_path);
ret = XB_FIL_CUR_ERROR;
break;
}
if (retry_count == 9) {
msg("[%02u] xtrabackup: "
"Database page corruption detected at page "
"%lu, retrying...\n", cursor->thread_n,
page_no);
os_thread_sleep(100000);
goto read_retry;
ULINTPF ", retrying...\n",
cursor->thread_n, page_no);
}
os_thread_sleep(100000);
goto read_retry;
}
cursor->buf_read += cursor->page_size;
cursor->buf_read += page_size;
cursor->buf_npages++;
}
@@ -399,11 +393,10 @@ xb_fil_cur_close(
{
cursor->read_filter->deinit(&cursor->read_filter_ctxt);
if (cursor->orig_buf != NULL) {
ut_free(cursor->orig_buf);
}
free(cursor->orig_buf);
if (cursor->node != NULL) {
xb_fil_node_close_file(cursor->node);
cursor->file = XB_FILE_UNDEFINED;
cursor->file = OS_FILE_CLOSED;
}
}

21
extra/mariabackup/fil_cur.h
View File

@@ -27,6 +27,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
#include <my_dir.h>
#include "read_filt.h"
#include "srv0start.h"
struct xb_fil_cur_t {
pfs_os_file_t file; /*!< source file handle */
@@ -36,14 +37,7 @@ struct xb_fil_cur_t {
char abs_path[FN_REFLEN];
/*!< absolute file path */
MY_STAT statinfo; /*!< information about the file */
ulint zip_size; /*!< compressed page size in bytes or 0
for uncompressed pages */
ulint page_size; /*!< = zip_size for compressed pages or
UNIV_PAGE_SIZE for uncompressed ones */
ulint page_size_shift;/*!< bit shift corresponding to
page_size */
my_bool is_system; /*!< TRUE for system tablespace, FALSE
otherwise */
page_size_t page_size; /*!< page size */
xb_read_filt_t* read_filter; /*!< read filter */
xb_read_filt_ctxt_t read_filter_ctxt;
/*!< read filter context */
@@ -61,6 +55,17 @@ struct xb_fil_cur_t {
uint thread_n; /*!< thread number for diagnostics */
ulint space_id; /*!< ID of tablespace */
ulint space_size; /*!< space size in pages */
/** TODO: remove this default constructor */
xb_fil_cur_t() : page_size(0), read_filter_ctxt() {}
/** @return whether this is not a file-per-table tablespace */
bool is_system() const
{
ut_ad(space_id != SRV_TMP_SPACE_ID);
return(space_id == TRX_SYS_SPACE
|| srv_is_undo_tablespace(space_id));
}
};
typedef enum {

21
extra/mariabackup/innobackupex.cc
View File

@@ -45,7 +45,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA
#include <mysql.h>
#include <my_dir.h>
#include <ut0mem.h>
#include <os0sync.h>
#include <os0file.h>
#include <srv0start.h>
#include <algorithm>
@@ -70,7 +69,6 @@ using std::max;
my_bool opt_ibx_version = FALSE;
my_bool opt_ibx_help = FALSE;
my_bool opt_ibx_apply_log = FALSE;
my_bool opt_ibx_redo_only = FALSE;
my_bool opt_ibx_incremental = FALSE;
my_bool opt_ibx_notimestamp = FALSE;
@@ -226,21 +224,11 @@ static struct my_option ibx_long_options[] =
GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
{"apply-log", OPT_APPLY_LOG, "Prepare a backup in BACKUP-DIR by "
"applying the transaction log file named \"xtrabackup_logfile\" "
"located in the same directory. Also, create new transaction logs. "
"applying the redo log 'ib_logfile0' and creating new redo log. "
"The InnoDB configuration is read from the file \"backup-my.cnf\".",
(uchar*) &opt_ibx_apply_log, (uchar*) &opt_ibx_apply_log,
0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
{"redo-only", OPT_REDO_ONLY, "This option should be used when "
"preparing the base full backup and when merging all incrementals "
"except the last one. This forces xtrabackup to skip the \"rollback\" "
"phase and do a \"redo\" only. This is necessary if the backup will "
"have incremental changes applied to it later. See the xtrabackup "
"documentation for details.",
(uchar *) &opt_ibx_redo_only, (uchar *) &opt_ibx_redo_only, 0,
GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
{"copy-back", OPT_COPY_BACK, "Copy all the files in a previously made "
"backup from the backup directory to their original locations.",
(uchar *) &opt_ibx_copy_back, (uchar *) &opt_ibx_copy_back, 0,
@@ -682,7 +670,7 @@ innobackupex [--compress] [--compress-threads=NUMBER-OF-THREADS] [--compress-chu
\n\
innobackupex --apply-log [--use-memory=B]\n\
[--defaults-file=MY.CNF]\n\
[--export] [--redo-only] [--ibbackup=IBBACKUP-BINARY]\n\
[--export] [--ibbackup=IBBACKUP-BINARY]\n\
BACKUP-DIR\n\
\n\
innobackupex --copy-back [--defaults-file=MY.CNF] [--defaults-group=GROUP-NAME] BACKUP-DIR\n\
@@ -710,7 +698,7 @@ process.\n\
\n\
The --apply-log command prepares a backup for starting a MySQL\n\
server on the backup. This command recovers InnoDB data files as specified\n\
in BACKUP-DIR/backup-my.cnf using BACKUP-DIR/xtrabackup_logfile,\n\
in BACKUP-DIR/backup-my.cnf using BACKUP-DIR/ib_logfile0,\n\
and creates new InnoDB log files as specified in BACKUP-DIR/backup-my.cnf.\n\
The BACKUP-DIR should be the path to a backup directory created by\n\
xtrabackup. This command runs xtrabackup as a child process, but it does not \n\
@@ -980,9 +968,6 @@ ibx_init()
switch (ibx_mode) {
case IBX_MODE_APPLY_LOG:
xtrabackup_prepare = TRUE;
if (opt_ibx_redo_only) {
xtrabackup_apply_log_only = TRUE;
}
xtrabackup_target_dir = ibx_position_arg;
run = "apply-log";
break;

15
extra/mariabackup/read_filt.cc
View File

@@ -127,10 +127,11 @@ rf_bitmap_get_next_batch(
of pages */
{
ulint start_page_id;
const ulint page_size = ctxt->page_size.physical();
start_page_id = (ulint)(ctxt->offset / ctxt->page_size);
start_page_id = (ulint)(ctxt->offset / page_size);
xb_a (ctxt->offset % ctxt->page_size == 0);
xb_a (ctxt->offset % page_size == 0);
if (start_page_id == ctxt->filter_batch_end) {
@@ -146,7 +147,7 @@ rf_bitmap_get_next_batch(
return;
}
ctxt->offset = next_page_id * ctxt->page_size;
ctxt->offset = next_page_id * page_size;
/* Find the end of the current changed page block by searching
for the next cleared bitmap bit */
@@ -162,7 +163,7 @@ rf_bitmap_get_next_batch(
remaining pages. */
*read_batch_len = ctxt->data_file_size - ctxt->offset;
} else {
*read_batch_len = ctxt->filter_batch_end * ctxt->page_size
*read_batch_len = ctxt->filter_batch_end * page_size
- ctxt->offset;
}
@@ -175,9 +176,9 @@ rf_bitmap_get_next_batch(
}
ctxt->offset += *read_batch_len;
xb_a (ctxt->offset % ctxt->page_size == 0);
xb_a (*read_batch_start % ctxt->page_size == 0);
xb_a (*read_batch_len % ctxt->page_size == 0);
xb_a (ctxt->offset % page_size == 0);
xb_a (*read_batch_start % page_size == 0);
xb_a (*read_batch_len % page_size == 0);
}
/****************************************************************//**

8
extra/mariabackup/read_filt.h
View File

@@ -27,6 +27,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
#include "changed_page_bitmap.h"
typedef ulint space_id_t;
struct xb_fil_cur_t;
/* The read filter context */
@@ -34,15 +36,17 @@ struct xb_read_filt_ctxt_t {
ib_int64_t offset; /*!< current file offset */
ib_int64_t data_file_size; /*!< data file size */
size_t buffer_capacity;/*!< read buffer capacity */
ib_int64_t space_id; /*!< space id */
space_id_t space_id; /*!< space id */
/* The following fields used only in bitmap filter */
/* Move these to union if any other filters are added in future */
xb_page_bitmap_range *bitmap_range; /*!< changed page bitmap range
iterator for space_id */
size_t page_size; /*!< page size */
page_size_t page_size; /*!< page size */
ulint filter_batch_end;/*!< the ending page id of the
current changed page block in
the bitmap */
/** TODO: remove this default constructor */
xb_read_filt_ctxt_t() : page_size(0) {}
};
/* The read filter */

20
extra/mariabackup/write_filt.cc
View File

@@ -67,7 +67,6 @@ wf_incremental_init(xb_write_filt_ctxt_t *ctxt, char *dst_name,
xb_fil_cur_t *cursor)
{
char meta_name[FN_REFLEN];
xb_delta_info_t info;
ulint buf_size;
xb_wf_incremental_ctxt_t *cp =
&(ctxt->u.wf_incremental_ctxt);
@@ -75,8 +74,9 @@ wf_incremental_init(xb_write_filt_ctxt_t *ctxt, char *dst_name,
ctxt->cursor = cursor;
/* allocate buffer for incremental backup (4096 pages) */
buf_size = (cursor->page_size / 4 + 1) * cursor->page_size;
cp->delta_buf_base = static_cast<byte *>(ut_malloc(buf_size));
buf_size = (cursor->page_size.physical() / 4 + 1)
* cursor->page_size.physical();
cp->delta_buf_base = static_cast<byte *>(malloc(buf_size));
memset(cp->delta_buf_base, 0, buf_size);
cp->delta_buf = static_cast<byte *>
(ut_align(cp->delta_buf_base, UNIV_PAGE_SIZE_MAX));
@@ -84,9 +84,7 @@ wf_incremental_init(xb_write_filt_ctxt_t *ctxt, char *dst_name,
/* write delta meta info */
snprintf(meta_name, sizeof(meta_name), "%s%s", dst_name,
XB_DELTA_INFO_SUFFIX);
info.page_size = cursor->page_size;
info.zip_size = cursor->zip_size;
info.space_id = cursor->space_id;
const xb_delta_info_t info(cursor->page_size, cursor->space_id);
if (!xb_write_delta_metadata(meta_name, &info)) {
msg("[%02u] xtrabackup: Error: "
"failed to write meta info for %s\n",
@@ -113,8 +111,9 @@ wf_incremental_process(xb_write_filt_ctxt_t *ctxt, ds_file_t *dstfile)
{
ulint i;
xb_fil_cur_t *cursor = ctxt->cursor;
ulint page_size = cursor->page_size;
byte *page;
const ulint page_size
= cursor->page_size.physical();
xb_wf_incremental_ctxt_t *cp = &(ctxt->u.wf_incremental_ctxt);
for (i = 0, page = cursor->buf; i < cursor->buf_npages;
@@ -159,7 +158,8 @@ static my_bool
wf_incremental_finalize(xb_write_filt_ctxt_t *ctxt, ds_file_t *dstfile)
{
xb_fil_cur_t *cursor = ctxt->cursor;
ulint page_size = cursor->page_size;
const ulint page_size
= cursor->page_size.physical();
xb_wf_incremental_ctxt_t *cp = &(ctxt->u.wf_incremental_ctxt);
if (cp->npages != page_size / 4) {
@@ -184,9 +184,7 @@ wf_incremental_deinit(xb_write_filt_ctxt_t *ctxt)
{
xb_wf_incremental_ctxt_t *cp = &(ctxt->u.wf_incremental_ctxt);
if (cp->delta_buf_base != NULL) {
ut_free(cp->delta_buf_base);
}
free(cp->delta_buf_base);
}
/************************************************************************

36
extra/mariabackup/xb0xb.h
View File

@@ -21,43 +21,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
#ifndef xb0xb_h
#define xb0xb_h
extern void os_io_init_simple(void);
extern pfs_os_file_t files[1000];
extern const char *innodb_checksum_algorithm_names[];
extern TYPELIB innodb_checksum_algorithm_typelib;
extern dberr_t open_or_create_data_files(
bool* create_new_db,
#ifdef UNIV_LOG_ARCHIVE
lsn_t* min_arch_log_no,
lsn_t* max_arch_log_no,
#endif
lsn_t* flushed_lsn,
ulint* sum_of_new_sizes)
;
int
fil_file_readdir_next_file(
/*=======================*/
dberr_t* err, /*!< out: this is set to DB_ERROR if an error
was encountered, otherwise not changed */
const char* dirname,/*!< in: directory name or path */
os_file_dir_t dir, /*!< in: directory stream */
os_file_stat_t* info) /*!< in/out: buffer where the
info is returned */;
fil_space_t*
fil_space_get_by_name(const char *);
ibool
recv_check_cp_is_consistent(const byte* buf);
void
innodb_log_checksum_func_update(
/*============================*/
ulint algorithm) /*!< in: algorithm */;
dberr_t
srv_undo_tablespaces_init(
/*======================*/
ibool create_new_db,
ibool backup_mode,
const ulint n_conf_tablespaces,
ulint* n_opened);
#endif

2646
extra/mariabackup/xtrabackup.cc
View File

File diff suppressed because it is too large Load Diff

50
extra/mariabackup/xtrabackup.h
View File

@@ -26,26 +26,14 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
#include "xbstream.h"
#include "changed_page_bitmap.h"
#ifdef __WIN__
#define XB_FILE_UNDEFINED INVALID_HANDLE_VALUE
#else
#define XB_FILE_UNDEFINED (-1)
#endif
struct xb_delta_info_t
{
xb_delta_info_t(page_size_t page_size, ulint space_id)
: page_size(page_size), space_id(space_id) {}
typedef struct {
ulint page_size;
ulint zip_size;
ulint space_id;
} xb_delta_info_t;
/* ======== Datafiles iterator ======== */
typedef struct {
fil_system_t *system;
fil_space_t *space;
fil_node_t *node;
ibool started;
os_ib_mutex_t mutex;
} datafiles_iter_t;
page_size_t page_size;
ulint space_id;
};
/* value of the --incremental option */
extern lsn_t incremental_lsn;
@@ -84,15 +72,11 @@ extern ibool xtrabackup_compress;
extern my_bool xtrabackup_backup;
extern my_bool xtrabackup_prepare;
extern my_bool xtrabackup_apply_log_only;
extern my_bool xtrabackup_copy_back;
extern my_bool xtrabackup_move_back;
extern my_bool xtrabackup_decrypt_decompress;
extern char *innobase_data_file_path;
extern char *innobase_doublewrite_file;
extern longlong innobase_log_file_size;
extern long innobase_log_files_in_group;
extern longlong innobase_page_size;
extern int xtrabackup_parallel;
@@ -108,9 +92,7 @@ extern "C"{
}
#endif
extern my_bool xtrabackup_export;
extern char *xtrabackup_incremental_basedir;
extern char *xtrabackup_extra_lsndir;
extern char *xtrabackup_incremental_dir;
extern ulint xtrabackup_log_copy_interval;
extern char *xtrabackup_stream_str;
extern long xtrabackup_throttle;
@@ -166,14 +148,9 @@ void xtrabackup_io_throttling(void);
my_bool xb_write_delta_metadata(const char *filename,
const xb_delta_info_t *info);
datafiles_iter_t *datafiles_iter_new(fil_system_t *f_system);
fil_node_t *datafiles_iter_next(datafiles_iter_t *it);
void datafiles_iter_free(datafiles_iter_t *it);
/***********************************************************************
Reads the space flags from a given data file and returns the compressed
page size, or 0 if the space is not compressed. */
ulint xb_get_zip_size(pfs_os_file_t file);
/** @return the tablespace flags from a given data file
@retval ULINT_UNDEFINED if the file is not readable */
ulint xb_get_space_flags(pfs_os_file_t file);
/************************************************************************
Checks if a table specified as a name in the form "database/name" (InnoDB 5.6)
@@ -204,17 +181,10 @@ bool
check_if_param_set(const char *param);
#if defined(HAVE_OPENSSL)
extern my_bool opt_use_ssl;
extern my_bool opt_ssl_verify_server_cert;
#if !defined(HAVE_YASSL)
extern char *opt_server_public_key;
#endif
#endif
void
xtrabackup_backup_func(void);
my_bool
xb_get_one_option(int optid,
const struct my_option *opt __attribute__((unused)),

1
mysql-test/suite/mariabackup/bug1509812-master.opt
View File

@@ -1 +0,0 @@
--loose-skip-log-bin

2
mysql-test/suite/mariabackup/disabled.def Normal file
View File

@@ -0,0 +1,2 @@
partial : xtrabackup --export does not work
xb_page_compress : xtrabackup --export does not work

2
mysql-test/suite/mariabackup/full_backup.test
View File

@@ -1,3 +1,5 @@
--source include/innodb_page_size.inc
CREATE TABLE t(i INT) ENGINE INNODB;
INSERT INTO t VALUES(1);
echo # xtrabackup backup;

10
mysql-test/suite/mariabackup/include/restart_and_restore.inc
View File

@@ -1,15 +1,9 @@
let $_server_id= `SELECT @@server_id`;
let $_datadir= `SELECT @@datadir`;
let $_expect_file_name= $MYSQLTEST_VARDIR/tmp/mysqld.$_server_id.expect;
exec echo "wait" > $_expect_file_name;
echo # shutdown server;
shutdown_server;
--source include/shutdown_mysqld.inc
echo # remove datadir;
rmdir $_datadir;
echo # xtrabackup move back;
exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --copy-back --datadir=$_datadir --target-dir=$targetdir --parallel=2 --throttle=1;
echo # restart server;
exec echo "restart" > $_expect_file_name;
enable_reconnect;
source include/wait_until_connected_again.inc;
disable_reconnect;
--source include/start_mysqld.inc

7
mysql-test/suite/mariabackup/incremental_backup.test
View File

@@ -1,10 +1,10 @@
call mtr.add_suppression("InnoDB: New log files created");
--source include/innodb_page_size.inc
call mtr.add_suppression("InnoDB: New log files created");
let $basedir=$MYSQLTEST_VARDIR/tmp/backup;
let $incremental_dir=$MYSQLTEST_VARDIR/tmp/backup_inc1;
CREATE TABLE t(i INT) ENGINE INNODB;
INSERT INTO t VALUES(1);
@@ -18,7 +18,7 @@ exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir
--disable_result_log
echo # Prepare full backup, apply incremental one;
exec $XTRABACKUP --prepare --apply-log-only --target-dir=$basedir;
exec $XTRABACKUP --prepare --target-dir=$basedir;
exec $XTRABACKUP --prepare --target-dir=$basedir --incremental-dir=$incremental_dir ;
echo # Restore and check results;
@@ -32,4 +32,3 @@ DROP TABLE t;
# Cleanup
rmdir $basedir;
rmdir $incremental_dir;

4
mysql-test/suite/mariabackup/incremental_encrypted.test
View File

@@ -1,3 +1,5 @@
--source include/innodb_page_size.inc
if (!$EXAMPLE_KEY_MANAGEMENT_SO)
{
--skip needs example_key_management plugin
@@ -24,7 +26,7 @@ SELECT * FROM t;
--disable_result_log
exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$incremental_dir --incremental-basedir=$basedir;
echo # Prepare full backup, apply incremental one;
exec $XTRABACKUP --prepare --apply-log-only --target-dir=$basedir;
exec $XTRABACKUP --prepare --target-dir=$basedir;
exec $XTRABACKUP --prepare --target-dir=$basedir --incremental-dir=$incremental_dir;
echo # Restore and check results;

3
mysql-test/suite/mariabackup/partial.test
View File

@@ -1,3 +1,5 @@
#--source include/innodb_page_size.inc
# Export single table from backup
# (xtrabackup with --prepare --export)
@@ -28,4 +30,3 @@ SELECT * FROM t1;
DROP TABLE t1;
DROP TABLE t2;
rmdir $targetdir;

3
mysql-test/suite/mariabackup/partial_exclude.test
View File

@@ -1,3 +1,5 @@
#--source include/innodb_page_size.inc
# Test --databases-exclude and --tables-exclude feature of xtrabackup 2.3.8
CREATE TABLE t1(i INT) ENGINE INNODB;
@@ -27,4 +29,3 @@ DROP TABLE t1;
DROP TABLE t2;
DROP DATABASE db2;
rmdir $targetdir;

3
mysql-test/suite/mariabackup/small_ibd.test
View File

@@ -1,3 +1,5 @@
--source include/innodb_page_size.inc
# Check if ibd smaller than page size are skipped
# It is possible, due to race conditions that new file
# is created by server while xtrabackup is running
@@ -15,4 +17,3 @@ exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=
--enable_result_log
remove_file $_datadir/test/small.ibd;
rmdir $targetdir;

2
mysql-test/suite/mariabackup/suite.opt
View File

@@ -1 +1 @@
--innodb --changed_page_bitmaps --innodb-file-format=Barracuda
--innodb --loose-changed_page_bitmaps --innodb-file-format=Barracuda

2
mysql-test/suite/mariabackup/xb_aws_key_management.test
View File

@@ -1,3 +1,5 @@
--source include/innodb_page_size.inc
if (`SELECT COUNT(*)=0 FROM INFORMATION_SCHEMA.PLUGINS WHERE PLUGIN_NAME = 'aws_key_management' AND PLUGIN_STATUS='ACTIVE'`)
{
--skip needs aws_key_management plugin plugin

9
mysql-test/suite/mariabackup/xb_compressed_encrypted.test
View File

@@ -1,7 +1,8 @@
source include/have_file_key_management.inc;
#--source include/innodb_page_size_small.inc
--source include/have_file_key_management.inc
CREATE TABLE t1(c1 INT, b VARCHAR(2400), index(b(100),c1)) ENGINE=INNODB ROW_FORMAT=compressed ENCRYPTED=YES;
DELIMITER //;
CREATE PROCEDURE innodb_insert_proc (REPEAT_COUNT INT)
BEGIN
@@ -14,12 +15,12 @@ BEGIN
END//
DELIMITER ;//
COMMIT;
SET AUTOCOMMIT=0;
CALL innodb_insert_proc(50000);
COMMIT;
echo # xtrabackup backup;
--disable_result_log
let $targetdir=$MYSQLTEST_VARDIR/tmp/backup;

4
mysql-test/suite/mariabackup/xb_file_key_management.result
View File

@@ -1,7 +1,7 @@
CREATE TABLE t(c VARCHAR(10)) ENGINE INNODB encrypted=yes;
INSERT INTO t VALUES('foobar1');
# xtrabackup backup
NOT FOUND /foobar1/ in xtrabackup_logfile
NOT FOUND /foobar1/ in ib_logfile0
# expect NOT FOUND
INSERT INTO t VALUES('foobar2');
# xtrabackup prepare
@@ -9,8 +9,6 @@ INSERT INTO t VALUES('foobar2');
# remove datadir
# xtrabackup move back
# restart server
NOT FOUND /foobar1/ in xtrabackup_logfile
# expect NOT FOUND
SELECT * FROM t;
c
foobar1

17
mysql-test/suite/mariabackup/xb_file_key_management.test
View File

@@ -1,4 +1,5 @@
source include/have_file_key_management.inc;
#--source include/innodb_page_size.inc
--source include/have_file_key_management.inc
CREATE TABLE t(c VARCHAR(10)) ENGINE INNODB encrypted=yes;
INSERT INTO t VALUES('foobar1');
@@ -10,7 +11,7 @@ exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=
--let SEARCH_RANGE = 10000000
--let SEARCH_PATTERN=foobar1
--let SEARCH_FILE=$targetdir/xtrabackup_logfile
--let SEARCH_FILE=$targetdir/ib_logfile0
--source include/search_pattern_in_file.inc
--echo # expect NOT FOUND
@@ -22,18 +23,8 @@ exec $XTRABACKUP --prepare --target-dir=$targetdir;
-- source include/restart_and_restore.inc
--enable_result_log
#
# Recheck that plain text data (
# in not in the log, after prepare
# (MDEV-11538)
--let SEARCH_RANGE = 10000000
--let SEARCH_PATTERN=foobar1
--let SEARCH_FILE=$targetdir/xtrabackup_logfile
--source include/search_pattern_in_file.inc
--echo # expect NOT FOUND
--list_files $targetdir ib_logfile*
SELECT * FROM t;
DROP TABLE t;
rmdir $targetdir;

3
mysql-test/suite/mariabackup/xb_fulltext_encrypted.test
View File

@@ -1,4 +1,5 @@
source include/have_file_key_management.inc;
#--source include/innodb_page_size.inc
--source include/have_file_key_management.inc
CREATE TABLE film_text (
film_id SMALLINT NOT NULL,

2
mysql-test/suite/mariabackup/xb_history.test
View File

@@ -1,3 +1,5 @@
#--source include/innodb_page_size.inc
let $targetdir=$MYSQLTEST_VARDIR/tmp/backup;
--disable_result_log
exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --history=foo --backup --target-dir=$targetdir;

3
mysql-test/suite/mariabackup/xb_page_compress.test
View File

@@ -1,3 +1,5 @@
#--source include/innodb_page_size.inc
CREATE TABLE t1(c1 INT, b CHAR(20)) ENGINE=INNODB PAGE_COMPRESSED=1;
DELIMITER //;
@@ -41,4 +43,3 @@ SELECT COUNT(*) FROM t1;
DROP PROCEDURE innodb_insert_proc;
DROP TABLE t1;
rmdir $targetdir;

4
mysql-test/suite/mariabackup/xb_partition.test
View File

@@ -1,3 +1,4 @@
#--source include/innodb_page_size.inc
--source include/have_partition.inc
CREATE TABLE t1(a INT) ENGINE=InnoDB;
@@ -68,9 +69,8 @@ INSERT INTO isam_p VALUES (401), (501);
--disable_result_log
exec $INNOBACKUPEX --defaults-file=$MYSQLTEST_VARDIR/my.cnf --incremental --no-timestamp --incremental-basedir=$targetdir/full $targetdir/inc;
exec $INNOBACKUPEX --defaults-file=$MYSQLTEST_VARDIR/my.cnf --apply-log --redo-only $targetdir/full;
exec $INNOBACKUPEX --defaults-file=$MYSQLTEST_VARDIR/my.cnf --apply-log --redo-only --incremental-dir=$targetdir/inc $targetdir/full;
exec $INNOBACKUPEX --defaults-file=$MYSQLTEST_VARDIR/my.cnf --apply-log $targetdir/full;
exec $INNOBACKUPEX --defaults-file=$MYSQLTEST_VARDIR/my.cnf --apply-log --incremental-dir=$targetdir/inc $targetdir/full;
let $targetdir=$targetdir/full;
-- source include/restart_and_restore.inc

2
mysql-test/suite/mariabackup/xbstream.test
View File

@@ -1,3 +1,5 @@
--source include/innodb_page_size.inc
CREATE TABLE t(i INT) ENGINE INNODB;
INSERT INTO t VALUES(1);

2
scripts/wsrep_sst_mariabackup.sh
View File

@@ -841,7 +841,7 @@ then
rm -f "${DATA}/${IST_FILE}"
# May need xtrabackup_checkpoints later on
rm -f ${DATA}/xtrabackup_binary ${DATA}/xtrabackup_galera_info ${DATA}/xtrabackup_logfile
rm -f ${DATA}/xtrabackup_binary ${DATA}/xtrabackup_galera_info ${DATA}/ib_logfile0
ADDR=${WSREP_SST_OPT_ADDR}
if [ -z "${SST_PORT}" ]

2
storage/innobase/CMakeLists.txt
View File

@@ -188,3 +188,5 @@ IF(CMAKE_COMPILER_IS_GNUCXX AND CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64")
)
ENDIF()
ENDIF()
ADD_SUBDIRECTORY(${CMAKE_SOURCE_DIR}/extra/mariabackup ${CMAKE_BINARY_DIR}/extra/mariabackup)

11
storage/innobase/fil/fil0fil.cc
View File

@@ -326,13 +326,12 @@ fil_space_get_by_id(
return(space);
}
/*******************************************************************//**
Returns the table space by a given name, NULL if not found. */
UNIV_INLINE
/** Look up a tablespace.
@param[in] name tablespace name
@return tablespace
@retval NULL if not found */
fil_space_t*
fil_space_get_by_name(
/*==================*/
const char* name) /*!< in: space name */
fil_space_get_by_name(const char* name)
{
fil_space_t* space;
ulint fold;

2
storage/innobase/fsp/fsp0file.cc
View File

@@ -334,6 +334,8 @@ Datafile::read_first_page(bool read_only_mode)
break;
} else if (srv_operation == SRV_OPERATION_BACKUP) {
break;
} else {
ib::error()

4
storage/innobase/handler/ha_innodb.cc
View File

@@ -407,7 +407,7 @@ static TYPELIB innodb_stats_method_typelib = {
};
/** Possible values of the parameter innodb_checksum_algorithm */
static const char* innodb_checksum_algorithm_names[] = {
const char* innodb_checksum_algorithm_names[] = {
"crc32",
"strict_crc32",
"innodb",
@@ -419,7 +419,7 @@ static const char* innodb_checksum_algorithm_names[] = {
/** Used to define an enumerate type of the system variable
innodb_checksum_algorithm. */
static TYPELIB innodb_checksum_algorithm_typelib = {
TYPELIB innodb_checksum_algorithm_typelib = {
array_elements(innodb_checksum_algorithm_names) - 1,
"innodb_checksum_algorithm_typelib",
innodb_checksum_algorithm_names,

14
storage/innobase/include/fil0fil.h
View File

@@ -849,6 +849,13 @@ public:
return(m_space);
}
/** Member accessor
@return the wrapped object */
const fil_space_t* operator->() const
{
return(m_space);
}
/** Explicit type conversion
@return the wrapped object */
const fil_space_t* operator()() const
@@ -1526,6 +1533,13 @@ fil_space_get_by_id(
/*================*/
ulint id); /*!< in: space id */
/** Look up a tablespace.
@param[in] name tablespace name
@return tablespace
@retval NULL if not found */
fil_space_t*
fil_space_get_by_name(const char* name);
/*******************************************************************//**
by redo log.
@param[in,out] space tablespace */

9
storage/innobase/include/log0log.h
View File

@@ -2,7 +2,7 @@
Copyright (c) 1995, 2017, Oracle and/or its affiliates. All rights reserved.
Copyright (c) 2009, Google Inc.
Copyright (c) 2017, MariaDB Corporation. All Rights Reserved.
Copyright (c) 2017, MariaDB Corporation.
Portions of this file contain modifications contributed and copyrighted by
Google, Inc. Those modifications are gratefully acknowledged and are described
@@ -39,6 +39,7 @@ Created 12/9/1995 Heikki Tuuri
#include "sync0rw.h"
#include "log0types.h"
#include "os0event.h"
#include "os0file.h"
/** Redo log group */
struct log_group_t;
@@ -575,6 +576,12 @@ struct log_group_t{
{
return((format & LOG_HEADER_FORMAT_ENCRYPTED) != 0);
}
/** @return capacity in bytes */
inline lsn_t capacity() const
{
return((file_size - LOG_FILE_HDR_SIZE) * n_files);
}
};
/** Redo log buffer */

1
storage/innobase/include/log0log.ic
View File

@@ -24,7 +24,6 @@ Database log
Created 12/9/1995 Heikki Tuuri
*******************************************************/
#include "os0file.h"
#include "mach0data.h"
#include "srv0mon.h"
#include "srv0srv.h"

14
storage/innobase/include/log0recv.h
View File

@@ -92,6 +92,20 @@ Frees the recovery system. */
void
recv_sys_debug_free(void);
/*=====================*/
/** Read a log segment to a buffer.
@param[out] buf buffer
@param[in] group redo log files
@param[in] start_lsn read area start
@param[in] end_lsn read area end
@return valid end_lsn */
lsn_t
log_group_read_log_seg(
byte* buf,
const log_group_t* group,
lsn_t start_lsn,
lsn_t end_lsn);
/********************************************************//**
Reset the state of the recovery system variables. */
void

13
storage/innobase/include/srv0srv.h
View File

@@ -500,6 +500,19 @@ extern ulong srv_max_purge_lag_delay;
extern ulong srv_replication_delay;
/*-------------------------------------------*/
/** Modes of operation */
enum srv_operation_mode {
/** Normal mode (MariaDB Server) */
SRV_OPERATION_NORMAL,
/** Mariabackup taking a backup */
SRV_OPERATION_BACKUP,
/** Mariabackup restoring a backup */
SRV_OPERATION_RESTORE
};
/** Current mode of operation */
extern enum srv_operation_mode srv_operation;
extern my_bool srv_print_innodb_monitor;
extern my_bool srv_print_innodb_lock_monitor;
extern ibool srv_print_verbose_log;

9
storage/innobase/include/srv0start.h
View File

@@ -38,6 +38,12 @@ struct dict_table_t;
only one buffer pool instance is used. */
#define BUF_POOL_SIZE_THRESHOLD (1024 * 1024 * 1024)
/** Open the configured number of dedicated undo tablespaces.
@param[in] create_new_db whether the database is being initialized
@return DB_SUCCESS or error code */
dberr_t
srv_undo_tablespaces_init(bool create_new_db);
/****************************************************************//**
Starts Innobase and creates a new database if database files
are not found and the user wants.
@@ -127,4 +133,7 @@ extern bool srv_undo_sources;
/** At a shutdown this value climbs from SRV_SHUTDOWN_NONE to
SRV_SHUTDOWN_CLEANUP and then to SRV_SHUTDOWN_LAST_PHASE, and so on */
extern enum srv_shutdown_t srv_shutdown_state;
/** Files comprising the system tablespace */
extern pfs_os_file_t files[1000];
#endif

2
storage/innobase/include/univ.i
View File

@@ -618,7 +618,7 @@ Windows, so define a typedef for it and a macro to use at the end of such
functions. */
#ifdef _WIN32
typedef ulint os_thread_ret_t;
typedef DWORD os_thread_ret_t;
# define OS_THREAD_DUMMY_RETURN return(0)
# define OS_PATH_SEPARATOR '\\'
# define OS_PATH_SEPARATOR_ALT '/'

19
storage/innobase/log/log0log.cc
View File

@@ -548,23 +548,6 @@ function_exit:
return(lsn);
}
/******************************************************//**
Calculates the data capacity of a log group, when the log file headers are not
included.
@return capacity in bytes */
static
lsn_t
log_group_get_capacity(
/*===================*/
const log_group_t* group) /*!< in: log group */
{
/* The lsn parameters are updated while holding both the mutexes
and it is ok to have either of them while reading */
ut_ad(log_mutex_own() || log_write_mutex_own());
return((group->file_size - LOG_FILE_HDR_SIZE) * group->n_files);
}
/******************************************************//**
Calculates the offset within a log group, when the log file headers are not
included.
@@ -628,7 +611,7 @@ log_group_calc_lsn_offset(
gr_lsn_size_offset = log_group_calc_size_offset(
group->lsn_offset, group);
group_size = log_group_get_capacity(group);
group_size = group->capacity();
if (lsn >= gr_lsn) {

55
storage/innobase/log/log0recv.cc
View File

@@ -640,7 +640,6 @@ recv_sys_debug_free(void)
@param[in] start_lsn read area start
@param[in] end_lsn read area end
@return valid end_lsn */
static
lsn_t
log_group_read_log_seg(
byte* buf,
@@ -1931,27 +1930,29 @@ recv_read_in_area(
void
recv_apply_hashed_log_recs(bool last_batch)
{
for (;;) {
mutex_enter(&recv_sys->mutex);
ut_ad(srv_operation == SRV_OPERATION_NORMAL
|| srv_operation == SRV_OPERATION_RESTORE);
if (!recv_sys->apply_batch_on) {
break;
}
mutex_enter(&recv_sys->mutex);
if (recv_sys->found_corrupt_log) {
mutex_exit(&recv_sys->mutex);
while (recv_sys->apply_batch_on) {
bool abort = recv_sys->found_corrupt_log;
mutex_exit(&recv_sys->mutex);
if (abort) {
return;
}
mutex_exit(&recv_sys->mutex);
os_thread_sleep(500000);
mutex_enter(&recv_sys->mutex);
}
ut_ad(!last_batch == log_mutex_own());
if (!last_batch) {
recv_no_ibuf_operations = true;
}
recv_no_ibuf_operations = !last_batch
|| srv_operation == SRV_OPERATION_RESTORE;
ut_d(recv_no_log_write = recv_no_ibuf_operations);
if (ulint n = recv_sys->n_addrs) {
const char* msg = last_batch
@@ -2023,10 +2024,11 @@ recv_apply_hashed_log_recs(bool last_batch)
/* Wait until all the pages have been processed */
while (recv_sys->n_addrs != 0) {
bool abort = recv_sys->found_corrupt_log;
mutex_exit(&(recv_sys->mutex));
if (recv_sys->found_corrupt_log) {
if (abort) {
return;
}
@@ -2039,7 +2041,6 @@ recv_apply_hashed_log_recs(bool last_batch)
/* Flush all the file pages to disk and invalidate them in
the buffer pool */
ut_d(recv_no_log_write = true);
mutex_exit(&(recv_sys->mutex));
log_mutex_exit();
@@ -2062,9 +2063,6 @@ recv_apply_hashed_log_recs(bool last_batch)
log_mutex_enter();
mutex_enter(&(recv_sys->mutex));
ut_d(recv_no_log_write = false);
recv_no_ibuf_operations = false;
}
recv_sys->apply_log_recs = FALSE;
@@ -2414,6 +2412,13 @@ loop:
recv_sys->recovered_lsn);
}
/* fall through */
case MLOG_INDEX_LOAD:
/* Mariabackup FIXME: Report an error
when encountering MLOG_INDEX_LOAD on
--prepare or already on --backup. */
ut_a(type != MLOG_INDEX_LOAD
|| srv_operation == SRV_OPERATION_NORMAL);
/* fall through */
case MLOG_FILE_NAME:
case MLOG_FILE_DELETE:
case MLOG_FILE_CREATE2:
@@ -2422,7 +2427,6 @@ loop:
/* These were already handled by
recv_parse_log_rec() and
recv_parse_or_apply_log_rec_body(). */
case MLOG_INDEX_LOAD:
DBUG_PRINT("ib_log",
("scan " LSN_PF ": log rec %s"
" len " ULINTPF
@@ -2560,11 +2564,16 @@ loop:
for something else. */
break;
#endif /* UNIV_LOG_LSN_DEBUG */
case MLOG_INDEX_LOAD:
/* Mariabackup FIXME: Report an error
when encountering MLOG_INDEX_LOAD on
--prepare or already on --backup. */
ut_a(srv_operation == SRV_OPERATION_NORMAL);
break;
case MLOG_FILE_NAME:
case MLOG_FILE_DELETE:
case MLOG_FILE_CREATE2:
case MLOG_FILE_RENAME2:
case MLOG_INDEX_LOAD:
case MLOG_TRUNCATE:
/* These were already handled by
recv_parse_log_rec() and
@@ -2960,6 +2969,14 @@ static
dberr_t
recv_init_missing_space(dberr_t err, const recv_spaces_t::const_iterator& i)
{
if (srv_operation == SRV_OPERATION_RESTORE) {
ib::warn() << "Tablespace " << i->first << " was not"
" found at " << i->second.name << " when"
" restoring a (partial?) backup. All redo log"
" for this file will be ignored!";
return(err);
}
if (srv_force_recovery == 0) {
ib::error() << "Tablespace " << i->first << " was not"
" found at " << i->second.name << ".";

2
storage/innobase/mtr/mtr0mtr.cc
View File

@@ -557,7 +557,7 @@ mtr_t::commit()
m_impl.m_state = MTR_STATE_COMMITTING;
/* This is a dirty read, for debugging. */
ut_ad(!recv_no_log_write);
ut_ad(!m_impl.m_modifications || !recv_no_log_write);
Command cmd(this);

18
storage/innobase/os/os0file.cc
View File

@@ -3796,6 +3796,7 @@ os_file_create_simple_func(
ut_a(!(create_mode & OS_FILE_ON_ERROR_SILENT));
ut_a(!(create_mode & OS_FILE_ON_ERROR_NO_EXIT));
ut_ad(srv_operation == SRV_OPERATION_NORMAL);
if (create_mode == OS_FILE_OPEN) {
@@ -4119,7 +4120,9 @@ os_file_create_func(
);
DWORD create_flag;
DWORD share_mode = FILE_SHARE_READ;
DWORD share_mode = srv_operation != SRV_OPERATION_NORMAL
? FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE
: FILE_SHARE_READ;
if (create_mode != OS_FILE_OPEN && create_mode != OS_FILE_OPEN_RAW) {
WAIT_ALLOW_WRITES();
@@ -4131,8 +4134,7 @@ os_file_create_func(
on_error_silent = create_mode & OS_FILE_ON_ERROR_SILENT
? true : false;
create_mode &= ~OS_FILE_ON_ERROR_NO_EXIT;
create_mode &= ~OS_FILE_ON_ERROR_SILENT;
create_mode &= ~(OS_FILE_ON_ERROR_NO_EXIT | OS_FILE_ON_ERROR_SILENT);
if (create_mode == OS_FILE_OPEN_RAW) {
@@ -4325,7 +4327,9 @@ os_file_create_simple_no_error_handling_func(
DWORD access;
DWORD create_flag;
DWORD attributes = 0;
DWORD share_mode = FILE_SHARE_READ;
DWORD share_mode = srv_operation != SRV_OPERATION_NORMAL
? FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE
: FILE_SHARE_READ;
ut_a(name);
@@ -4374,7 +4378,8 @@ os_file_create_simple_no_error_handling_func(
/*!< A backup program has to give mysqld the maximum
freedom to do what it likes with the file */
share_mode |= FILE_SHARE_DELETE | FILE_SHARE_WRITE;
share_mode |= FILE_SHARE_DELETE | FILE_SHARE_WRITE
| FILE_SHARE_READ;
} else {
ib::error()
@@ -4676,7 +4681,8 @@ os_file_get_status_win32(
fh = CreateFile(
(LPCTSTR) path, // File to open
access,
0, // No sharing
FILE_SHARE_READ | FILE_SHARE_WRITE
| FILE_SHARE_DELETE, // Full sharing
NULL, // Default security
OPEN_EXISTING, // Existing file only
FILE_ATTRIBUTE_NORMAL, // Normal file

3
storage/innobase/srv/srv0srv.cc
View File

@@ -446,6 +446,9 @@ UNIV_INTERN uint srv_defragment_frequency;
@see innodb_defragment_frequency_update() */
UNIV_INTERN ulonglong srv_defragment_interval;
/** Current mode of operation */
UNIV_INTERN enum srv_operation_mode srv_operation;
/* Set the following to 0 if you want InnoDB to write messages on
stderr on startup/shutdown. Not enabled on the embedded server. */
ibool srv_print_verbose_log;

114
storage/innobase/srv/srv0start.cc
View File

@@ -184,7 +184,7 @@ SRV_SHUTDOWN_CLEANUP and then to SRV_SHUTDOWN_LAST_PHASE, and so on */
enum srv_shutdown_t srv_shutdown_state = SRV_SHUTDOWN_NONE;
/** Files comprising the system tablespace */
static pfs_os_file_t files[1000];
pfs_os_file_t files[1000];
/** io_handler_thread parameters for thread identification */
static ulint n[SRV_MAX_N_IO_THREADS + 6];
@@ -395,6 +395,29 @@ create_log_file(
/** Initial number of the first redo log file */
#define INIT_LOG_FILE0 (SRV_N_LOG_FILES_MAX + 1)
/** Delete all log files.
@param[in,out] logfilename buffer for log file name
@param[in] dirnamelen length of the directory path
@param[in] n_files number of files to delete */
static
void
delete_log_files(char* logfilename, size_t dirnamelen, unsigned n_files)
{
/* Remove any old log files. */
for (unsigned i = 0; i < n_files; i++) {
sprintf(logfilename + dirnamelen, "ib_logfile%u", i);
/* Ignore errors about non-existent files or files
that cannot be removed. The create_log_file() will
return an error when the file exists. */
#ifdef _WIN32
DeleteFile((LPCTSTR) logfilename);
#else
unlink(logfilename);
#endif
}
}
/*********************************************************************//**
Creates all log files.
@return DB_SUCCESS or error code */
@@ -414,24 +437,14 @@ create_log_files(
return(DB_READ_ONLY);
}
/* Remove any old log files. */
for (unsigned i = 0; i <= INIT_LOG_FILE0; i++) {
sprintf(logfilename + dirnamelen, "ib_logfile%u", i);
/* Crashing after deleting the first file should be
recoverable. The buffer pool was clean, and we can simply
create all log files from the scratch. */
DBUG_EXECUTE_IF("innodb_log_abort_6",
delete_log_files(logfilename, dirnamelen, 1);
return(DB_ERROR););
/* Ignore errors about non-existent files or files
that cannot be removed. The create_log_file() will
return an error when the file exists. */
#ifdef _WIN32
DeleteFile((LPCTSTR) logfilename);
#else
unlink(logfilename);
#endif
/* Crashing after deleting the first
file should be recoverable. The buffer
pool was clean, and we can simply create
all log files from the scratch. */
DBUG_EXECUTE_IF("innodb_log_abort_6", return(DB_ERROR););
}
delete_log_files(logfilename, dirnamelen, INIT_LOG_FILE0 + 1);
DBUG_PRINT("ib_log", ("After innodb_log_abort_6"));
ut_ad(!buf_pool_check_no_pending_io());
@@ -820,7 +833,6 @@ undo::undo_spaces_t undo::Truncate::s_fix_up_spaces;
/** Open the configured number of dedicated undo tablespaces.
@param[in] create_new_db whether the database is being initialized
@return DB_SUCCESS or error code */
static
dberr_t
srv_undo_tablespaces_init(bool create_new_db)
{
@@ -833,6 +845,7 @@ srv_undo_tablespaces_init(bool create_new_db)
srv_undo_tablespaces_open = 0;
ut_a(srv_undo_tablespaces <= TRX_SYS_N_RSEGS);
ut_a(!create_new_db || srv_operation == SRV_OPERATION_NORMAL);
memset(undo_tablespace_ids, 0x0, sizeof(undo_tablespace_ids));
@@ -878,7 +891,7 @@ srv_undo_tablespaces_init(bool create_new_db)
we build the undo_tablespace_ids ourselves since they don't
already exist. */
if (!create_new_db) {
if (!create_new_db && srv_operation == SRV_OPERATION_NORMAL) {
n_undo_tablespaces = trx_rseg_get_n_undo_tablespaces(
undo_tablespace_ids);
@@ -1289,8 +1302,15 @@ srv_shutdown_all_bg_threads()
}
}
if (!buf_page_cleaner_is_active && os_aio_all_slots_free()) {
os_aio_wake_all_threads_at_shutdown();
switch (srv_operation) {
case SRV_OPERATION_BACKUP:
break;
case SRV_OPERATION_NORMAL:
case SRV_OPERATION_RESTORE:
if (!buf_page_cleaner_is_active
&& os_aio_all_slots_free()) {
os_aio_wake_all_threads_at_shutdown();
}
}
if (!os_thread_count) {
@@ -1461,6 +1481,9 @@ innobase_start_or_create_for_mysql()
size_t dirnamelen;
unsigned i = 0;
ut_ad(srv_operation == SRV_OPERATION_NORMAL
|| srv_operation == SRV_OPERATION_RESTORE);
if (srv_force_recovery == SRV_FORCE_NO_LOG_REDO) {
srv_read_only_mode = true;
}
@@ -1960,6 +1983,10 @@ innobase_start_or_create_for_mysql()
if (err == DB_NOT_FOUND) {
if (i == 0) {
if (srv_operation
== SRV_OPERATION_RESTORE) {
return(DB_SUCCESS);
}
if (flushed_lsn
< static_cast<lsn_t>(1000)) {
ib::error()
@@ -2306,6 +2333,26 @@ files_checked:
recv_recovery_from_checkpoint_finish();
if (srv_operation == SRV_OPERATION_RESTORE) {
/* After applying the redo log from
SRV_OPERATION_BACKUP, flush the changes
to the data files and delete the log file.
No further change to InnoDB files is needed. */
ut_ad(!srv_force_recovery);
ut_ad(srv_n_log_files_found <= 1);
ut_ad(recv_no_log_write);
buf_flush_sync_all_buf_pools();
err = fil_write_flushed_lsn(log_get_lsn());
ut_ad(!buf_pool_check_no_pending_io());
fil_close_log_files(true);
log_group_close_all();
if (err == DB_SUCCESS) {
delete_log_files(logfilename, dirnamelen,
srv_n_log_files_found);
}
return(err);
}
/* Upgrade or resize or rebuild the redo logs before
generating any dirty pages, so that the old redo log
files will not be written to. */
@@ -2559,7 +2606,7 @@ files_checked:
return(srv_init_abort(err));
}
if (!srv_read_only_mode) {
if (!srv_read_only_mode && srv_operation == SRV_OPERATION_NORMAL) {
/* Initialize the innodb_temporary tablespace and keep
it open until shutdown. */
err = srv_open_tmp_tablespace(create_new_db);
@@ -2586,7 +2633,7 @@ files_checked:
srv_start_state_set(SRV_START_STATE_MASTER);
}
if (!srv_read_only_mode
if (!srv_read_only_mode && srv_operation == SRV_OPERATION_NORMAL
&& srv_force_recovery < SRV_FORCE_NO_BACKGROUND) {
srv_undo_sources = true;
/* Create the dict stats gathering thread */
@@ -2770,13 +2817,20 @@ innodb_shutdown()
ut_ad(!srv_running);
ut_ad(!srv_undo_sources);
/* Shut down the persistent files. */
logs_empty_and_mark_files_at_shutdown();
switch (srv_operation) {
case SRV_OPERATION_BACKUP:
case SRV_OPERATION_RESTORE:
fil_close_all_files();
break;
case SRV_OPERATION_NORMAL:
/* Shut down the persistent files. */
logs_empty_and_mark_files_at_shutdown();
if (ulint n_threads = srv_conc_get_active_threads()) {
ib::warn() << "Query counter shows "
<< n_threads << " queries still"
" inside InnoDB at shutdown";
if (ulint n_threads = srv_conc_get_active_threads()) {
ib::warn() << "Query counter shows "
<< n_threads << " queries still"
" inside InnoDB at shutdown";
}
}
/* Exit any remaining threads. */