diff --git a/extra/mariabackup/backup_copy.cc b/extra/mariabackup/backup_copy.cc index 306009e2139..156e0b20e7c 100644 --- a/extra/mariabackup/backup_copy.cc +++ b/extra/mariabackup/backup_copy.cc @@ -1774,7 +1774,7 @@ copy_back() const char *ext_list[] = {"backup-my.cnf", "xtrabackup_logfile", "xtrabackup_binary", "xtrabackup_binlog_info", "xtrabackup_checkpoints", ".qp", ".pmap", ".tmp", - ".xbcrypt", NULL}; + NULL}; const char *filename; char c_tmp; int i_tmp; @@ -1807,7 +1807,7 @@ copy_back() filename = base_name(node.filepath); - /* skip .qp and .xbcrypt files */ + /* skip .qp files */ if (filename_matches(filename, ext_list)) { continue; } @@ -1899,24 +1899,8 @@ decrypt_decompress_file(const char *filepath, uint thread_n) cmd << IF_WIN("type ","cat ") << filepath; - if (ends_with(filepath, ".xbcrypt") && opt_decrypt) { - cmd << " | xbcrypt --decrypt --encrypt-algo=" - << xtrabackup_encrypt_algo_names[opt_decrypt_algo]; - if (xtrabackup_encrypt_key) { - cmd << " --encrypt-key=" << xtrabackup_encrypt_key; - } else { - cmd << " --encrypt-key-file=" - << xtrabackup_encrypt_key_file; - } - dest_filepath[strlen(dest_filepath) - 8] = 0; - message << "decrypting"; - needs_action = true; - } - if (opt_decompress - && (ends_with(filepath, ".qp") - || (ends_with(filepath, ".qp.xbcrypt") - && opt_decrypt))) { + && ends_with(filepath, ".qp")) { cmd << " | qpress -dio "; dest_filepath[strlen(dest_filepath) - 3] = 0; if (needs_action) { @@ -1967,8 +1951,7 @@ decrypt_decompress_thread_func(void *arg) continue; } - if (!ends_with(node.filepath, ".qp") - && !ends_with(node.filepath, ".xbcrypt")) { + if (!ends_with(node.filepath, ".qp")) { continue; } diff --git a/extra/mariabackup/backup_mysql.cc b/extra/mariabackup/backup_mysql.cc index 6299afffc6e..2353c7692cb 100644 --- a/extra/mariabackup/backup_mysql.cc +++ b/extra/mariabackup/backup_mysql.cc @@ -1435,9 +1435,7 @@ write_xtrabackup_info(MYSQL *connection) "partial = %s\n" "incremental = %s\n" "format = %s\n" - "compact = %s\n" - "compressed = %s\n" - "encrypted = %s\n", + "compressed = %s\n", uuid, /* uuid */ opt_history ? opt_history : "", /* name */ tool_name, /* tool_name */ @@ -1455,9 +1453,7 @@ write_xtrabackup_info(MYSQL *connection) is_partial? "Y" : "N", xtrabackup_incremental ? "Y" : "N", /* incremental */ xb_stream_name[xtrabackup_stream_fmt], /* format */ - "N", /* compact */ - xtrabackup_compress ? "compressed" : "N", /* compressed */ - xtrabackup_encrypt ? "Y" : "N"); /* encrypted */ + xtrabackup_compress ? "compressed" : "N"); /* compressed */ if (!opt_history) { goto cleanup; @@ -1483,9 +1479,7 @@ write_xtrabackup_info(MYSQL *connection) "partial ENUM('Y', 'N') DEFAULT NULL," "incremental ENUM('Y', 'N') DEFAULT NULL," "format ENUM('file', 'tar', 'xbstream') DEFAULT NULL," - "compact ENUM('Y', 'N') DEFAULT NULL," - "compressed ENUM('Y', 'N') DEFAULT NULL," - "encrypted ENUM('Y', 'N') DEFAULT NULL" + "compressed ENUM('Y', 'N') DEFAULT NULL" ") CHARACTER SET utf8 ENGINE=innodb", false); @@ -1495,8 +1489,8 @@ write_xtrabackup_info(MYSQL *connection) << "uuid, name, tool_name, tool_command, tool_version," << "ibbackup_version, server_version, start_time, end_time," << "lock_time, binlog_pos, innodb_from_lsn, innodb_to_lsn," - << "partial, incremental, format, compact, compressed, " - << "encrypted) values(" + << "partial, incremental, format, compressed) " + << "values(" << escape_and_quote(connection, uuid) << "," << escape_and_quote(connection, opt_history) << "," << escape_and_quote(connection, tool_name) << "," @@ -1513,9 +1507,7 @@ write_xtrabackup_info(MYSQL *connection) << ESCAPE_BOOL(is_partial) << "," << ESCAPE_BOOL(xtrabackup_incremental)<< "," << escape_and_quote(connection,xb_stream_name[xtrabackup_stream_fmt]) <<"," - << ESCAPE_BOOL(false) << "," - << ESCAPE_BOOL(xtrabackup_compress) << "," - << ESCAPE_BOOL(xtrabackup_encrypt) <<")"; + << ESCAPE_BOOL(xtrabackup_compress) << ")"; xb_mysql_query(mysql_connection, oss.str().c_str(), false); @@ -1581,14 +1573,6 @@ char *make_argv(char *buf, size_t len, int argc, char **argv) if (strncmp(*argv, "--password", strlen("--password")) == 0) { arg = "--password=..."; } - if (strncmp(*argv, "--encrypt-key", - strlen("--encrypt-key")) == 0) { - arg = "--encrypt-key=..."; - } - if (strncmp(*argv, "--encrypt_key", - strlen("--encrypt_key")) == 0) { - arg = "--encrypt_key=..."; - } left-= ut_snprintf(buf + len - left, left, "%s%c", arg, argc > 1 ? ' ' : 0); ++argv; --argc; diff --git a/extra/mariabackup/ds_decrypt.c b/extra/mariabackup/ds_decrypt.c deleted file mode 100644 index e897ca101e5..00000000000 --- a/extra/mariabackup/ds_decrypt.c +++ /dev/null @@ -1,665 +0,0 @@ -/****************************************************** -Copyright (c) 2017 Percona LLC and/or its affiliates. - -Encryption datasink implementation for XtraBackup. - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - -*******************************************************/ - - -#include -#include "common.h" -#include "datasink.h" -#include "xbcrypt.h" -#include "xbcrypt_common.h" -#include "crc_glue.h" - -typedef struct { - pthread_t id; - uint num; - pthread_mutex_t ctrl_mutex; - pthread_cond_t ctrl_cond; - pthread_mutex_t data_mutex; - pthread_cond_t data_cond; - my_bool started; - my_bool data_avail; - my_bool cancelled; - my_bool failed; - const uchar *from; - size_t from_len; - uchar *to; - size_t to_len; - size_t to_size; - const uchar *iv; - size_t iv_len; - unsigned long long offset; - my_bool hash_appended; - gcry_cipher_hd_t cipher_handle; - xb_rcrypt_result_t parse_result; -} crypt_thread_ctxt_t; - -typedef struct { - crypt_thread_ctxt_t *threads; - uint nthreads; - int encrypt_algo; - size_t chunk_size; - char *encrypt_key; - char *encrypt_key_file; -} ds_decrypt_ctxt_t; - -typedef struct { - ds_decrypt_ctxt_t *crypt_ctxt; - size_t bytes_processed; - ds_file_t *dest_file; - uchar *buf; - size_t buf_len; - size_t buf_size; -} ds_decrypt_file_t; - -int ds_decrypt_encrypt_threads = 1; - -static ds_ctxt_t *decrypt_init(const char *root); -static ds_file_t *decrypt_open(ds_ctxt_t *ctxt, const char *path, - MY_STAT *mystat); -static int decrypt_write(ds_file_t *file, const void *buf, size_t len); -static int decrypt_close(ds_file_t *file); -static void decrypt_deinit(ds_ctxt_t *ctxt); - -datasink_t datasink_decrypt = { - &decrypt_init, - &decrypt_open, - &decrypt_write, - &decrypt_close, - &decrypt_deinit -}; - -static crypt_thread_ctxt_t *create_worker_threads(uint n); -static void destroy_worker_threads(crypt_thread_ctxt_t *threads, uint n); -static void *decrypt_worker_thread_func(void *arg); - -static -ds_ctxt_t * -decrypt_init(const char *root) -{ - ds_ctxt_t *ctxt; - ds_decrypt_ctxt_t *decrypt_ctxt; - crypt_thread_ctxt_t *threads; - - if (xb_crypt_init(NULL)) { - return NULL; - } - - /* Create and initialize the worker threads */ - threads = create_worker_threads(ds_decrypt_encrypt_threads); - if (threads == NULL) { - msg("decrypt: failed to create worker threads.\n"); - return NULL; - } - - ctxt = (ds_ctxt_t *) my_malloc(sizeof(ds_ctxt_t) + - sizeof(ds_decrypt_ctxt_t), - MYF(MY_FAE)); - - decrypt_ctxt = (ds_decrypt_ctxt_t *) (ctxt + 1); - decrypt_ctxt->threads = threads; - decrypt_ctxt->nthreads = ds_decrypt_encrypt_threads; - - ctxt->ptr = decrypt_ctxt; - ctxt->root = my_strdup(root, MYF(MY_FAE)); - - return ctxt; -} - -static -ds_file_t * -decrypt_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *mystat) -{ - ds_ctxt_t *dest_ctxt; - - ds_decrypt_ctxt_t *crypt_ctxt; - ds_decrypt_file_t *crypt_file; - - char new_name[FN_REFLEN]; - ds_file_t *file; - - xb_ad(ctxt->pipe_ctxt != NULL); - dest_ctxt = ctxt->pipe_ctxt; - - crypt_ctxt = (ds_decrypt_ctxt_t *) ctxt->ptr; - - - file = (ds_file_t *) my_malloc(sizeof(ds_file_t) + - sizeof(ds_decrypt_file_t), - MYF(MY_FAE|MY_ZEROFILL)); - - crypt_file = (ds_decrypt_file_t *) (file + 1); - - /* Remove the .xbcrypt extension from the filename */ - strncpy(new_name, path, FN_REFLEN); - new_name[strlen(new_name) - 8] = 0; - crypt_file->dest_file = ds_open(dest_ctxt, new_name, mystat); - if (crypt_file->dest_file == NULL) { - msg("decrypt: ds_open(\"%s\") failed.\n", new_name); - goto err; - } - - crypt_file->crypt_ctxt = crypt_ctxt; - crypt_file->buf = NULL; - crypt_file->buf_size = 0; - crypt_file->buf_len = 0; - - file->ptr = crypt_file; - file->path = crypt_file->dest_file->path; - - return file; - -err: - if (crypt_file->dest_file) { - ds_close(crypt_file->dest_file); - } - my_free(file); - return NULL; -} - -#define CHECK_BUF_SIZE(ptr, size, buf, len) \ - if (ptr + size - buf > (ssize_t) len) { \ - result = XB_CRYPT_READ_INCOMPLETE; \ - goto exit; \ - } - -static -xb_rcrypt_result_t -parse_xbcrypt_chunk(crypt_thread_ctxt_t *thd, const uchar *buf, size_t len, - size_t *bytes_processed) -{ - const uchar *ptr; - uint version; - ulong checksum, checksum_exp; - ulonglong tmp; - xb_rcrypt_result_t result = XB_CRYPT_READ_CHUNK; - - *bytes_processed = 0; - ptr = buf; - - CHECK_BUF_SIZE(ptr, XB_CRYPT_CHUNK_MAGIC_SIZE, buf, len); - if (memcmp(ptr, XB_CRYPT_CHUNK_MAGIC3, - XB_CRYPT_CHUNK_MAGIC_SIZE) == 0) { - version = 3; - } else if (memcmp(ptr, XB_CRYPT_CHUNK_MAGIC2, - XB_CRYPT_CHUNK_MAGIC_SIZE) == 0) { - version = 2; - } else if (memcmp(ptr, XB_CRYPT_CHUNK_MAGIC1, - XB_CRYPT_CHUNK_MAGIC_SIZE) == 0) { - version = 1; - } else { - msg("%s:%s: wrong chunk magic at offset 0x%llx.\n", - my_progname, __FUNCTION__, thd->offset); - result = XB_CRYPT_READ_ERROR; - goto exit; - } - - ptr += XB_CRYPT_CHUNK_MAGIC_SIZE; - thd->offset += XB_CRYPT_CHUNK_MAGIC_SIZE; - - CHECK_BUF_SIZE(ptr, 8, buf, len); - tmp = uint8korr(ptr); /* reserved */ - ptr += 8; - thd->offset += 8; - - CHECK_BUF_SIZE(ptr, 8, buf, len); - tmp = uint8korr(ptr); /* original size */ - ptr += 8; - if (tmp > INT_MAX) { - msg("%s:%s: invalid original size at offset 0x%llx.\n", - my_progname, __FUNCTION__, thd->offset); - result = XB_CRYPT_READ_ERROR; - goto exit; - } - thd->offset += 8; - thd->to_len = (size_t)tmp; - - if (thd->to_size < thd->to_len + XB_CRYPT_HASH_LEN) { - thd->to = (uchar *) my_realloc( - thd->to, - thd->to_len + XB_CRYPT_HASH_LEN, - MYF(MY_FAE | MY_ALLOW_ZERO_PTR)); - thd->to_size = thd->to_len; - } - - CHECK_BUF_SIZE(ptr, 8, buf, len); - tmp = uint8korr(ptr); /* encrypted size */ - ptr += 8; - if (tmp > INT_MAX) { - msg("%s:%s: invalid encrypted size at offset 0x%llx.\n", - my_progname, __FUNCTION__, thd->offset); - result = XB_CRYPT_READ_ERROR; - goto exit; - } - thd->offset += 8; - thd->from_len = (size_t)tmp; - - xb_a(thd->from_len <= thd->to_len + XB_CRYPT_HASH_LEN); - - CHECK_BUF_SIZE(ptr, 4, buf, len); - checksum_exp = uint4korr(ptr); /* checksum */ - ptr += 4; - thd->offset += 4; - - /* iv size */ - if (version == 1) { - thd->iv_len = 0; - thd->iv = NULL; - } else { - CHECK_BUF_SIZE(ptr, 8, buf, len); - - tmp = uint8korr(ptr); - if (tmp > INT_MAX) { - msg("%s:%s: invalid iv size at offset 0x%llx.\n", - my_progname, __FUNCTION__, thd->offset); - result = XB_CRYPT_READ_ERROR; - goto exit; - } - ptr += 8; - thd->offset += 8; - thd->iv_len = (size_t)tmp; - } - - if (thd->iv_len > 0) { - CHECK_BUF_SIZE(ptr, thd->iv_len, buf, len); - thd->iv = ptr; - ptr += thd->iv_len; - } - - /* for version euqals 2 we need to read in the iv data but do not init - CTR with it */ - if (version == 2) { - thd->iv_len = 0; - thd->iv = 0; - } - - if (thd->from_len > 0) { - CHECK_BUF_SIZE(ptr, thd->from_len, buf, len); - thd->from = ptr; - ptr += thd->from_len; - } - - xb_ad(thd->from_len <= thd->to_len); - - checksum = crc32_iso3309(0, thd->from, thd->from_len); - if (checksum != checksum_exp) { - msg("%s:%s invalid checksum at offset 0x%llx, " - "expected 0x%lx, actual 0x%lx.\n", my_progname, - __FUNCTION__, thd->offset, checksum_exp, checksum); - result = XB_CRYPT_READ_ERROR; - goto exit; - } - - thd->offset += thd->from_len; - - thd->hash_appended = version > 2; - -exit: - - *bytes_processed = (size_t) (ptr - buf); - - return result; -} - -static -int -decrypt_write(ds_file_t *file, const void *buf, size_t len) -{ - ds_decrypt_file_t *crypt_file; - ds_decrypt_ctxt_t *crypt_ctxt; - crypt_thread_ctxt_t *threads; - crypt_thread_ctxt_t *thd; - uint nthreads; - uint i; - size_t bytes_processed; - xb_rcrypt_result_t parse_result = XB_CRYPT_READ_CHUNK; - my_bool err = FALSE; - - crypt_file = (ds_decrypt_file_t *) file->ptr; - crypt_ctxt = crypt_file->crypt_ctxt; - - threads = crypt_ctxt->threads; - nthreads = crypt_ctxt->nthreads; - - if (crypt_file->buf_len > 0) { - thd = threads; - - pthread_mutex_lock(&thd->ctrl_mutex); - - do { - if (parse_result == XB_CRYPT_READ_INCOMPLETE) { - crypt_file->buf_size = crypt_file->buf_size * 2; - crypt_file->buf = (uchar *) my_realloc( - crypt_file->buf, - crypt_file->buf_size, - MYF(MY_FAE|MY_ALLOW_ZERO_PTR)); - } - - memcpy(crypt_file->buf + crypt_file->buf_len, - buf, MY_MIN(crypt_file->buf_size - - crypt_file->buf_len, len)); - - parse_result = parse_xbcrypt_chunk( - thd, crypt_file->buf, - crypt_file->buf_size, &bytes_processed); - - if (parse_result == XB_CRYPT_READ_ERROR) { - pthread_mutex_unlock(&thd->ctrl_mutex); - return 1; - } - - } while (parse_result == XB_CRYPT_READ_INCOMPLETE && - crypt_file->buf_size < len); - - if (parse_result != XB_CRYPT_READ_CHUNK) { - msg("decrypt: incomplete data.\n"); - pthread_mutex_unlock(&thd->ctrl_mutex); - return 1; - } - - pthread_mutex_lock(&thd->data_mutex); - thd->data_avail = TRUE; - pthread_cond_signal(&thd->data_cond); - pthread_mutex_unlock(&thd->data_mutex); - - len -= bytes_processed - crypt_file->buf_len; - buf += bytes_processed - crypt_file->buf_len; - - /* reap */ - - pthread_mutex_lock(&thd->data_mutex); - while (thd->data_avail == TRUE) { - pthread_cond_wait(&thd->data_cond, - &thd->data_mutex); - } - - if (thd->failed) { - msg("decrypt: failed to decrypt chunk.\n"); - err = TRUE; - } - - xb_a(thd->to_len > 0); - - if (!err && - ds_write(crypt_file->dest_file, thd->to, thd->to_len)) { - msg("decrypt: write to destination failed.\n"); - err = TRUE; - } - - crypt_file->bytes_processed += thd->from_len; - - pthread_mutex_unlock(&thd->data_mutex); - pthread_mutex_unlock(&thd->ctrl_mutex); - - crypt_file->buf_len = 0; - - if (err) { - return 1; - } - } - - while (parse_result == XB_CRYPT_READ_CHUNK && len > 0) { - uint max_thread; - - for (i = 0; i < nthreads; i++) { - thd = threads + i; - - pthread_mutex_lock(&thd->ctrl_mutex); - - parse_result = parse_xbcrypt_chunk( - thd, buf, len, &bytes_processed); - - if (parse_result == XB_CRYPT_READ_ERROR) { - pthread_mutex_unlock(&thd->ctrl_mutex); - err = TRUE; - break; - } - - thd->parse_result = parse_result; - - if (parse_result != XB_CRYPT_READ_CHUNK) { - pthread_mutex_unlock(&thd->ctrl_mutex); - break; - } - - pthread_mutex_lock(&thd->data_mutex); - thd->data_avail = TRUE; - pthread_cond_signal(&thd->data_cond); - pthread_mutex_unlock(&thd->data_mutex); - - len -= bytes_processed; - buf += bytes_processed; - } - - max_thread = (i < nthreads) ? i : nthreads - 1; - - /* Reap and write decrypted data */ - for (i = 0; i <= max_thread; i++) { - thd = threads + i; - - if (thd->parse_result != XB_CRYPT_READ_CHUNK) { - break; - } - - pthread_mutex_lock(&thd->data_mutex); - while (thd->data_avail == TRUE) { - pthread_cond_wait(&thd->data_cond, - &thd->data_mutex); - } - - if (thd->failed) { - msg("decrypt: failed to decrypt chunk.\n"); - err = TRUE; - } - - xb_a(thd->to_len > 0); - - if (!err && ds_write(crypt_file->dest_file, thd->to, - thd->to_len)) { - msg("decrypt: write to destination failed.\n"); - err = TRUE; - } - - crypt_file->bytes_processed += thd->from_len; - - pthread_mutex_unlock(&thd->data_mutex); - pthread_mutex_unlock(&thd->ctrl_mutex); - } - - if (err) { - return 1; - } - } - - if (parse_result == XB_CRYPT_READ_INCOMPLETE && len > 0) { - crypt_file->buf_len = len; - if (crypt_file->buf_size < len) { - crypt_file->buf = (uchar *) my_realloc( - crypt_file->buf, - crypt_file->buf_len, - MYF(MY_FAE | MY_ALLOW_ZERO_PTR)); - crypt_file->buf_size = len; - } - memcpy(crypt_file->buf, buf, len); - } - - return 0; -} - -static -int -decrypt_close(ds_file_t *file) -{ - ds_decrypt_file_t *crypt_file; - ds_file_t *dest_file; - int rc = 0; - - crypt_file = (ds_decrypt_file_t *) file->ptr; - dest_file = crypt_file->dest_file; - - if (ds_close(dest_file)) { - rc = 1; - } - - my_free(crypt_file->buf); - my_free(file); - - return rc; -} - -static -void -decrypt_deinit(ds_ctxt_t *ctxt) -{ - ds_decrypt_ctxt_t *crypt_ctxt; - - xb_ad(ctxt->pipe_ctxt != NULL); - - crypt_ctxt = (ds_decrypt_ctxt_t *) ctxt->ptr; - - destroy_worker_threads(crypt_ctxt->threads, crypt_ctxt->nthreads); - - my_free(ctxt->root); - my_free(ctxt); -} - -static -crypt_thread_ctxt_t * -create_worker_threads(uint n) -{ - crypt_thread_ctxt_t *threads; - uint i; - - threads = (crypt_thread_ctxt_t *) - my_malloc(sizeof(crypt_thread_ctxt_t) * n, - MYF(MY_FAE | MY_ZEROFILL)); - - for (i = 0; i < n; i++) { - crypt_thread_ctxt_t *thd = threads + i; - - thd->num = i + 1; - - /* Initialize the control mutex and condition var */ - if (pthread_mutex_init(&thd->ctrl_mutex, NULL) || - pthread_cond_init(&thd->ctrl_cond, NULL)) { - goto err; - } - - /* Initialize and data mutex and condition var */ - if (pthread_mutex_init(&thd->data_mutex, NULL) || - pthread_cond_init(&thd->data_cond, NULL)) { - goto err; - } - - xb_crypt_cipher_open(&thd->cipher_handle); - - pthread_mutex_lock(&thd->ctrl_mutex); - - if (pthread_create(&thd->id, NULL, decrypt_worker_thread_func, - thd)) { - msg("decrypt: pthread_create() failed: " - "errno = %d\n", errno); - goto err; - } - } - - /* Wait for the threads to start */ - for (i = 0; i < n; i++) { - crypt_thread_ctxt_t *thd = threads + i; - - while (thd->started == FALSE) - pthread_cond_wait(&thd->ctrl_cond, &thd->ctrl_mutex); - pthread_mutex_unlock(&thd->ctrl_mutex); - } - - return threads; - -err: - return NULL; -} - -static -void -destroy_worker_threads(crypt_thread_ctxt_t *threads, uint n) -{ - uint i; - - for (i = 0; i < n; i++) { - crypt_thread_ctxt_t *thd = threads + i; - - pthread_mutex_lock(&thd->data_mutex); - threads[i].cancelled = TRUE; - pthread_cond_signal(&thd->data_cond); - pthread_mutex_unlock(&thd->data_mutex); - - pthread_join(thd->id, NULL); - - pthread_cond_destroy(&thd->data_cond); - pthread_mutex_destroy(&thd->data_mutex); - pthread_cond_destroy(&thd->ctrl_cond); - pthread_mutex_destroy(&thd->ctrl_mutex); - - xb_crypt_cipher_close(thd->cipher_handle); - - my_free(thd->to); - } - - my_free(threads); -} - -static -void * -decrypt_worker_thread_func(void *arg) -{ - crypt_thread_ctxt_t *thd = (crypt_thread_ctxt_t *) arg; - - pthread_mutex_lock(&thd->ctrl_mutex); - - pthread_mutex_lock(&thd->data_mutex); - - thd->started = TRUE; - pthread_cond_signal(&thd->ctrl_cond); - - pthread_mutex_unlock(&thd->ctrl_mutex); - - while (1) { - thd->data_avail = FALSE; - pthread_cond_signal(&thd->data_cond); - - while (!thd->data_avail && !thd->cancelled) { - pthread_cond_wait(&thd->data_cond, &thd->data_mutex); - } - - if (thd->cancelled) - break; - - if (xb_crypt_decrypt(thd->cipher_handle, thd->from, - thd->from_len, thd->to, &thd->to_len, - thd->iv, thd->iv_len, - thd->hash_appended)) { - thd->failed = TRUE; - continue; - } - - } - - pthread_mutex_unlock(&thd->data_mutex); - - return NULL; -} diff --git a/extra/mariabackup/ds_decrypt.h b/extra/mariabackup/ds_decrypt.h deleted file mode 100644 index 3bb4de55f54..00000000000 --- a/extra/mariabackup/ds_decrypt.h +++ /dev/null @@ -1,30 +0,0 @@ -/****************************************************** -Copyright (c) 2017 Percona LLC and/or its affiliates. - -Encryption interface for XtraBackup. - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - -*******************************************************/ - -#ifndef DS_DECRYPT_H -#define DS_DECRYPT_H - -#include "datasink.h" - -extern datasink_t datasink_decrypt; - -extern int ds_decrypt_encrypt_threads; - -#endif diff --git a/extra/mariabackup/ds_encrypt.c b/extra/mariabackup/ds_encrypt.c deleted file mode 100644 index 576ea207eb1..00000000000 --- a/extra/mariabackup/ds_encrypt.c +++ /dev/null @@ -1,446 +0,0 @@ -/****************************************************** -Copyright (c) 2013 Percona LLC and/or its affiliates. - -Encryption datasink implementation for XtraBackup. - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - -*******************************************************/ - - -#include -#include "common.h" -#include "datasink.h" -#include "xbcrypt_common.h" -#ifdef HAVE_GRYPT -#include "xbcrypt.h" - -#define XB_CRYPT_CHUNK_SIZE ((size_t) (ds_encrypt_encrypt_chunk_size)) - -typedef struct { - pthread_t id; - uint num; - pthread_mutex_t ctrl_mutex; - pthread_cond_t ctrl_cond; - pthread_mutex_t data_mutex; - pthread_cond_t data_cond; - my_bool started; - my_bool data_avail; - my_bool cancelled; - const uchar *from; - size_t from_len; - uchar *to; - uchar *iv; - size_t to_len; - gcry_cipher_hd_t cipher_handle; -} crypt_thread_ctxt_t; - -typedef struct { - crypt_thread_ctxt_t *threads; - uint nthreads; -} ds_encrypt_ctxt_t; - -typedef struct { - xb_wcrypt_t *xbcrypt_file; - ds_encrypt_ctxt_t *crypt_ctxt; - size_t bytes_processed; - ds_file_t *dest_file; -} ds_encrypt_file_t; - -/* Encryption options */ -uint ds_encrypt_encrypt_threads; -ulonglong ds_encrypt_encrypt_chunk_size; - -static ds_ctxt_t *encrypt_init(const char *root); -static ds_file_t *encrypt_open(ds_ctxt_t *ctxt, const char *path, - MY_STAT *mystat); -static int encrypt_write(ds_file_t *file, const void *buf, size_t len); -static int encrypt_close(ds_file_t *file); -static void encrypt_deinit(ds_ctxt_t *ctxt); - -datasink_t datasink_encrypt = { - &encrypt_init, - &encrypt_open, - &encrypt_write, - &encrypt_close, - &encrypt_deinit -}; - -static crypt_thread_ctxt_t *create_worker_threads(uint n); -static void destroy_worker_threads(crypt_thread_ctxt_t *threads, uint n); -static void *encrypt_worker_thread_func(void *arg); - -static uint encrypt_iv_len = 0; - -static -ssize_t -my_xb_crypt_write_callback(void *userdata, const void *buf, size_t len) -{ - ds_encrypt_file_t *encrypt_file; - - encrypt_file = (ds_encrypt_file_t *) userdata; - - xb_ad(encrypt_file != NULL); - xb_ad(encrypt_file->dest_file != NULL); - - if (!ds_write(encrypt_file->dest_file, buf, len)) { - return len; - } - return -1; -} - -static -ds_ctxt_t * -encrypt_init(const char *root) -{ - ds_ctxt_t *ctxt; - ds_encrypt_ctxt_t *encrypt_ctxt; - crypt_thread_ctxt_t *threads; - - if (xb_crypt_init(&encrypt_iv_len)) { - return NULL; - } - - /* Create and initialize the worker threads */ - threads = create_worker_threads(ds_encrypt_encrypt_threads); - if (threads == NULL) { - msg("encrypt: failed to create worker threads.\n"); - return NULL; - } - - ctxt = (ds_ctxt_t *) my_malloc(sizeof(ds_ctxt_t) + - sizeof(ds_encrypt_ctxt_t), - MYF(MY_FAE)); - - encrypt_ctxt = (ds_encrypt_ctxt_t *) (ctxt + 1); - encrypt_ctxt->threads = threads; - encrypt_ctxt->nthreads = ds_encrypt_encrypt_threads; - - ctxt->ptr = encrypt_ctxt; - ctxt->root = my_strdup(root, MYF(MY_FAE)); - - return ctxt; -} - -static -ds_file_t * -encrypt_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *mystat) -{ - ds_ctxt_t *dest_ctxt; - - ds_encrypt_ctxt_t *crypt_ctxt; - ds_encrypt_file_t *crypt_file; - - char new_name[FN_REFLEN]; - ds_file_t *file; - - xb_ad(ctxt->pipe_ctxt != NULL); - dest_ctxt = ctxt->pipe_ctxt; - - crypt_ctxt = (ds_encrypt_ctxt_t *) ctxt->ptr; - - - file = (ds_file_t *) my_malloc(sizeof(ds_file_t) + - sizeof(ds_encrypt_file_t), - MYF(MY_FAE|MY_ZEROFILL)); - - crypt_file = (ds_encrypt_file_t *) (file + 1); - - /* Append the .xbcrypt extension to the filename */ - fn_format(new_name, path, "", ".xbcrypt", MYF(MY_APPEND_EXT)); - crypt_file->dest_file = ds_open(dest_ctxt, new_name, mystat); - if (crypt_file->dest_file == NULL) { - msg("encrypt: ds_open(\"%s\") failed.\n", new_name); - goto err; - } - - crypt_file->crypt_ctxt = crypt_ctxt; - crypt_file->xbcrypt_file = xb_crypt_write_open(crypt_file, - my_xb_crypt_write_callback); - - if (crypt_file->xbcrypt_file == NULL) { - msg("encrypt: xb_crypt_write_open() failed.\n"); - goto err; - } - - - file->ptr = crypt_file; - file->path = crypt_file->dest_file->path; - - return file; - -err: - if (crypt_file->dest_file) { - ds_close(crypt_file->dest_file); - } - my_free(file); - return NULL; -} - -static -int -encrypt_write(ds_file_t *file, const void *buf, size_t len) -{ - ds_encrypt_file_t *crypt_file; - ds_encrypt_ctxt_t *crypt_ctxt; - crypt_thread_ctxt_t *threads; - crypt_thread_ctxt_t *thd; - uint nthreads; - uint i; - const uchar *ptr; - - crypt_file = (ds_encrypt_file_t *) file->ptr; - crypt_ctxt = crypt_file->crypt_ctxt; - - threads = crypt_ctxt->threads; - nthreads = crypt_ctxt->nthreads; - - ptr = (const uchar *) buf; - while (len > 0) { - uint max_thread; - - /* Send data to worker threads for encryption */ - for (i = 0; i < nthreads; i++) { - size_t chunk_len; - - thd = threads + i; - - pthread_mutex_lock(&thd->ctrl_mutex); - - chunk_len = (len > XB_CRYPT_CHUNK_SIZE) ? - XB_CRYPT_CHUNK_SIZE : len; - thd->from = ptr; - thd->from_len = chunk_len; - - pthread_mutex_lock(&thd->data_mutex); - thd->data_avail = TRUE; - pthread_cond_signal(&thd->data_cond); - pthread_mutex_unlock(&thd->data_mutex); - - len -= chunk_len; - if (len == 0) { - break; - } - ptr += chunk_len; - } - - max_thread = (i < nthreads) ? i : nthreads - 1; - - /* Reap and stream the encrypted data */ - for (i = 0; i <= max_thread; i++) { - thd = threads + i; - - pthread_mutex_lock(&thd->data_mutex); - while (thd->data_avail == TRUE) { - pthread_cond_wait(&thd->data_cond, - &thd->data_mutex); - } - - xb_a(threads[i].to_len > 0); - - if (xb_crypt_write_chunk(crypt_file->xbcrypt_file, - threads[i].to, - threads[i].from_len + - XB_CRYPT_HASH_LEN, - threads[i].to_len, - threads[i].iv, - encrypt_iv_len)) { - msg("encrypt: write to the destination file " - "failed.\n"); - return 1; - } - - crypt_file->bytes_processed += threads[i].from_len; - - pthread_mutex_unlock(&threads[i].data_mutex); - pthread_mutex_unlock(&threads[i].ctrl_mutex); - } - } - - return 0; -} - -static -int -encrypt_close(ds_file_t *file) -{ - ds_encrypt_file_t *crypt_file; - ds_file_t *dest_file; - int rc = 0; - - crypt_file = (ds_encrypt_file_t *) file->ptr; - dest_file = crypt_file->dest_file; - - rc = xb_crypt_write_close(crypt_file->xbcrypt_file); - - if (ds_close(dest_file)) { - rc = 1; - } - - my_free(file); - - return rc; -} - -static -void -encrypt_deinit(ds_ctxt_t *ctxt) -{ - ds_encrypt_ctxt_t *crypt_ctxt; - - xb_ad(ctxt->pipe_ctxt != NULL); - - crypt_ctxt = (ds_encrypt_ctxt_t *) ctxt->ptr; - - destroy_worker_threads(crypt_ctxt->threads, crypt_ctxt->nthreads); - - my_free(ctxt->root); - my_free(ctxt); -} - -static -crypt_thread_ctxt_t * -create_worker_threads(uint n) -{ - crypt_thread_ctxt_t *threads; - uint i; - - threads = (crypt_thread_ctxt_t *) - my_malloc(sizeof(crypt_thread_ctxt_t) * n, MYF(MY_FAE)); - - for (i = 0; i < n; i++) { - crypt_thread_ctxt_t *thd = threads + i; - - thd->num = i + 1; - thd->started = FALSE; - thd->cancelled = FALSE; - thd->data_avail = FALSE; - - thd->to = (uchar *) my_malloc(XB_CRYPT_CHUNK_SIZE + - XB_CRYPT_HASH_LEN, MYF(MY_FAE)); - - thd->iv = (uchar *) my_malloc(encrypt_iv_len, MYF(MY_FAE)); - - /* Initialize the control mutex and condition var */ - if (pthread_mutex_init(&thd->ctrl_mutex, NULL) || - pthread_cond_init(&thd->ctrl_cond, NULL)) { - goto err; - } - - /* Initialize and data mutex and condition var */ - if (pthread_mutex_init(&thd->data_mutex, NULL) || - pthread_cond_init(&thd->data_cond, NULL)) { - goto err; - } - - if (xb_crypt_cipher_open(&thd->cipher_handle)) { - goto err; - } - - pthread_mutex_lock(&thd->ctrl_mutex); - - if (pthread_create(&thd->id, NULL, encrypt_worker_thread_func, - thd)) { - msg("encrypt: pthread_create() failed: " - "errno = %d\n", errno); - goto err; - } - } - - /* Wait for the threads to start */ - for (i = 0; i < n; i++) { - crypt_thread_ctxt_t *thd = threads + i; - - while (thd->started == FALSE) - pthread_cond_wait(&thd->ctrl_cond, &thd->ctrl_mutex); - pthread_mutex_unlock(&thd->ctrl_mutex); - } - - return threads; - -err: - return NULL; -} - -static -void -destroy_worker_threads(crypt_thread_ctxt_t *threads, uint n) -{ - uint i; - - for (i = 0; i < n; i++) { - crypt_thread_ctxt_t *thd = threads + i; - - pthread_mutex_lock(&thd->data_mutex); - threads[i].cancelled = TRUE; - pthread_cond_signal(&thd->data_cond); - pthread_mutex_unlock(&thd->data_mutex); - - pthread_join(thd->id, NULL); - - pthread_cond_destroy(&thd->data_cond); - pthread_mutex_destroy(&thd->data_mutex); - pthread_cond_destroy(&thd->ctrl_cond); - pthread_mutex_destroy(&thd->ctrl_mutex); - - xb_crypt_cipher_close(thd->cipher_handle); - - my_free(thd->to); - my_free(thd->iv); - } - - my_free(threads); -} - -static -void * -encrypt_worker_thread_func(void *arg) -{ - crypt_thread_ctxt_t *thd = (crypt_thread_ctxt_t *) arg; - - pthread_mutex_lock(&thd->ctrl_mutex); - - pthread_mutex_lock(&thd->data_mutex); - - thd->started = TRUE; - pthread_cond_signal(&thd->ctrl_cond); - - pthread_mutex_unlock(&thd->ctrl_mutex); - - while (1) { - thd->data_avail = FALSE; - pthread_cond_signal(&thd->data_cond); - - while (!thd->data_avail && !thd->cancelled) { - pthread_cond_wait(&thd->data_cond, &thd->data_mutex); - } - - if (thd->cancelled) - break; - - thd->to_len = thd->from_len; - - if (xb_crypt_encrypt(thd->cipher_handle, thd->from, - thd->from_len, thd->to, &thd->to_len, - thd->iv)) { - thd->to_len = 0; - continue; - } - } - - pthread_mutex_unlock(&thd->data_mutex); - - return NULL; -} -#endif /* HAVE_GCRYPT*/ diff --git a/extra/mariabackup/ds_encrypt.h b/extra/mariabackup/ds_encrypt.h deleted file mode 100644 index c4d8d7f8427..00000000000 --- a/extra/mariabackup/ds_encrypt.h +++ /dev/null @@ -1,33 +0,0 @@ -/****************************************************** -Copyright (c) 2013 Percona LLC and/or its affiliates. - -Encryption interface for XtraBackup. - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - -*******************************************************/ - -#ifndef DS_ENCRYPT_H -#define DS_ENCRYPT_H - -#include "datasink.h" -#ifdef HAVE_GCRYPT -extern datasink_t datasink_encrypt; -#endif -/* Encryption options */ -extern uint ds_encrypt_encrypt_threads; -extern ulonglong ds_encrypt_encrypt_chunk_size; - - -#endif diff --git a/extra/mariabackup/encryption_plugin.cc b/extra/mariabackup/encryption_plugin.cc index 9f2782d89a1..8f7741b057a 100644 --- a/extra/mariabackup/encryption_plugin.cc +++ b/extra/mariabackup/encryption_plugin.cc @@ -126,7 +126,6 @@ void encryption_plugin_prepare_init(int argc, char **argv) if (!xb_plugin_load) { - /* This prevents crashes e.g in --stats with wrong my.cnf*/ finalize_encryption_plugin(0); return; } diff --git a/extra/mariabackup/innobackupex.cc b/extra/mariabackup/innobackupex.cc index 2114e546440..7e97b258489 100644 --- a/extra/mariabackup/innobackupex.cc +++ b/extra/mariabackup/innobackupex.cc @@ -101,8 +101,6 @@ char *opt_ibx_login_path = NULL; ulong opt_ibx_lock_wait_query_type; ulong opt_ibx_kill_long_query_type; -ulong opt_ibx_decrypt_algo = 0; - uint opt_ibx_kill_long_queries_timeout = 0; uint opt_ibx_lock_wait_timeout = 0; uint opt_ibx_lock_wait_threshold = 0; @@ -110,7 +108,6 @@ uint opt_ibx_debug_sleep_before_unlock = 0; uint opt_ibx_safe_slave_backup_timeout = 0; const char *opt_ibx_history = NULL; -bool opt_ibx_decrypt = false; char *opt_ibx_include = NULL; char *opt_ibx_databases = NULL; @@ -121,15 +118,9 @@ char *ibx_backup_directory = NULL; /* copy of proxied xtrabackup options */ my_bool ibx_xb_close_files; -my_bool ibx_xtrabackup_compact; const char *ibx_xtrabackup_compress_alg; uint ibx_xtrabackup_compress_threads; ulonglong ibx_xtrabackup_compress_chunk_size; -ulong ibx_xtrabackup_encrypt_algo; -char *ibx_xtrabackup_encrypt_key; -char *ibx_xtrabackup_encrypt_key_file; -uint ibx_xtrabackup_encrypt_threads; -ulonglong ibx_xtrabackup_encrypt_chunk_size; my_bool ibx_xtrabackup_export; char *ibx_xtrabackup_extra_lsndir; char *ibx_xtrabackup_incremental_basedir; @@ -138,8 +129,6 @@ my_bool ibx_xtrabackup_incremental_force_scan; ulint ibx_xtrabackup_log_copy_interval; char *ibx_xtrabackup_incremental; int ibx_xtrabackup_parallel; -my_bool ibx_xtrabackup_rebuild_indexes; -ulint ibx_xtrabackup_rebuild_threads; char *ibx_xtrabackup_stream_str; char *ibx_xtrabackup_tables_file; long ibx_xtrabackup_throttle; @@ -201,7 +190,6 @@ enum innobackupex_options OPT_NO_VERSION_CHECK, OPT_NO_BACKUP_LOCKS, OPT_DATABASES, - OPT_DECRYPT, OPT_DECOMPRESS, /* options wich are passed directly to xtrabackup */ @@ -210,11 +198,6 @@ enum innobackupex_options OPT_COMPRESS, OPT_COMPRESS_THREADS, OPT_COMPRESS_CHUNK_SIZE, - OPT_ENCRYPT, - OPT_ENCRYPT_KEY, - OPT_ENCRYPT_KEY_FILE, - OPT_ENCRYPT_THREADS, - OPT_ENCRYPT_CHUNK_SIZE, OPT_EXPORT, OPT_EXTRA_LSNDIR, OPT_INCREMENTAL_BASEDIR, @@ -430,12 +413,6 @@ static struct my_option ibx_long_options[] = (uchar*) &opt_ibx_incremental_history_uuid, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, - {"decrypt", OPT_DECRYPT, "Decrypts all files with the .xbcrypt " - "extension in a backup previously made with --encrypt option.", - &opt_ibx_decrypt_algo, &opt_ibx_decrypt_algo, - &xtrabackup_encrypt_algo_typelib, GET_ENUM, REQUIRED_ARG, - 0, 0, 0, 0, 0, 0}, - {"ftwrl-wait-query-type", OPT_LOCK_WAIT_QUERY_TYPE, "This option specifies which types of queries are allowed to complete " "before innobackupex will issue the global lock. Default is all.", @@ -533,12 +510,6 @@ static struct my_option ibx_long_options[] = (uchar*) &ibx_xb_close_files, (uchar*) &ibx_xb_close_files, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, - {"compact", OPT_COMPACT, "Create a compact backup with all secondary " - "index pages omitted. This option is passed directly to xtrabackup. " - "See xtrabackup documentation for details.", - (uchar*) &ibx_xtrabackup_compact, (uchar*) &ibx_xtrabackup_compact, - 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, - {"compress", OPT_COMPRESS, "This option instructs xtrabackup to " "compress backup copies of InnoDB data files. It is passed directly " "to the xtrabackup child process. Try 'xtrabackup --help' for more " @@ -560,46 +531,6 @@ static struct my_option ibx_long_options[] = (uchar*) &ibx_xtrabackup_compress_chunk_size, 0, GET_ULL, REQUIRED_ARG, (1 << 16), 1024, ULONGLONG_MAX, 0, 0, 0}, - {"encrypt", OPT_ENCRYPT, "This option instructs xtrabackup to encrypt " - "backup copies of InnoDB data files using the algorithm specified in " - "the ENCRYPTION-ALGORITHM. It is passed directly to the xtrabackup " - "child process. Try 'xtrabackup --help' for more details.", - &ibx_xtrabackup_encrypt_algo, &ibx_xtrabackup_encrypt_algo, - &xtrabackup_encrypt_algo_typelib, GET_ENUM, REQUIRED_ARG, - 0, 0, 0, 0, 0, 0}, - - {"encrypt-key", OPT_ENCRYPT_KEY, "This option instructs xtrabackup to " - "use the given ENCRYPTION-KEY when using the --encrypt or --decrypt " - "options. During backup it is passed directly to the xtrabackup child " - "process. Try 'xtrabackup --help' for more details.", - (uchar*) &ibx_xtrabackup_encrypt_key, - (uchar*) &ibx_xtrabackup_encrypt_key, 0, - GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, - - {"encrypt-key-file", OPT_ENCRYPT_KEY_FILE, "This option instructs " - "xtrabackup to use the encryption key stored in the given " - "ENCRYPTION-KEY-FILE when using the --encrypt or --decrypt options.", - (uchar*) &ibx_xtrabackup_encrypt_key_file, - (uchar*) &ibx_xtrabackup_encrypt_key_file, 0, - GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, - - {"encrypt-threads", OPT_ENCRYPT_THREADS, - "This option specifies the number of worker threads that will be used " - "for parallel encryption. It is passed directly to the xtrabackup " - "child process. Try 'xtrabackup --help' for more details.", - (uchar*) &ibx_xtrabackup_encrypt_threads, - (uchar*) &ibx_xtrabackup_encrypt_threads, - 0, GET_UINT, REQUIRED_ARG, 1, 1, UINT_MAX, 0, 0, 0}, - - {"encrypt-chunk-size", OPT_ENCRYPT_CHUNK_SIZE, - "This option specifies the size of the internal working buffer for " - "each encryption thread, measured in bytes. It is passed directly to " - "the xtrabackup child process. Try 'xtrabackup --help' for more " - "details.", - (uchar*) &ibx_xtrabackup_encrypt_chunk_size, - (uchar*) &ibx_xtrabackup_encrypt_chunk_size, - 0, GET_ULL, REQUIRED_ARG, (1 << 16), 1024, ULONGLONG_MAX, 0, 0, 0}, - {"export", OPT_EXPORT, "This option is passed directly to xtrabackup's " "--export option. It enables exporting individual tables for import " "into another server. See the xtrabackup documentation for details.", @@ -735,8 +666,6 @@ You can download full text of the license on http://www.gnu.org/licenses/gpl-2.0 SYNOPOSIS\n\ \n\ innobackupex [--compress] [--compress-threads=NUMBER-OF-THREADS] [--compress-chunk-size=CHUNK-SIZE]\n\ - [--encrypt=ENCRYPTION-ALGORITHM] [--encrypt-threads=NUMBER-OF-THREADS] [--encrypt-chunk-size=CHUNK-SIZE]\n\ - [--encrypt-key=LITERAL-ENCRYPTION-KEY] | [--encryption-key-file=MY.KEY]\n\ [--include=REGEXP] [--user=NAME]\n\ [--password=WORD] [--port=PORT] [--socket=SOCKET]\n\ [--no-timestamp] [--ibbackup=IBBACKUP-BINARY]\n\ @@ -748,7 +677,7 @@ innobackupex [--compress] [--compress-threads=NUMBER-OF-THREADS] [--compress-chu [--incremental] [--incremental-basedir]\n\ [--incremental-dir] [--incremental-force-scan] [--incremental-lsn]\n\ [--incremental-history-name=NAME] [--incremental-history-uuid=UUID]\n\ - [--close-files] [--compact] \n\ + [--close-files]\n\ BACKUP-ROOT-DIR\n\ \n\ innobackupex --apply-log [--use-memory=B]\n\ @@ -760,8 +689,7 @@ innobackupex --copy-back [--defaults-file=MY.CNF] [--defaults-group=GROUP-NAME] \n\ innobackupex --move-back [--defaults-file=MY.CNF] [--defaults-group=GROUP-NAME] BACKUP-DIR\n\ \n\ -innobackupex [--decompress] [--decrypt=ENCRYPTION-ALGORITHM]\n\ - [--encrypt-key=LITERAL-ENCRYPTION-KEY] | [--encryption-key-file=MY.KEY]\n\ +innobackupex [--decompress]\n\ [--parallel=NUMBER-OF-FORKS] BACKUP-DIR\n\ \n\ DESCRIPTION\n\ @@ -798,15 +726,12 @@ it moves files to their original locations rather than copies them. As this\n\ option removes backup files, it must be used with caution. It may be useful in\n\ cases when there is not enough free disk space to copy files.\n\ \n\ -The --decompress --decrypt command will decrypt and/or decompress a backup made\n\ -with the --compress and/or --encrypt options. When decrypting, the encryption\n\ -algorithm and key used when the backup was taken MUST be provided via the\n\ -specified options. --decrypt and --decompress may be used together at the same\n\ -time to completely normalize a previously compressed and encrypted backup. The\n\ ---parallel option will allow multiple files to be decrypted and/or decompressed\n\ +The --decompress command will decompress a backup made\n\ +with the --compress option. The\n\ +--parallel option will allow multiple files to be decompressed\n\ simultaneously. In order to decompress, the qpress utility MUST be installed\n\ and accessable within the path. This process will remove the original\n\ -compressed/encrypted files and leave the results in the same location.\n\ +compressed files and leave the results in the same location.\n\ \n\ On success the exit code innobackupex is 0. A non-zero exit code \n\ indicates an error.\n"); @@ -839,14 +764,6 @@ ibx_get_one_option(int optid, opt_ibx_history = ""; } break; - case OPT_DECRYPT: - if (argument == NULL) { - ibx_msg("Missing --decrypt argument, must specify a " - "valid encryption algorithm.\n"); - return(1); - } - opt_ibx_decrypt = true; - break; case OPT_STREAM: if (!strcasecmp(argument, "xbstream")) xtrabackup_stream_fmt = XB_STREAM_FMT_XBSTREAM; @@ -866,15 +783,6 @@ ibx_get_one_option(int optid, } xtrabackup_compress = TRUE; break; - case OPT_ENCRYPT: - if (argument == NULL) - { - msg("Missing --encrypt argument, must specify a " - "valid encryption algorithm.\n"); - return 1; - } - xtrabackup_encrypt = TRUE; - break; case 'p': if (argument) { @@ -928,7 +836,7 @@ ibx_handle_options(int *argc, char ***argv) ibx_mode = IBX_MODE_COPY_BACK; } else if (opt_ibx_move_back) { ibx_mode = IBX_MODE_MOVE_BACK; - } else if (opt_ibx_decrypt || opt_ibx_decompress) { + } else if (opt_ibx_decompress) { ibx_mode = IBX_MODE_DECRYPT_DECOMPRESS; } else { ibx_mode = IBX_MODE_BACKUP; @@ -1006,8 +914,6 @@ ibx_init() opt_lock_wait_query_type = opt_ibx_lock_wait_query_type; opt_kill_long_query_type = opt_ibx_kill_long_query_type; - opt_decrypt_algo = opt_ibx_decrypt_algo; - opt_kill_long_queries_timeout = opt_ibx_kill_long_queries_timeout; opt_lock_wait_timeout = opt_ibx_lock_wait_timeout; opt_lock_wait_threshold = opt_ibx_lock_wait_threshold; @@ -1015,18 +921,12 @@ ibx_init() opt_safe_slave_backup_timeout = opt_ibx_safe_slave_backup_timeout; opt_history = opt_ibx_history; - opt_decrypt = opt_ibx_decrypt; /* setup xtrabackup options */ xb_close_files = ibx_xb_close_files; xtrabackup_compress_alg = ibx_xtrabackup_compress_alg; xtrabackup_compress_threads = ibx_xtrabackup_compress_threads; xtrabackup_compress_chunk_size = ibx_xtrabackup_compress_chunk_size; - xtrabackup_encrypt_algo = ibx_xtrabackup_encrypt_algo; - xtrabackup_encrypt_key = ibx_xtrabackup_encrypt_key; - xtrabackup_encrypt_key_file = ibx_xtrabackup_encrypt_key_file; - xtrabackup_encrypt_threads = ibx_xtrabackup_encrypt_threads; - xtrabackup_encrypt_chunk_size = ibx_xtrabackup_encrypt_chunk_size; xtrabackup_export = ibx_xtrabackup_export; xtrabackup_extra_lsndir = ibx_xtrabackup_extra_lsndir; xtrabackup_incremental_basedir = ibx_xtrabackup_incremental_basedir; @@ -1107,7 +1007,7 @@ ibx_init() case IBX_MODE_DECRYPT_DECOMPRESS: xtrabackup_decrypt_decompress = TRUE; xtrabackup_target_dir = ibx_position_arg; - run = "decrypt and decompress"; + run = "decompress"; break; default: ut_error; diff --git a/extra/mariabackup/xb0xb.h b/extra/mariabackup/xb0xb.h index cb1aedd5d13..a8b17f59579 100644 --- a/extra/mariabackup/xb0xb.h +++ b/extra/mariabackup/xb0xb.h @@ -44,19 +44,6 @@ dberr_t* err, /*!< out: this is set to DB_ERROR if an error os_file_dir_t dir, /*!< in: directory stream */ os_file_stat_t* info) /*!< in/out: buffer where the info is returned */; -buf_block_t* btr_node_ptr_get_child( - const rec_t* node_ptr,/*!< in: node pointer */ - dict_index_t* index, /*!< in: index */ - const ulint* offsets,/*!< in: array returned by rec_get_offsets() */ - mtr_t* mtr) /*!< in: mtr */; - -buf_block_t* -btr_root_block_get( -/*===============*/ -const dict_index_t* index, /*!< in: index tree */ -ulint mode, /*!< in: either RW_S_LATCH - or RW_X_LATCH */ - mtr_t* mtr) /*!< in: mtr */; fil_space_t* fil_space_get_by_name(const char *); ibool diff --git a/extra/mariabackup/xbcrypt.c b/extra/mariabackup/xbcrypt.c deleted file mode 100644 index 3da70e171f7..00000000000 --- a/extra/mariabackup/xbcrypt.c +++ /dev/null @@ -1,696 +0,0 @@ -/****************************************************** -Copyright (c) 2013 Percona LLC and/or its affiliates. - -The xbcrypt utility: decrypt files in the XBCRYPT format. - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - -*******************************************************/ - -#include -#include -#include "common.h" -#include "xbcrypt.h" -#include "xbcrypt_common.h" -#include "crc_glue.h" - -#if !defined(GCRYPT_VERSION_NUMBER) || (GCRYPT_VERSION_NUMBER < 0x010600) -GCRY_THREAD_OPTION_PTHREAD_IMPL; -#endif - -#define XBCRYPT_VERSION "1.1" - -typedef enum { - RUN_MODE_NONE, - RUN_MODE_ENCRYPT, - RUN_MODE_DECRYPT -} run_mode_t; - -const char *xbcrypt_encrypt_algo_names[] = -{ "NONE", "AES128", "AES192", "AES256", NullS}; -TYPELIB xbcrypt_encrypt_algo_typelib= -{array_elements(xbcrypt_encrypt_algo_names)-1,"", - xbcrypt_encrypt_algo_names, NULL}; - -static run_mode_t opt_run_mode = RUN_MODE_ENCRYPT; -static char *opt_input_file = NULL; -static char *opt_output_file = NULL; -static ulong opt_encrypt_algo; -static char *opt_encrypt_key_file = NULL; -static void *opt_encrypt_key = NULL; -static ulonglong opt_encrypt_chunk_size = 0; -static my_bool opt_verbose = FALSE; - -static uint encrypt_algos[] = { GCRY_CIPHER_NONE, - GCRY_CIPHER_AES128, - GCRY_CIPHER_AES192, - GCRY_CIPHER_AES256 }; -static int encrypt_algo = 0; -static int encrypt_mode = GCRY_CIPHER_MODE_CTR; -static uint encrypt_key_len = 0; -static size_t encrypt_iv_len = 0; - -static struct my_option my_long_options[] = -{ - {"help", '?', "Display this help and exit.", - 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, - - {"decrypt", 'd', "Decrypt data input to output.", - 0, 0, 0, - GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, - - {"input", 'i', "Optional input file. If not specified, input" - " will be read from standard input.", - &opt_input_file, &opt_input_file, 0, - GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, - - {"output", 'o', "Optional output file. If not specified, output" - " will be written to standard output.", - &opt_output_file, &opt_output_file, 0, - GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, - - {"encrypt-algo", 'a', "Encryption algorithm.", - &opt_encrypt_algo, &opt_encrypt_algo, &xbcrypt_encrypt_algo_typelib, - GET_ENUM, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, - - {"encrypt-key", 'k', "Encryption key.", - &opt_encrypt_key, &opt_encrypt_key, 0, - GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, - - {"encrypt-key-file", 'f', "File which contains encryption key.", - &opt_encrypt_key_file, &opt_encrypt_key_file, 0, - GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, - - {"encrypt-chunk-size", 's', "Size of working buffer for encryption in" - " bytes. The default value is 64K.", - &opt_encrypt_chunk_size, &opt_encrypt_chunk_size, 0, - GET_ULL, REQUIRED_ARG, (1 << 16), 1024, ULONGLONG_MAX, 0, 0, 0}, - - {"verbose", 'v', "Display verbose status output.", - &opt_verbose, &opt_verbose, - 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, - {0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0} -}; - -static -int -get_options(int *argc, char ***argv); - -static -my_bool -get_one_option(int optid, const struct my_option *opt __attribute__((unused)), - char *argument __attribute__((unused))); - -static -void -print_version(void); - -static -void -usage(void); - -static -int -mode_decrypt(File filein, File fileout); - -static -int -mode_encrypt(File filein, File fileout); - -int -main(int argc, char **argv) -{ -#if !defined(GCRYPT_VERSION_NUMBER) || (GCRYPT_VERSION_NUMBER < 0x010600) - gcry_error_t gcry_error; -#endif - File filein = 0; - File fileout = 0; - - MY_INIT(argv[0]); - - crc_init(); - - if (get_options(&argc, &argv)) { - goto err; - } - - /* Acording to gcrypt docs (and my testing), setting up the threading - callbacks must be done first, so, lets give it a shot */ -#if !defined(GCRYPT_VERSION_NUMBER) || (GCRYPT_VERSION_NUMBER < 0x010600) - gcry_error = gcry_control(GCRYCTL_SET_THREAD_CBS, &gcry_threads_pthread); - if (gcry_error) { - msg("%s: unable to set libgcrypt thread cbs - " - "%s : %s\n", my_progname, - gcry_strsource(gcry_error), - gcry_strerror(gcry_error)); - return 1; - } -#endif - - /* Version check should be the very first call because it - makes sure that important subsystems are intialized. */ - if (!gcry_control(GCRYCTL_ANY_INITIALIZATION_P)) { - const char *gcrypt_version; - gcrypt_version = gcry_check_version(NULL); - /* No other library has already initialized libgcrypt. */ - if (!gcrypt_version) { - msg("%s: failed to initialize libgcrypt\n", - my_progname); - return 1; - } else if (opt_verbose) { - msg("%s: using gcrypt %s\n", my_progname, - gcrypt_version); - } - } - gcry_control(GCRYCTL_DISABLE_SECMEM, 0); - gcry_control(GCRYCTL_INITIALIZATION_FINISHED, 0); - - /* Determine the algorithm */ - encrypt_algo = encrypt_algos[opt_encrypt_algo]; - - /* Set up the iv length */ - encrypt_iv_len = gcry_cipher_get_algo_blklen(encrypt_algo); - - /* Now set up the key */ - if (opt_encrypt_key == NULL && opt_encrypt_key_file == NULL) { - msg("%s: no encryption key or key file specified.\n", - my_progname); - return 1; - } else if (opt_encrypt_key && opt_encrypt_key_file) { - msg("%s: both encryption key and key file specified.\n", - my_progname); - return 1; - } else if (opt_encrypt_key_file) { - if (!xb_crypt_read_key_file(opt_encrypt_key_file, - &opt_encrypt_key, - &encrypt_key_len)) { - msg("%s: unable to read encryption key file \"%s\".\n", - opt_encrypt_key_file, my_progname); - return 1; - } - } else { - encrypt_key_len = strlen(opt_encrypt_key); - } - - if (opt_input_file) { - MY_STAT mystat; - - if (opt_verbose) - msg("%s: input file \"%s\".\n", my_progname, - opt_input_file); - - if (my_stat(opt_input_file, &mystat, MYF(MY_WME)) == NULL) { - goto err; - } - if (!MY_S_ISREG(mystat.st_mode)) { - msg("%s: \"%s\" is not a regular file, exiting.\n", - my_progname, opt_input_file); - goto err; - } - if ((filein = my_open(opt_input_file, O_RDONLY, MYF(MY_WME))) - < 0) { - msg("%s: failed to open \"%s\".\n", my_progname, - opt_input_file); - goto err; - } - } else { - if (opt_verbose) - msg("%s: input from standard input.\n", my_progname); - filein = fileno(stdin); - } - - if (opt_output_file) { - if (opt_verbose) - msg("%s: output file \"%s\".\n", my_progname, - opt_output_file); - - if ((fileout = my_create(opt_output_file, 0, - O_WRONLY|O_BINARY|O_EXCL|O_NOFOLLOW, - MYF(MY_WME))) < 0) { - msg("%s: failed to create output file \"%s\".\n", - my_progname, opt_output_file); - goto err; - } - } else { - if (opt_verbose) - msg("%s: output to standard output.\n", my_progname); - fileout = fileno(stdout); - } - - if (opt_run_mode == RUN_MODE_DECRYPT - && mode_decrypt(filein, fileout)) { - goto err; - } else if (opt_run_mode == RUN_MODE_ENCRYPT - && mode_encrypt(filein, fileout)) { - goto err; - } - - if (opt_input_file && filein) { - my_close(filein, MYF(MY_WME)); - } - if (opt_output_file && fileout) { - my_close(fileout, MYF(MY_WME)); - } - - my_cleanup_options(my_long_options); - - my_end(0); - - return EXIT_SUCCESS; -err: - if (opt_input_file && filein) { - my_close(filein, MYF(MY_WME)); - } - if (opt_output_file && fileout) { - my_close(fileout, MYF(MY_WME)); - } - - my_cleanup_options(my_long_options); - - my_end(0); - - exit(EXIT_FAILURE); - -} - - -static -size_t -my_xb_crypt_read_callback(void *userdata, void *buf, size_t len) -{ - File* file = (File *) userdata; - return xb_read_full(*file, buf, len); -} - -static -int -mode_decrypt(File filein, File fileout) -{ - xb_rcrypt_t *xbcrypt_file = NULL; - void *chunkbuf = NULL; - size_t chunksize; - size_t originalsize; - void *ivbuf = NULL; - size_t ivsize; - void *decryptbuf = NULL; - size_t decryptbufsize = 0; - ulonglong ttlchunksread = 0; - ulonglong ttlbytesread = 0; - xb_rcrypt_result_t result; - gcry_cipher_hd_t cipher_handle; - gcry_error_t gcry_error; - my_bool hash_appended; - - if (encrypt_algo != GCRY_CIPHER_NONE) { - gcry_error = gcry_cipher_open(&cipher_handle, - encrypt_algo, - encrypt_mode, 0); - if (gcry_error) { - msg("%s:decrypt: unable to open libgcrypt" - " cipher - %s : %s\n", my_progname, - gcry_strsource(gcry_error), - gcry_strerror(gcry_error)); - return 1; - } - - gcry_error = gcry_cipher_setkey(cipher_handle, - opt_encrypt_key, - encrypt_key_len); - if (gcry_error) { - msg("%s:decrypt: unable to set libgcrypt cipher" - "key - %s : %s\n", my_progname, - gcry_strsource(gcry_error), - gcry_strerror(gcry_error)); - goto err; - } - } - - /* Initialize the xb_crypt format reader */ - xbcrypt_file = xb_crypt_read_open(&filein, my_xb_crypt_read_callback); - if (xbcrypt_file == NULL) { - msg("%s:decrypt: xb_crypt_read_open() failed.\n", my_progname); - goto err; - } - - /* Walk the encrypted chunks, decrypting them and writing out */ - while ((result = xb_crypt_read_chunk(xbcrypt_file, &chunkbuf, - &originalsize, &chunksize, - &ivbuf, &ivsize, &hash_appended)) - == XB_CRYPT_READ_CHUNK) { - - if (encrypt_algo != GCRY_CIPHER_NONE) { - gcry_error = gcry_cipher_reset(cipher_handle); - if (gcry_error) { - msg("%s:decrypt: unable to reset libgcrypt" - " cipher - %s : %s\n", my_progname, - gcry_strsource(gcry_error), - gcry_strerror(gcry_error)); - goto err; - } - - if (ivsize) { - gcry_error = gcry_cipher_setctr(cipher_handle, - ivbuf, - ivsize); - } - if (gcry_error) { - msg("%s:decrypt: unable to set cipher iv - " - "%s : %s\n", my_progname, - gcry_strsource(gcry_error), - gcry_strerror(gcry_error)); - continue; - } - - if (decryptbufsize < originalsize) { - decryptbuf = my_realloc(decryptbuf, - originalsize, - MYF(MY_WME | MY_ALLOW_ZERO_PTR)); - decryptbufsize = originalsize; - } - - /* Try to decrypt it */ - gcry_error = gcry_cipher_decrypt(cipher_handle, - decryptbuf, - originalsize, - chunkbuf, - chunksize); - if (gcry_error) { - msg("%s:decrypt: unable to decrypt chunk - " - "%s : %s\n", my_progname, - gcry_strsource(gcry_error), - gcry_strerror(gcry_error)); - gcry_cipher_close(cipher_handle); - goto err; - } - - } else { - decryptbuf = chunkbuf; - } - - if (hash_appended) { - uchar hash[XB_CRYPT_HASH_LEN]; - - originalsize -= XB_CRYPT_HASH_LEN; - - /* ensure that XB_CRYPT_HASH_LEN is the correct length - of XB_CRYPT_HASH hashing algorithm output */ - xb_a(gcry_md_get_algo_dlen(XB_CRYPT_HASH) == - XB_CRYPT_HASH_LEN); - gcry_md_hash_buffer(XB_CRYPT_HASH, hash, decryptbuf, - originalsize); - if (memcmp(hash, (char *) decryptbuf + originalsize, - XB_CRYPT_HASH_LEN) != 0) { - msg("%s:%s invalid plaintext hash. " - "Wrong encrytion key specified?\n", - my_progname, __FUNCTION__); - result = XB_CRYPT_READ_ERROR; - goto err; - } - } - - /* Write it out */ - if (my_write(fileout, (const uchar *) decryptbuf, originalsize, - MYF(MY_WME | MY_NABP))) { - msg("%s:decrypt: unable to write output chunk.\n", - my_progname); - goto err; - } - ttlchunksread++; - ttlbytesread += chunksize; - if (opt_verbose) - msg("%s:decrypt: %llu chunks read, %llu bytes read\n.", - my_progname, ttlchunksread, ttlbytesread); - } - - xb_crypt_read_close(xbcrypt_file); - - if (encrypt_algo != GCRY_CIPHER_NONE) - gcry_cipher_close(cipher_handle); - - if (decryptbuf && decryptbufsize) - my_free(decryptbuf); - - if (opt_verbose) - msg("\n%s:decrypt: done\n", my_progname); - - return 0; -err: - if (xbcrypt_file) - xb_crypt_read_close(xbcrypt_file); - - if (encrypt_algo != GCRY_CIPHER_NONE) - gcry_cipher_close(cipher_handle); - - if (decryptbuf && decryptbufsize) - my_free(decryptbuf); - - return 1; -} - -static -ssize_t -my_xb_crypt_write_callback(void *userdata, const void *buf, size_t len) -{ - File* file = (File *) userdata; - - ssize_t ret = my_write(*file, buf, len, MYF(MY_WME)); - posix_fadvise(*file, 0, 0, POSIX_FADV_DONTNEED); - return ret; -} - -static -int -mode_encrypt(File filein, File fileout) -{ - size_t bytesread; - size_t chunkbuflen; - uchar *chunkbuf = NULL; - void *ivbuf = NULL; - size_t encryptbuflen = 0; - size_t encryptedlen = 0; - void *encryptbuf = NULL; - ulonglong ttlchunkswritten = 0; - ulonglong ttlbyteswritten = 0; - xb_wcrypt_t *xbcrypt_file = NULL; - gcry_cipher_hd_t cipher_handle; - gcry_error_t gcry_error; - - if (encrypt_algo != GCRY_CIPHER_NONE) { - gcry_error = gcry_cipher_open(&cipher_handle, - encrypt_algo, - encrypt_mode, 0); - if (gcry_error) { - msg("%s:encrypt: unable to open libgcrypt cipher - " - "%s : %s\n", my_progname, - gcry_strsource(gcry_error), - gcry_strerror(gcry_error)); - return 1; - } - - gcry_error = gcry_cipher_setkey(cipher_handle, - opt_encrypt_key, - encrypt_key_len); - if (gcry_error) { - msg("%s:encrypt: unable to set libgcrypt cipher key - " - "%s : %s\n", my_progname, - gcry_strsource(gcry_error), - gcry_strerror(gcry_error)); - goto err; - } - } - - posix_fadvise(filein, 0, 0, POSIX_FADV_SEQUENTIAL); - - xbcrypt_file = xb_crypt_write_open(&fileout, - my_xb_crypt_write_callback); - if (xbcrypt_file == NULL) { - msg("%s:encrypt: xb_crypt_write_open() failed.\n", - my_progname); - goto err; - } - - ivbuf = my_malloc(encrypt_iv_len, MYF(MY_FAE)); - - /* now read in data in chunk size, encrypt and write out */ - chunkbuflen = opt_encrypt_chunk_size + XB_CRYPT_HASH_LEN; - chunkbuf = (uchar *) my_malloc(chunkbuflen, MYF(MY_FAE)); - while ((bytesread = my_read(filein, chunkbuf, opt_encrypt_chunk_size, - MYF(MY_WME))) > 0) { - - size_t origbuflen = bytesread + XB_CRYPT_HASH_LEN; - - /* ensure that XB_CRYPT_HASH_LEN is the correct length - of XB_CRYPT_HASH hashing algorithm output */ - xb_a(XB_CRYPT_HASH_LEN == gcry_md_get_algo_dlen(XB_CRYPT_HASH)); - gcry_md_hash_buffer(XB_CRYPT_HASH, chunkbuf + bytesread, - chunkbuf, bytesread); - - if (encrypt_algo != GCRY_CIPHER_NONE) { - gcry_error = gcry_cipher_reset(cipher_handle); - - if (gcry_error) { - msg("%s:encrypt: unable to reset cipher - " - "%s : %s\n", my_progname, - gcry_strsource(gcry_error), - gcry_strerror(gcry_error)); - goto err; - } - - xb_crypt_create_iv(ivbuf, encrypt_iv_len); - gcry_error = gcry_cipher_setctr(cipher_handle, - ivbuf, - encrypt_iv_len); - - if (gcry_error) { - msg("%s:encrypt: unable to set cipher iv - " - "%s : %s\n", my_progname, - gcry_strsource(gcry_error), - gcry_strerror(gcry_error)); - continue; - } - - if (encryptbuflen < origbuflen) { - encryptbuf = my_realloc(encryptbuf, origbuflen, - MYF(MY_WME | MY_ALLOW_ZERO_PTR)); - encryptbuflen = origbuflen; - } - - gcry_error = gcry_cipher_encrypt(cipher_handle, - encryptbuf, - encryptbuflen, - chunkbuf, - origbuflen); - - encryptedlen = origbuflen; - - if (gcry_error) { - msg("%s:encrypt: unable to encrypt chunk - " - "%s : %s\n", my_progname, - gcry_strsource(gcry_error), - gcry_strerror(gcry_error)); - gcry_cipher_close(cipher_handle); - goto err; - } - } else { - encryptedlen = origbuflen; - encryptbuf = chunkbuf; - } - - if (xb_crypt_write_chunk(xbcrypt_file, encryptbuf, - bytesread + XB_CRYPT_HASH_LEN, - encryptedlen, ivbuf, encrypt_iv_len)) { - msg("%s:encrypt: abcrypt_write_chunk() failed.\n", - my_progname); - goto err; - } - - ttlchunkswritten++; - ttlbyteswritten += encryptedlen; - - if (opt_verbose) - msg("%s:encrypt: %llu chunks written, %llu bytes " - "written\n.", my_progname, ttlchunkswritten, - ttlbyteswritten); - } - - my_free(ivbuf); - my_free(chunkbuf); - - if (encryptbuf && encryptbuflen) - my_free(encryptbuf); - - xb_crypt_write_close(xbcrypt_file); - - if (encrypt_algo != GCRY_CIPHER_NONE) - gcry_cipher_close(cipher_handle); - - if (opt_verbose) - msg("\n%s:encrypt: done\n", my_progname); - - return 0; -err: - if (chunkbuf) - my_free(chunkbuf); - - if (encryptbuf && encryptbuflen) - my_free(encryptbuf); - - if (xbcrypt_file) - xb_crypt_write_close(xbcrypt_file); - - if (encrypt_algo != GCRY_CIPHER_NONE) - gcry_cipher_close(cipher_handle); - - return 1; -} - -static -int -get_options(int *argc, char ***argv) -{ - int ho_error; - - if ((ho_error= handle_options(argc, argv, my_long_options, - get_one_option))) { - exit(EXIT_FAILURE); - } - - return 0; -} - -static -my_bool -get_one_option(int optid, const struct my_option *opt __attribute__((unused)), - char *argument __attribute__((unused))) -{ - switch (optid) { - case 'd': - opt_run_mode = RUN_MODE_DECRYPT; - break; - case '?': - usage(); - exit(0); - } - - return FALSE; -} - -static -void -print_version(void) -{ - printf("%s Ver %s for %s (%s)\n", my_progname, XBCRYPT_VERSION, - SYSTEM_TYPE, MACHINE_TYPE); -} - -static -void -usage(void) -{ - print_version(); - puts("Copyright (C) 2011 Percona Inc."); - puts("This software comes with ABSOLUTELY NO WARRANTY. " - "This is free software,\nand you are welcome to modify and " - "redistribute it under the GPL license.\n"); - - puts("Encrypt or decrypt files in the XBCRYPT format.\n"); - - puts("Usage: "); - printf(" %s [OPTIONS...]" - " # read data from specified input, encrypting or decrypting " - " and writing the result to the specified output.\n", - my_progname); - puts("\nOptions:"); - my_print_help(my_long_options); -} diff --git a/extra/mariabackup/xbcrypt.h b/extra/mariabackup/xbcrypt.h deleted file mode 100644 index 0e832266847..00000000000 --- a/extra/mariabackup/xbcrypt.h +++ /dev/null @@ -1,79 +0,0 @@ -/****************************************************** -Copyright (c) 2011 Percona LLC and/or its affiliates. - -Encryption interface for XtraBackup. - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - -*******************************************************/ - -#ifndef XBCRYPT_H -#define XBCRYPT_H - -#include -#include "common.h" - -#define XB_CRYPT_CHUNK_MAGIC1 "XBCRYP01" -#define XB_CRYPT_CHUNK_MAGIC2 "XBCRYP02" -#define XB_CRYPT_CHUNK_MAGIC3 "XBCRYP03" /* must be same size as ^^ */ -#define XB_CRYPT_CHUNK_MAGIC_CURRENT XB_CRYPT_CHUNK_MAGIC3 -#define XB_CRYPT_CHUNK_MAGIC_SIZE (sizeof(XB_CRYPT_CHUNK_MAGIC1)-1) - -#define XB_CRYPT_HASH GCRY_MD_SHA256 -#define XB_CRYPT_HASH_LEN 32 - -/****************************************************************************** -Write interface */ -typedef struct xb_wcrypt_struct xb_wcrypt_t; - -/* Callback on write for i/o, must return # of bytes written or -1 on error */ -typedef ssize_t xb_crypt_write_callback(void *userdata, - const void *buf, size_t len); - -xb_wcrypt_t *xb_crypt_write_open(void *userdata, - xb_crypt_write_callback *onwrite); - -/* Takes buffer, original length, encrypted length iv and iv length, formats - output buffer and calls write callback. - Returns 0 on success, 1 on error */ -int xb_crypt_write_chunk(xb_wcrypt_t *crypt, const void *buf, size_t olen, - size_t elen, const void *iv, size_t ivlen); - -/* Returns 0 on success, 1 on error */ -int xb_crypt_write_close(xb_wcrypt_t *crypt); - -/****************************************************************************** -Read interface */ -typedef struct xb_rcrypt_struct xb_rcrypt_t; - -/* Callback on read for i/o, must return # of bytes read or -1 on error */ -typedef size_t xb_crypt_read_callback(void *userdata, void *buf, size_t len); - -xb_rcrypt_t *xb_crypt_read_open(void *userdata, - xb_crypt_read_callback *onread); - -typedef enum { - XB_CRYPT_READ_CHUNK, - XB_CRYPT_READ_INCOMPLETE, - XB_CRYPT_READ_EOF, - XB_CRYPT_READ_ERROR -} xb_rcrypt_result_t; - -xb_rcrypt_result_t xb_crypt_read_chunk(xb_rcrypt_t *crypt, void **buf, - size_t *olen, size_t *elen, void **iv, - size_t *ivlen, my_bool *hash_appended); - -int xb_crypt_read_close(xb_rcrypt_t *crypt); - -#endif diff --git a/extra/mariabackup/xbcrypt_common.c b/extra/mariabackup/xbcrypt_common.c deleted file mode 100644 index 0cdb54dc66d..00000000000 --- a/extra/mariabackup/xbcrypt_common.c +++ /dev/null @@ -1,328 +0,0 @@ -/****************************************************** -Copyright (c) 2013, 2017 Percona LLC and/or its affiliates. - -Encryption configuration file interface for XtraBackup. - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - -*******************************************************/ - -#include -#include "common.h" -#include "xbcrypt.h" -#include "xbcrypt_common.h" - -/* Encryption options */ -char *ds_encrypt_key = NULL; -char *ds_encrypt_key_file = NULL; -ulong ds_encrypt_algo; - -static uint encrypt_key_len; -static uint encrypt_iv_len; - -static const uint encrypt_mode = GCRY_CIPHER_MODE_CTR; - -static uint encrypt_algos[] = { GCRY_CIPHER_NONE, GCRY_CIPHER_AES128, - GCRY_CIPHER_AES192, GCRY_CIPHER_AES256 }; -static uint encrypt_algo; - -#if !defined(GCRYPT_VERSION_NUMBER) || (GCRYPT_VERSION_NUMBER < 0x010600) -GCRY_THREAD_OPTION_PTHREAD_IMPL; -#endif - - -my_bool -xb_crypt_read_key_file(const char *filename, void** key, uint *keylength) -{ - FILE *fp; - - if (!(fp = my_fopen(filename, O_RDONLY, MYF(0)))) { - msg("%s:%s: unable to open config file \"%s\", errno(%d)\n", - my_progname, __FUNCTION__, filename, my_errno); - return FALSE; - } - - fseek(fp, 0 , SEEK_END); - *keylength = ftell(fp); - rewind(fp); - *key = my_malloc(*keylength, MYF(MY_FAE)); - *keylength = fread(*key, 1, *keylength, fp); - my_fclose(fp, MYF(0)); - return TRUE; -} - -void -xb_crypt_create_iv(void* ivbuf, size_t ivlen) -{ - gcry_create_nonce(ivbuf, ivlen); -} - -gcry_error_t -xb_crypt_init(uint *iv_len) -{ - gcry_error_t gcry_error; - - /* Acording to gcrypt docs (and my testing), setting up the threading - callbacks must be done first, so, lets give it a shot */ -#if !defined(GCRYPT_VERSION_NUMBER) || (GCRYPT_VERSION_NUMBER < 0x010600) - gcry_error = gcry_control(GCRYCTL_SET_THREAD_CBS, &gcry_threads_pthread); - if (gcry_error) { - msg("encryption: unable to set libgcrypt thread cbs - " - "%s : %s\n", - gcry_strsource(gcry_error), - gcry_strerror(gcry_error)); - return gcry_error; - } -#endif - - /* Version check should be the very next call because it - makes sure that important subsystems are intialized. */ - if (!gcry_control(GCRYCTL_ANY_INITIALIZATION_P)) { - const char *gcrypt_version; - gcrypt_version = gcry_check_version(NULL); - /* No other library has already initialized libgcrypt. */ - if (!gcrypt_version) { - msg("encryption: failed to initialize libgcrypt\n"); - return 1; - } else { - msg("encryption: using gcrypt %s\n", gcrypt_version); - } - } - - /* Disable the gcry secure memory, not dealing with this for now */ - gcry_error = gcry_control(GCRYCTL_DISABLE_SECMEM, 0); - if (gcry_error) { - msg("encryption: unable to disable libgcrypt secmem - " - "%s : %s\n", - gcry_strsource(gcry_error), - gcry_strerror(gcry_error)); - return gcry_error; - } - - /* Finalize gcry initialization. */ - gcry_error = gcry_control(GCRYCTL_INITIALIZATION_FINISHED, 0); - if (gcry_error) { - msg("encryption: unable to finish libgcrypt initialization - " - "%s : %s\n", - gcry_strsource(gcry_error), - gcry_strerror(gcry_error)); - return gcry_error; - } - - /* Determine the algorithm */ - encrypt_algo = encrypt_algos[ds_encrypt_algo]; - - /* Set up the iv length */ - encrypt_iv_len = gcry_cipher_get_algo_blklen(encrypt_algo); - xb_a(encrypt_iv_len > 0); - if (iv_len != NULL) { - *iv_len = encrypt_iv_len; - } - - /* Now set up the key */ - if (ds_encrypt_key == NULL && - ds_encrypt_key_file == NULL) { - msg("encryption: no encryption key or key file specified.\n"); - return gcry_error; - } else if (ds_encrypt_key && ds_encrypt_key_file) { - msg("encryption: both encryption key and key file specified.\n"); - return gcry_error; - } else if (ds_encrypt_key_file) { - if (!xb_crypt_read_key_file(ds_encrypt_key_file, - (void**)&ds_encrypt_key, - &encrypt_key_len)) { - msg("encryption: unable to read encryption key file" - " \"%s\".\n", ds_encrypt_key_file); - return gcry_error; - } - } else if (ds_encrypt_key) { - encrypt_key_len = strlen(ds_encrypt_key); - } else { - msg("encryption: no encryption key or key file specified.\n"); - return gcry_error; - } - - return 0; -} - -gcry_error_t -xb_crypt_cipher_open(gcry_cipher_hd_t *cipher_handle) -{ - if (encrypt_algo != GCRY_CIPHER_NONE) { - gcry_error_t gcry_error; - - gcry_error = gcry_cipher_open(cipher_handle, - encrypt_algo, - encrypt_mode, 0); - if (gcry_error) { - msg("encryption: unable to open libgcrypt" - " cipher - %s : %s\n", - gcry_strsource(gcry_error), - gcry_strerror(gcry_error)); - gcry_cipher_close(*cipher_handle); - return gcry_error; - } - - gcry_error = gcry_cipher_setkey(*cipher_handle, - ds_encrypt_key, - encrypt_key_len); - if (gcry_error) { - msg("encryption: unable to set libgcrypt" - " cipher key - %s : %s\n", - gcry_strsource(gcry_error), - gcry_strerror(gcry_error)); - gcry_cipher_close(*cipher_handle); - return gcry_error; - } - return gcry_error; - } - return 0; -} - -void -xb_crypt_cipher_close(gcry_cipher_hd_t cipher_handle) -{ - if (encrypt_algo != GCRY_CIPHER_NONE) - gcry_cipher_close(cipher_handle); -} - -gcry_error_t -xb_crypt_decrypt(gcry_cipher_hd_t cipher_handle, const uchar *from, - size_t from_len, uchar *to, size_t *to_len, - const uchar *iv, size_t iv_len, my_bool hash_appended) -{ - *to_len = from_len; - - if (encrypt_algo != GCRY_CIPHER_NONE) { - - gcry_error_t gcry_error; - - gcry_error = gcry_cipher_reset(cipher_handle); - if (gcry_error) { - msg("%s:encryption: unable to reset libgcrypt" - " cipher - %s : %s\n", my_progname, - gcry_strsource(gcry_error), - gcry_strerror(gcry_error)); - return gcry_error; - } - - if (iv_len > 0) { - gcry_error = gcry_cipher_setctr(cipher_handle, - iv, iv_len); - } - if (gcry_error) { - msg("%s:encryption: unable to set cipher iv - " - "%s : %s\n", my_progname, - gcry_strsource(gcry_error), - gcry_strerror(gcry_error)); - return gcry_error; - } - - /* Try to decrypt it */ - gcry_error = gcry_cipher_decrypt(cipher_handle, to, *to_len, - from, from_len); - if (gcry_error) { - msg("%s:encryption: unable to decrypt chunk - " - "%s : %s\n", my_progname, - gcry_strsource(gcry_error), - gcry_strerror(gcry_error)); - gcry_cipher_close(cipher_handle); - return gcry_error; - } - - if (hash_appended) { - uchar hash[XB_CRYPT_HASH_LEN]; - - *to_len -= XB_CRYPT_HASH_LEN; - - /* ensure that XB_CRYPT_HASH_LEN is the correct length - of XB_CRYPT_HASH hashing algorithm output */ - xb_ad(gcry_md_get_algo_dlen(XB_CRYPT_HASH) == - XB_CRYPT_HASH_LEN); - gcry_md_hash_buffer(XB_CRYPT_HASH, hash, to, - *to_len); - if (memcmp(hash, (char *) to + *to_len, - XB_CRYPT_HASH_LEN) != 0) { - msg("%s:%s invalid plaintext hash. " - "Wrong encrytion key specified?\n", - my_progname, __FUNCTION__); - return 1; - } - } - - } else { - memcpy(to, from, *to_len); - } - - return 0; -} - -gcry_error_t -xb_crypt_encrypt(gcry_cipher_hd_t cipher_handle, const uchar *from, - size_t from_len, uchar *to, size_t *to_len, uchar *iv) -{ - gcry_error_t gcry_error; - - /* ensure that XB_CRYPT_HASH_LEN is the correct length - of XB_CRYPT_HASH hashing algorithm output */ - xb_ad(gcry_md_get_algo_dlen(XB_CRYPT_HASH) == - XB_CRYPT_HASH_LEN); - - memcpy(to, from, from_len); - gcry_md_hash_buffer(XB_CRYPT_HASH, to + from_len, - from, from_len); - - *to_len = from_len; - - if (encrypt_algo != GCRY_CIPHER_NONE) { - - gcry_error = gcry_cipher_reset(cipher_handle); - if (gcry_error) { - msg("encrypt: unable to reset cipher - " - "%s : %s\n", - gcry_strsource(gcry_error), - gcry_strerror(gcry_error)); - return gcry_error; - } - - xb_crypt_create_iv(iv, encrypt_iv_len); - gcry_error = gcry_cipher_setctr(cipher_handle, iv, - encrypt_iv_len); - if (gcry_error) { - msg("encrypt: unable to set cipher ctr - " - "%s : %s\n", - gcry_strsource(gcry_error), - gcry_strerror(gcry_error)); - return gcry_error; - } - - gcry_error = gcry_cipher_encrypt(cipher_handle, to, - *to_len + XB_CRYPT_HASH_LEN, - to, - from_len + XB_CRYPT_HASH_LEN); - if (gcry_error) { - msg("encrypt: unable to encrypt buffer - " - "%s : %s\n", gcry_strsource(gcry_error), - gcry_strerror(gcry_error)); - return gcry_error; - } - } else { - memcpy(to, from, from_len + XB_CRYPT_HASH_LEN); - } - - *to_len += XB_CRYPT_HASH_LEN; - - return 0; -} -#endif \ No newline at end of file diff --git a/extra/mariabackup/xbcrypt_common.h b/extra/mariabackup/xbcrypt_common.h deleted file mode 100644 index 85d13c01fc4..00000000000 --- a/extra/mariabackup/xbcrypt_common.h +++ /dev/null @@ -1,64 +0,0 @@ -/****************************************************** -Copyright (c) 2017 Percona LLC and/or its affiliates. - -Encryption datasink implementation for XtraBackup. - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - -*******************************************************/ - -#include -#if HAVE_GCRYPT -#if GCC_VERSION >= 4002 -/* Workaround to avoid "gcry_ac_* is deprecated" warnings in gcrypt.h */ -# pragma GCC diagnostic ignored "-Wdeprecated-declarations" -#endif - -#include - -extern char *ds_encrypt_key; -extern char *ds_encrypt_key_file; -extern int ds_encrypt_threads; -extern ulong ds_encrypt_algo; - -/****************************************************************************** -Utility interface */ -my_bool xb_crypt_read_key_file(const char *filename, - void** key, uint *keylength); - -void xb_crypt_create_iv(void* ivbuf, size_t ivlen); - -/* Initialize gcrypt and setup encryption key and IV lengths */ -gcry_error_t -xb_crypt_init(uint *iv_len); - -/* Setup gcrypt cipher */ -gcry_error_t -xb_crypt_cipher_open(gcry_cipher_hd_t *cipher_handle); - -/* Close gcrypt cipher */ -void -xb_crypt_cipher_close(gcry_cipher_hd_t cipher_handle); - -/* Decrypt buffer */ -gcry_error_t -xb_crypt_decrypt(gcry_cipher_hd_t cipher_handle, const uchar *from, - size_t from_len, uchar *to, size_t *to_len, const uchar *iv, - size_t iv_len, my_bool hash_appended); - -/* Encrypt buffer */ -gcry_error_t -xb_crypt_encrypt(gcry_cipher_hd_t cipher_handle, const uchar *from, - size_t from_len, uchar *to, size_t *to_len, uchar *iv); -#endif diff --git a/extra/mariabackup/xbcrypt_read.c b/extra/mariabackup/xbcrypt_read.c deleted file mode 100644 index 41790c7035d..00000000000 --- a/extra/mariabackup/xbcrypt_read.c +++ /dev/null @@ -1,252 +0,0 @@ -/****************************************************** -Copyright (c) 2013 Percona LLC and/or its affiliates. - -The xbcrypt format reader implementation. - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - -*******************************************************/ - -#include "xbcrypt.h" -#include "crc_glue.h" - -struct xb_rcrypt_struct { - void *userdata; - xb_crypt_read_callback *read; - void *buffer; - size_t bufsize; - void *ivbuffer; - size_t ivbufsize; - ulonglong offset; -}; - -xb_rcrypt_t * -xb_crypt_read_open(void *userdata, xb_crypt_read_callback *onread) -{ - xb_rcrypt_t *crypt; - - xb_ad(onread); - - crypt = (xb_rcrypt_t *) my_malloc(sizeof(xb_rcrypt_t), MYF(MY_FAE)); - - crypt->userdata = userdata; - crypt->read = onread; - crypt->buffer = NULL; - crypt->bufsize = 0; - crypt->offset = 0; - crypt->ivbuffer = NULL; - crypt->ivbufsize = 0; - return crypt; -} - -xb_rcrypt_result_t -xb_crypt_read_chunk(xb_rcrypt_t *crypt, void **buf, size_t *olen, size_t *elen, - void **iv, size_t *ivlen, my_bool *hash_appended) - -{ - uchar tmpbuf[XB_CRYPT_CHUNK_MAGIC_SIZE + 8 + 8 + 8 + 4]; - uchar *ptr; - ulonglong tmp; - ulong checksum, checksum_exp, version; - size_t bytesread; - xb_rcrypt_result_t result = XB_CRYPT_READ_CHUNK; - - if ((bytesread = crypt->read(crypt->userdata, tmpbuf, sizeof(tmpbuf))) - != sizeof(tmpbuf)) { - if (bytesread == 0) { - result = XB_CRYPT_READ_EOF; - goto err; - } else { - msg("%s:%s: unable to read chunk header data at " - "offset 0x%llx.\n", - my_progname, __FUNCTION__, crypt->offset); - result = XB_CRYPT_READ_ERROR; - goto err; - } - } - - ptr = tmpbuf; - - if (memcmp(ptr, XB_CRYPT_CHUNK_MAGIC3, - XB_CRYPT_CHUNK_MAGIC_SIZE) == 0) { - version = 3; - } else if (memcmp(ptr, XB_CRYPT_CHUNK_MAGIC2, - XB_CRYPT_CHUNK_MAGIC_SIZE) == 0) { - version = 2; - } else if (memcmp(ptr, XB_CRYPT_CHUNK_MAGIC1, - XB_CRYPT_CHUNK_MAGIC_SIZE) == 0) { - version = 1; - } else { - msg("%s:%s: wrong chunk magic at offset 0x%llx.\n", - my_progname, __FUNCTION__, crypt->offset); - result = XB_CRYPT_READ_ERROR; - goto err; - } - - ptr += XB_CRYPT_CHUNK_MAGIC_SIZE; - crypt->offset += XB_CRYPT_CHUNK_MAGIC_SIZE; - - tmp = uint8korr(ptr); /* reserved */ - ptr += 8; - crypt->offset += 8; - - tmp = uint8korr(ptr); /* original size */ - ptr += 8; - if (tmp > INT_MAX) { - msg("%s:%s: invalid original size at offset 0x%llx.\n", - my_progname, __FUNCTION__, crypt->offset); - result = XB_CRYPT_READ_ERROR; - goto err; - } - crypt->offset += 8; - *olen = (size_t)tmp; - - tmp = uint8korr(ptr); /* encrypted size */ - ptr += 8; - if (tmp > INT_MAX) { - msg("%s:%s: invalid encrypted size at offset 0x%llx.\n", - my_progname, __FUNCTION__, crypt->offset); - result = XB_CRYPT_READ_ERROR; - goto err; - } - crypt->offset += 8; - *elen = (size_t)tmp; - - checksum_exp = uint4korr(ptr); /* checksum */ - ptr += 4; - crypt->offset += 4; - - /* iv size */ - if (version == 1) { - *ivlen = 0; - *iv = 0; - } else { - if ((bytesread = crypt->read(crypt->userdata, tmpbuf, 8)) - != 8) { - if (bytesread == 0) { - result = XB_CRYPT_READ_EOF; - goto err; - } else { - msg("%s:%s: unable to read chunk iv size at " - "offset 0x%llx.\n", - my_progname, __FUNCTION__, crypt->offset); - result = XB_CRYPT_READ_ERROR; - goto err; - } - } - - tmp = uint8korr(tmpbuf); - if (tmp > INT_MAX) { - msg("%s:%s: invalid iv size at offset 0x%llx.\n", - my_progname, __FUNCTION__, crypt->offset); - result = XB_CRYPT_READ_ERROR; - goto err; - } - crypt->offset += 8; - *ivlen = (size_t)tmp; - } - - if (*ivlen > crypt->ivbufsize) { - crypt->ivbuffer = my_realloc(crypt->ivbuffer, *ivlen, - MYF(MY_WME | MY_ALLOW_ZERO_PTR)); - if (crypt->ivbuffer == NULL) { - msg("%s:%s: failed to increase iv buffer to " - "%llu bytes.\n", my_progname, __FUNCTION__, - (ulonglong)*ivlen); - result = XB_CRYPT_READ_ERROR; - goto err; - } - crypt->ivbufsize = *ivlen; - } - - if (*ivlen > 0) { - if (crypt->read(crypt->userdata, crypt->ivbuffer, *ivlen) - != *ivlen) { - msg("%s:%s: failed to read %lld bytes for chunk iv " - "at offset 0x%llx.\n", my_progname, __FUNCTION__, - (ulonglong)*ivlen, crypt->offset); - result = XB_CRYPT_READ_ERROR; - goto err; - } - *iv = crypt->ivbuffer; - } - - /* for version euqals 2 we need to read in the iv data but do not init - CTR with it */ - if (version == 2) { - *ivlen = 0; - *iv = 0; - } - - if (*olen > crypt->bufsize) { - crypt->buffer = my_realloc(crypt->buffer, *olen, - MYF(MY_WME | MY_ALLOW_ZERO_PTR)); - if (crypt->buffer == NULL) { - msg("%s:%s: failed to increase buffer to " - "%llu bytes.\n", my_progname, __FUNCTION__, - (ulonglong)*olen); - result = XB_CRYPT_READ_ERROR; - goto err; - } - crypt->bufsize = *olen; - } - - if (*elen > 0) { - if (crypt->read(crypt->userdata, crypt->buffer, *elen) - != *elen) { - msg("%s:%s: failed to read %lld bytes for chunk payload " - "at offset 0x%llx.\n", my_progname, __FUNCTION__, - (ulonglong)*elen, crypt->offset); - result = XB_CRYPT_READ_ERROR; - goto err; - } - } - - checksum = crc32_iso3309(0, crypt->buffer, *elen); - if (checksum != checksum_exp) { - msg("%s:%s invalid checksum at offset 0x%llx, " - "expected 0x%lx, actual 0x%lx.\n", my_progname, __FUNCTION__, - crypt->offset, checksum_exp, checksum); - result = XB_CRYPT_READ_ERROR; - goto err; - } - - crypt->offset += *elen; - *buf = crypt->buffer; - - *hash_appended = version > 2; - - goto exit; - -err: - *buf = NULL; - *olen = 0; - *elen = 0; - *ivlen = 0; - *iv = 0; -exit: - return result; -} - -int xb_crypt_read_close(xb_rcrypt_t *crypt) -{ - if (crypt->buffer) - my_free(crypt->buffer); - if (crypt->ivbuffer) - my_free(crypt->ivbuffer); - my_free(crypt); - - return 0; -} - diff --git a/extra/mariabackup/xbcrypt_write.c b/extra/mariabackup/xbcrypt_write.c deleted file mode 100644 index 91dbfc4eb29..00000000000 --- a/extra/mariabackup/xbcrypt_write.c +++ /dev/null @@ -1,105 +0,0 @@ -/****************************************************** -Copyright (c) 2013 Percona LLC and/or its affiliates. - -The xbcrypt format writer implementation. - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - -*******************************************************/ - -#include "xbcrypt.h" -#include "crc_glue.h" - -struct xb_wcrypt_struct { - void *userdata; - xb_crypt_write_callback *write; -}; - -xb_wcrypt_t * -xb_crypt_write_open(void *userdata, xb_crypt_write_callback *onwrite) -{ - xb_wcrypt_t *crypt; - - xb_ad(onwrite); - - crypt = (xb_wcrypt_t *) my_malloc(sizeof(xb_wcrypt_t), MYF(MY_FAE)); - - crypt->userdata = userdata; - crypt->write = onwrite; - - return crypt; -} - -int xb_crypt_write_chunk(xb_wcrypt_t *crypt, const void *buf, size_t olen, - size_t elen, const void *iv, size_t ivlen) -{ - uchar tmpbuf[XB_CRYPT_CHUNK_MAGIC_SIZE + 8 + 8 + 8 + 4 + 8]; - uchar *ptr; - ulong checksum; - - xb_ad(olen <= INT_MAX); - if (olen > INT_MAX) - return 0; - - xb_ad(elen <= INT_MAX); - if (elen > INT_MAX) - return 0; - - xb_ad(ivlen <= INT_MAX); - if (ivlen > INT_MAX) - return 0; - - ptr = tmpbuf; - - memcpy(ptr, XB_CRYPT_CHUNK_MAGIC_CURRENT, XB_CRYPT_CHUNK_MAGIC_SIZE); - ptr += XB_CRYPT_CHUNK_MAGIC_SIZE; - - int8store(ptr, (ulonglong)0); /* reserved */ - ptr += 8; - - int8store(ptr, (ulonglong)olen); /* original size */ - ptr += 8; - - int8store(ptr, (ulonglong)elen); /* encrypted (actual) size */ - ptr += 8; - - checksum = crc32_iso3309(0, buf, elen); - int4store(ptr, checksum); /* checksum */ - ptr += 4; - - int8store(ptr, (ulonglong)ivlen); /* iv size */ - ptr += 8; - - xb_ad(ptr <= tmpbuf + sizeof(tmpbuf)); - - if (crypt->write(crypt->userdata, tmpbuf, ptr-tmpbuf) == -1) - return 1; - - if (crypt->write(crypt->userdata, iv, ivlen) == -1) - return 1; - - if (crypt->write(crypt->userdata, buf, elen) == -1) - return 1; - - return 0; -} - -int xb_crypt_write_close(xb_wcrypt_t *crypt) -{ - my_free(crypt); - - return 0; -} - - diff --git a/extra/mariabackup/xbstream.c b/extra/mariabackup/xbstream.c index 2cc47ec7273..edfe20a9e3c 100644 --- a/extra/mariabackup/xbstream.c +++ b/extra/mariabackup/xbstream.c @@ -25,9 +25,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA #include #include "common.h" #include "xbstream.h" -#include "xbcrypt_common.h" #include "datasink.h" -#include "ds_decrypt.h" #include "crc_glue.h" #define XBSTREAM_VERSION "1.0" @@ -41,33 +39,18 @@ typedef enum { RUN_MODE_EXTRACT } run_mode_t; -const char *xbstream_encrypt_algo_names[] = -{ "NONE", "AES128", "AES192", "AES256", NullS}; -TYPELIB xbstream_encrypt_algo_typelib= -{array_elements(xbstream_encrypt_algo_names)-1,"", - xbstream_encrypt_algo_names, NULL}; - /* Need the following definitions to avoid linking with ds_*.o and their link dependencies */ datasink_t datasink_archive; datasink_t datasink_xbstream; datasink_t datasink_compress; datasink_t datasink_tmpfile; -datasink_t datasink_encrypt; datasink_t datasink_buffer; static run_mode_t opt_mode; static char * opt_directory = NULL; static my_bool opt_verbose = 0; static int opt_parallel = 1; -static ulong opt_encrypt_algo; -static char *opt_encrypt_key_file = NULL; -static void *opt_encrypt_key = NULL; -static int opt_encrypt_threads = 1; - -enum { - OPT_ENCRYPT_THREADS = 256 -}; static struct my_option my_long_options[] = { @@ -86,20 +69,6 @@ static struct my_option my_long_options[] = {"parallel", 'p', "Number of worker threads for reading / writing.", &opt_parallel, &opt_parallel, 0, GET_INT, REQUIRED_ARG, 1, 1, INT_MAX, 0, 0, 0}, - {"decrypt", 'd', "Decrypt files ending with .xbcrypt.", - &opt_encrypt_algo, &opt_encrypt_algo, &xbstream_encrypt_algo_typelib, - GET_ENUM, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, - {"encrypt-key", 'k', "Encryption key.", - &opt_encrypt_key, &opt_encrypt_key, 0, - GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, - {"encrypt-key-file", 'f', "File which contains encryption key.", - &opt_encrypt_key_file, &opt_encrypt_key_file, 0, - GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, - {"encrypt-threads", OPT_ENCRYPT_THREADS, - "Number of threads for parallel data encryption. " - "The default value is 1.", - &opt_encrypt_threads, &opt_encrypt_threads, - 0, GET_INT, REQUIRED_ARG, 1, 1, INT_MAX, 0, 0, 0}, {0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0} }; @@ -108,7 +77,6 @@ typedef struct { HASH *filehash; xb_rstream_t *stream; ds_ctxt_t *ds_ctxt; - ds_ctxt_t *ds_decrypt_ctxt; pthread_mutex_t *mutex; } extract_ctxt_t; @@ -348,19 +316,6 @@ err: return 1; } -/************************************************************************ -Check if string ends with given suffix. -@return true if string ends with given suffix. */ -static -my_bool -ends_with(const char *str, const char *suffix) -{ - size_t suffix_len = strlen(suffix); - size_t str_len = strlen(str); - return(str_len >= suffix_len - && strcmp(str + str_len - suffix_len, suffix) == 0); -} - static file_entry_t * file_entry_new(extract_ctxt_t *ctxt, const char *path, uint pathlen) @@ -380,11 +335,8 @@ file_entry_new(extract_ctxt_t *ctxt, const char *path, uint pathlen) } entry->pathlen = pathlen; - if (ctxt->ds_decrypt_ctxt && ends_with(path, ".xbcrypt")) { - file = ds_open(ctxt->ds_decrypt_ctxt, path, NULL); - } else { - file = ds_open(ctxt->ds_ctxt, path, NULL); - } + file = ds_open(ctxt->ds_ctxt, path, NULL); + if (file == NULL) { msg("%s: failed to create file.\n", my_progname); goto err; @@ -534,7 +486,6 @@ mode_extract(int n_threads, int argc __attribute__((unused)), xb_rstream_t *stream = NULL; HASH filehash; ds_ctxt_t *ds_ctxt = NULL; - ds_ctxt_t *ds_decrypt_ctxt = NULL; extract_ctxt_t ctxt; int i; pthread_t *tids = NULL; @@ -574,7 +525,6 @@ mode_extract(int n_threads, int argc __attribute__((unused)), ctxt.stream = stream; ctxt.filehash = &filehash; ctxt.ds_ctxt = ds_ctxt; - ctxt.ds_decrypt_ctxt = ds_decrypt_ctxt; ctxt.mutex = &mutex; tids = malloc(sizeof(pthread_t) * n_threads); @@ -604,9 +554,6 @@ exit: if (ds_ctxt != NULL) { ds_destroy(ds_ctxt); } - if (ds_decrypt_ctxt) { - ds_destroy(ds_decrypt_ctxt); - } xb_stream_read_done(stream); return ret; diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index 28de33a5524..baad5b12118 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -59,7 +59,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA #include #include -#include #include #include #include @@ -112,7 +111,6 @@ char xtrabackup_real_target_dir[FN_REFLEN] = "./xtrabackup_backupfiles/"; char *xtrabackup_target_dir= xtrabackup_real_target_dir; my_bool xtrabackup_version = FALSE; my_bool xtrabackup_backup = FALSE; -my_bool xtrabackup_stats = FALSE; my_bool xtrabackup_prepare = FALSE; my_bool xtrabackup_copy_back = FALSE; my_bool xtrabackup_move_back = FALSE; @@ -144,8 +142,6 @@ char xtrabackup_real_incremental_basedir[FN_REFLEN]; char xtrabackup_real_extra_lsndir[FN_REFLEN]; char xtrabackup_real_incremental_dir[FN_REFLEN]; -lsn_t xtrabackup_archived_to_lsn = 0; /* for --archived-to-lsn */ - char *xtrabackup_tmpdir; char *xtrabackup_tables = NULL; @@ -197,21 +193,6 @@ ibool xtrabackup_compress = FALSE; uint xtrabackup_compress_threads; ulonglong xtrabackup_compress_chunk_size = 0; -const char *xtrabackup_encrypt_algo_names[] = -{ "NONE", "AES128", "AES192", "AES256", NullS}; -TYPELIB xtrabackup_encrypt_algo_typelib= -{array_elements(xtrabackup_encrypt_algo_names)-1,"", - xtrabackup_encrypt_algo_names, NULL}; - -ibool xtrabackup_encrypt = FALSE; -ulong xtrabackup_encrypt_algo; -char *xtrabackup_encrypt_key = NULL; -char *xtrabackup_encrypt_key_file = NULL; -uint xtrabackup_encrypt_threads; -ulonglong xtrabackup_encrypt_chunk_size = 0; - -ulint xtrabackup_rebuild_threads = 1; - /* sleep interval beetween log copy iterations in log copying thread in milliseconds (default is 1 second) */ ulint xtrabackup_log_copy_interval = 1000; @@ -271,7 +252,6 @@ are determined in innobase_init below: */ char* innobase_ignored_opt = NULL; char* innobase_data_home_dir = NULL; char* innobase_data_file_path = NULL; -char* innobase_log_arch_dir = NULL;/* unused */ /* The following has a misleading name: starting from 4.0.5, this also affects Windows: */ char* innobase_unix_file_flush_method = NULL; @@ -280,7 +260,6 @@ char* innobase_unix_file_flush_method = NULL; values */ ulong innobase_fast_shutdown = 1; -my_bool innobase_log_archive = FALSE;/* unused */ my_bool innobase_use_doublewrite = TRUE; my_bool innobase_use_checksums = TRUE; my_bool innobase_use_large_pages = FALSE; @@ -303,21 +282,11 @@ ulong innobase_active_counter = 0; static char *xtrabackup_debug_sync = NULL; -my_bool xtrabackup_compact = FALSE; -my_bool xtrabackup_rebuild_indexes = FALSE; - my_bool xtrabackup_incremental_force_scan = FALSE; /* The flushed lsn which is read from data files */ lsn_t flushed_lsn= 0; -/* The size of archived log file */ -ib_int64_t xtrabackup_arch_file_size = 0ULL; -/* The minimal LSN of found archived log files */ -lsn_t xtrabackup_arch_first_file_lsn = 0ULL; -/* The maximum LSN of found archived log files */ -lsn_t xtrabackup_arch_last_file_lsn = 0ULL; - ulong xb_open_files_limit= 0; char *xb_plugin_dir; char *xb_plugin_load; @@ -382,8 +351,6 @@ TYPELIB query_type_typelib= {array_elements(query_type_names) - 1, "", ulong opt_lock_wait_query_type; ulong opt_kill_long_query_type; -ulong opt_decrypt_algo = 0; - uint opt_kill_long_queries_timeout = 0; uint opt_lock_wait_timeout = 0; uint opt_lock_wait_threshold = 0; @@ -391,7 +358,6 @@ uint opt_debug_sleep_before_unlock = 0; uint opt_safe_slave_backup_timeout = 0; const char *opt_history = NULL; -my_bool opt_decrypt = FALSE; #if defined(HAVE_OPENSSL) my_bool opt_ssl_verify_server_cert = FALSE; @@ -495,7 +461,6 @@ enum options_xtrabackup OPT_XTRA_TARGET_DIR = 1000, /* make sure it is larger than OPT_MAX_CLIENT_OPTION */ OPT_XTRA_BACKUP, - OPT_XTRA_STATS, OPT_XTRA_PREPARE, OPT_XTRA_EXPORT, OPT_XTRA_APPLY_LOG_ONLY, @@ -507,7 +472,6 @@ enum options_xtrabackup OPT_XTRA_INCREMENTAL_BASEDIR, OPT_XTRA_EXTRA_LSNDIR, OPT_XTRA_INCREMENTAL_DIR, - OPT_XTRA_ARCHIVED_TO_LSN, OPT_XTRA_TABLES, OPT_XTRA_TABLES_FILE, OPT_XTRA_DATABASES, @@ -518,11 +482,6 @@ enum options_xtrabackup OPT_XTRA_COMPRESS, OPT_XTRA_COMPRESS_THREADS, OPT_XTRA_COMPRESS_CHUNK_SIZE, - OPT_XTRA_ENCRYPT, - OPT_XTRA_ENCRYPT_KEY, - OPT_XTRA_ENCRYPT_KEY_FILE, - OPT_XTRA_ENCRYPT_THREADS, - OPT_XTRA_ENCRYPT_CHUNK_SIZE, OPT_LOG, OPT_INNODB, OPT_INNODB_CHECKSUMS, @@ -535,8 +494,6 @@ enum options_xtrabackup OPT_INNODB_FLUSH_LOG_AT_TRX_COMMIT, OPT_INNODB_FLUSH_METHOD, OPT_INNODB_LOCKS_UNSAFE_FOR_BINLOG, - OPT_INNODB_LOG_ARCH_DIR, - OPT_INNODB_LOG_ARCHIVE, OPT_INNODB_LOG_GROUP_HOME_DIR, OPT_INNODB_MAX_DIRTY_PAGES_PCT, OPT_INNODB_MAX_PURGE_LAG, @@ -568,9 +525,6 @@ enum options_xtrabackup OPT_INNODB_THREAD_CONCURRENCY, OPT_INNODB_THREAD_SLEEP_DELAY, OPT_XTRA_DEBUG_SYNC, - OPT_XTRA_COMPACT, - OPT_XTRA_REBUILD_INDEXES, - OPT_XTRA_REBUILD_THREADS, OPT_INNODB_CHECKSUM_ALGORITHM, OPT_INNODB_UNDO_DIRECTORY, OPT_INNODB_UNDO_TABLESPACES, @@ -597,7 +551,6 @@ enum options_xtrabackup OPT_DECOMPRESS, OPT_INCREMENTAL_HISTORY_NAME, OPT_INCREMENTAL_HISTORY_UUID, - OPT_DECRYPT, OPT_REMOVE_ORIGINAL, OPT_LOCK_WAIT_QUERY_TYPE, OPT_KILL_LONG_QUERY_TYPE, @@ -628,9 +581,6 @@ struct my_option xb_client_options[] = {"backup", OPT_XTRA_BACKUP, "take backup to target-dir", (G_PTR*) &xtrabackup_backup, (G_PTR*) &xtrabackup_backup, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, - {"stats", OPT_XTRA_STATS, "calc statistic of datadir (offline mysqld is recommended)", - (G_PTR*) &xtrabackup_stats, (G_PTR*) &xtrabackup_stats, - 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, {"prepare", OPT_XTRA_PREPARE, "prepare a backup for starting mysql server on the backup.", (G_PTR*) &xtrabackup_prepare, (G_PTR*) &xtrabackup_prepare, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, @@ -669,10 +619,6 @@ struct my_option xb_client_options[] = {"incremental-dir", OPT_XTRA_INCREMENTAL_DIR, "(for --prepare): apply .delta files and logfile in the specified directory.", (G_PTR*) &xtrabackup_incremental_dir, (G_PTR*) &xtrabackup_incremental_dir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, - {"to-archived-lsn", OPT_XTRA_ARCHIVED_TO_LSN, - "Don't apply archived logs with bigger log sequence number.", - (G_PTR*) &xtrabackup_archived_to_lsn, (G_PTR*) &xtrabackup_archived_to_lsn, 0, - GET_LL, REQUIRED_ARG, 0, 0, LONGLONG_MAX, 0, 0, 0}, {"tables", OPT_XTRA_TABLES, "filtering by regexp for table names.", (G_PTR*) &xtrabackup_tables, (G_PTR*) &xtrabackup_tables, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, @@ -724,46 +670,6 @@ struct my_option xb_client_options[] = (G_PTR*) &xtrabackup_compress_chunk_size, (G_PTR*) &xtrabackup_compress_chunk_size, 0, GET_ULL, REQUIRED_ARG, (1 << 16), 1024, ULONGLONG_MAX, 0, 0, 0}, - {"encrypt", OPT_XTRA_ENCRYPT, "Encrypt individual backup files using the " - "specified encryption algorithm.", - &xtrabackup_encrypt_algo, &xtrabackup_encrypt_algo, - &xtrabackup_encrypt_algo_typelib, GET_ENUM, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, - - {"encrypt-key", OPT_XTRA_ENCRYPT_KEY, "Encryption key to use.", - (G_PTR*) &xtrabackup_encrypt_key, (G_PTR*) &xtrabackup_encrypt_key, 0, - GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, - - {"encrypt-key-file", OPT_XTRA_ENCRYPT_KEY_FILE, "File which contains encryption key to use.", - (G_PTR*) &xtrabackup_encrypt_key_file, (G_PTR*) &xtrabackup_encrypt_key_file, 0, - GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, - - {"encrypt-threads", OPT_XTRA_ENCRYPT_THREADS, - "Number of threads for parallel data encryption. The default value is 1.", - (G_PTR*) &xtrabackup_encrypt_threads, (G_PTR*) &xtrabackup_encrypt_threads, - 0, GET_UINT, REQUIRED_ARG, 1, 1, UINT_MAX, 0, 0, 0}, - - {"encrypt-chunk-size", OPT_XTRA_ENCRYPT_CHUNK_SIZE, - "Size of working buffer(S) for encryption threads in bytes. The default value is 64K.", - (G_PTR*) &xtrabackup_encrypt_chunk_size, (G_PTR*) &xtrabackup_encrypt_chunk_size, - 0, GET_ULL, REQUIRED_ARG, (1 << 16), 1024, ULONGLONG_MAX, 0, 0, 0}, - - {"compact", OPT_XTRA_COMPACT, - "Create a compact backup by skipping secondary index pages.", - (G_PTR*) &xtrabackup_compact, (G_PTR*) &xtrabackup_compact, - 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, - - {"rebuild_indexes", OPT_XTRA_REBUILD_INDEXES, - "Rebuild secondary indexes in InnoDB tables after applying the log. " - "Only has effect with --prepare.", - (G_PTR*) &xtrabackup_rebuild_indexes, (G_PTR*) &xtrabackup_rebuild_indexes, - 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, - - {"rebuild_threads", OPT_XTRA_REBUILD_THREADS, - "Use this number of threads to rebuild indexes in a compact backup. " - "Only has effect with --prepare and --rebuild-indexes.", - (G_PTR*) &xtrabackup_rebuild_threads, (G_PTR*) &xtrabackup_rebuild_threads, - 0, GET_UINT, REQUIRED_ARG, 1, 1, UINT_MAX, 0, 0, 0}, - {"incremental-force-scan", OPT_XTRA_INCREMENTAL_FORCE_SCAN, "Perform a full-scan incremental backup even in the presence of changed " "page bitmap data", @@ -934,18 +840,6 @@ struct my_option xb_client_options[] = (uchar*) &opt_incremental_history_uuid, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, - {"decrypt", OPT_DECRYPT, "Decrypts all files with the .xbcrypt " - "extension in a backup previously made with --encrypt option.", - &opt_decrypt_algo, &opt_decrypt_algo, - &xtrabackup_encrypt_algo_typelib, GET_ENUM, REQUIRED_ARG, - 0, 0, 0, 0, 0, 0}, - - {"remove-original", OPT_REMOVE_ORIGINAL, "Remove .qp and .xbcrypt files " - "after decryption and decompression.", - (uchar *) &opt_remove_original, - (uchar *) &opt_remove_original, - 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, - {"ftwrl-wait-query-type", OPT_LOCK_WAIT_QUERY_TYPE, "This option specifies which types of queries are allowed to complete " "before innobackupex will issue the global lock. Default is all.", @@ -1125,9 +1019,6 @@ Disable with --skip-innodb-doublewrite.", (G_PTR*) &innobase_use_doublewrite, (G_PTR*) &innobase_force_recovery, (G_PTR*) &innobase_force_recovery, 0, GET_LONG, REQUIRED_ARG, 0, 0, 6, 0, 1, 0}, - {"innodb_log_arch_dir", OPT_INNODB_LOG_ARCH_DIR, - "Where full logs should be archived.", (G_PTR*) &innobase_log_arch_dir, - (G_PTR*) &innobase_log_arch_dir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"innodb_log_buffer_size", OPT_INNODB_LOG_BUFFER_SIZE, "The size of the buffer which InnoDB uses to write log to the log files on disk.", (G_PTR*) &innobase_log_buffer_size, (G_PTR*) &innobase_log_buffer_size, 0, @@ -1466,24 +1357,6 @@ xb_get_one_option(int optid, } xtrabackup_compress = TRUE; break; - case OPT_XTRA_ENCRYPT: - if (argument == NULL) - { - msg("Missing --encrypt argument, must specify a valid encryption " - " algorithm.\n"); - return 1; - } - xtrabackup_encrypt = TRUE; - break; - case OPT_DECRYPT: - if (argument == NULL) { - msg("Missing --decrypt argument, must specify a " - "valid encryption algorithm.\n"); - return(1); - } - opt_decrypt = TRUE; - xtrabackup_decrypt_decompress = true; - break; case OPT_DECOMPRESS: opt_decompress = TRUE; xtrabackup_decrypt_decompress = true; @@ -1637,7 +1510,7 @@ innodb_init_param(void) /* Set InnoDB initialization parameters according to the values read from MySQL .cnf file */ - if (xtrabackup_backup || xtrabackup_stats) { + if (xtrabackup_backup) { msg("xtrabackup: using the following InnoDB configuration:\n"); } else { msg("xtrabackup: using the following InnoDB configuration " @@ -1648,7 +1521,7 @@ innodb_init_param(void) /* The default dir for data files is the datadir of MySQL */ - srv_data_home = ((xtrabackup_backup || xtrabackup_stats) && innobase_data_home_dir + srv_data_home = (xtrabackup_backup && innobase_data_home_dir ? innobase_data_home_dir : default_path); msg("xtrabackup: innodb_data_home_dir = %s\n", srv_data_home); @@ -1697,8 +1570,7 @@ mem_free_and_error: /* The default dir for log files is the datadir of MySQL */ - if (!((xtrabackup_backup || xtrabackup_stats) && - srv_log_group_home_dir)) { + if (!(xtrabackup_backup && srv_log_group_home_dir)) { srv_log_group_home_dir = default_path; } if (xtrabackup_prepare && xtrabackup_incremental_dir) { @@ -1731,7 +1603,6 @@ mem_free_and_error: msg("xtrabackup: innodb_log_file_size = %lld\n", (long long int) srv_log_file_size); - srv_log_archive_on = (ulint) innobase_log_archive; srv_log_buffer_size = (ulint) innobase_log_buffer_size; /* We set srv_pool_size here in units of 1 kB. InnoDB internally @@ -1760,11 +1631,8 @@ mem_free_and_error: os_use_large_pages = (ibool) innobase_use_large_pages; os_large_page_size = (ulint) innobase_large_page_size; - - if (!innobase_log_arch_dir) { - static char default_dir[3] = "./"; - srv_arch_dir = default_dir; - } + static char default_dir[3] = "./"; + srv_arch_dir = default_dir; row_rollback_on_timeout = (ibool) innobase_rollback_on_timeout; srv_file_per_table = (my_bool) innobase_file_per_table; @@ -1984,13 +1852,11 @@ xtrabackup_print_metadata(char *buf, size_t buf_len) "from_lsn = " UINT64PF "\n" "to_lsn = " UINT64PF "\n" "last_lsn = " UINT64PF "\n" - "compact = %d\n" "recover_binlog_info = %d\n", metadata_type, metadata_from_lsn, metadata_to_lsn, metadata_last_lsn, - MY_TEST(false), MY_TEST(opt_binlog_info == BINLOG_INFO_LOCKLESS)); } @@ -2428,25 +2294,13 @@ xb_get_copy_action(const char *dflt) if (xtrabackup_stream) { if (xtrabackup_compress) { - if (xtrabackup_encrypt) { - action = "Compressing, encrypting and streaming"; - } else { - action = "Compressing and streaming"; - } - } else if (xtrabackup_encrypt) { - action = "Encrypting and streaming"; + action = "Compressing and streaming"; } else { action = "Streaming"; } } else { if (xtrabackup_compress) { - if (xtrabackup_encrypt) { - action = "Compressing and encrypting"; - } else { - action = "Compressing"; - } - } else if (xtrabackup_encrypt) { - action = "Encrypting"; + action = "Compressing"; } else { action = dflt; } @@ -3012,28 +2866,6 @@ xtrabackup_init_datasinks(void) ds_redo = ds_meta = ds_data; } - /* Encryption */ - if (xtrabackup_encrypt) { - ds_ctxt_t *ds; - - - - ds = ds_create(xtrabackup_target_dir, DS_TYPE_ENCRYPT); - xtrabackup_add_datasink(ds); - - ds_set_pipe(ds, ds_data); - if (ds_data != ds_meta) { - ds_data = ds; - ds = ds_create(xtrabackup_target_dir, DS_TYPE_ENCRYPT); - xtrabackup_add_datasink(ds); - - ds_set_pipe(ds, ds_meta); - ds_redo = ds_meta = ds; - } else { - ds_redo = ds_data = ds_meta = ds; - } - } - /* Compression for ds_data and ds_redo */ if (xtrabackup_compress) { ds_ctxt_t *ds; @@ -4243,431 +4075,6 @@ reread_log_header: } } -/* ================= stats ================= */ -static my_bool -xtrabackup_stats_level( - dict_index_t* index, - ulint level) -{ - ulint space; - page_t* page; - - rec_t* node_ptr; - - ulint right_page_no; - - page_cur_t cursor; - - mtr_t mtr; - mem_heap_t* heap = mem_heap_create(256); - - ulint* offsets = NULL; - - ulonglong n_pages, n_pages_extern; - ulonglong sum_data, sum_data_extern; - ulonglong n_recs; - ulint page_size; - buf_block_t* block; - ulint zip_size; - - n_pages = sum_data = n_recs = 0; - n_pages_extern = sum_data_extern = 0; - - - if (level == 0) - fprintf(stdout, " leaf pages: "); - else - fprintf(stdout, " level %lu pages: ", level); - - mtr_start(&mtr); - - mtr_x_lock(&(index->lock), &mtr); - block = btr_root_block_get(index, RW_X_LATCH, &mtr); - page = buf_block_get_frame(block); - - space = page_get_space_id(page); - zip_size = fil_space_get_zip_size(space); - - while (level != btr_page_get_level(page, &mtr)) { - - ut_a(space == buf_block_get_space(block)); - ut_a(space == page_get_space_id(page)); - ut_a(!page_is_leaf(page)); - - page_cur_set_before_first(block, &cursor); - page_cur_move_to_next(&cursor); - - node_ptr = page_cur_get_rec(&cursor); - offsets = rec_get_offsets(node_ptr, index, offsets, - ULINT_UNDEFINED, &heap); - block = btr_node_ptr_get_child(node_ptr, index, offsets, &mtr); - page = buf_block_get_frame(block); - } - -loop: - mem_heap_empty(heap); - offsets = NULL; - mtr_x_lock(&(index->lock), &mtr); - - right_page_no = btr_page_get_next(page, &mtr); - - - /*=================================*/ - //fprintf(stdout, "%lu ", (ulint) buf_frame_get_page_no(page)); - - n_pages++; - sum_data += page_get_data_size(page); - n_recs += page_get_n_recs(page); - - - if (level == 0) { - page_cur_t cur; - ulint n_fields; - ulint i; - mem_heap_t* local_heap = NULL; - ulint offsets_[REC_OFFS_NORMAL_SIZE]; - ulint* local_offsets = offsets_; - - *offsets_ = (sizeof offsets_) / sizeof *offsets_; - - page_cur_set_before_first(block, &cur); - page_cur_move_to_next(&cur); - - for (;;) { - if (page_cur_is_after_last(&cur)) { - break; - } - - local_offsets = rec_get_offsets(cur.rec, index, local_offsets, - ULINT_UNDEFINED, &local_heap); - n_fields = rec_offs_n_fields(local_offsets); - - for (i = 0; i < n_fields; i++) { - if (rec_offs_nth_extern(local_offsets, i)) { - page_t* local_page; - ulint space_id; - ulint page_no; - ulint offset; - byte* blob_header; - ulint part_len; - mtr_t local_mtr; - ulint local_len; - byte* data; - buf_block_t* local_block; - - data = rec_get_nth_field(cur.rec, local_offsets, i, &local_len); - - ut_a(local_len >= BTR_EXTERN_FIELD_REF_SIZE); - local_len -= BTR_EXTERN_FIELD_REF_SIZE; - - space_id = mach_read_from_4(data + local_len + BTR_EXTERN_SPACE_ID); - page_no = mach_read_from_4(data + local_len + BTR_EXTERN_PAGE_NO); - offset = mach_read_from_4(data + local_len + BTR_EXTERN_OFFSET); - - if (offset != FIL_PAGE_DATA) - msg("\nWarning: several record may share same external page.\n"); - - for (;;) { - mtr_start(&local_mtr); - - local_block = btr_block_get(space_id, zip_size, page_no, RW_S_LATCH, index, &local_mtr); - local_page = buf_block_get_frame(local_block); - blob_header = local_page + offset; -#define BTR_BLOB_HDR_PART_LEN 0 -#define BTR_BLOB_HDR_NEXT_PAGE_NO 4 - //part_len = btr_blob_get_part_len(blob_header); - part_len = mach_read_from_4(blob_header + BTR_BLOB_HDR_PART_LEN); - - //page_no = btr_blob_get_next_page_no(blob_header); - page_no = mach_read_from_4(blob_header + BTR_BLOB_HDR_NEXT_PAGE_NO); - - offset = FIL_PAGE_DATA; - - - - - /*=================================*/ - //fprintf(stdout, "[%lu] ", (ulint) buf_frame_get_page_no(page)); - - n_pages_extern++; - sum_data_extern += part_len; - - - mtr_commit(&local_mtr); - - if (page_no == FIL_NULL) - break; - } - } - } - - page_cur_move_to_next(&cur); - } - } - - - - - mtr_commit(&mtr); - if (right_page_no != FIL_NULL) { - mtr_start(&mtr); - block = btr_block_get(space, zip_size, right_page_no, - RW_X_LATCH, index, &mtr); - page = buf_block_get_frame(block); - goto loop; - } - mem_heap_free(heap); - - if (zip_size) { - page_size = zip_size; - } else { - page_size = UNIV_PAGE_SIZE; - } - - if (level == 0) - fprintf(stdout, "recs=%llu, ", n_recs); - - fprintf(stdout, "pages=%llu, data=%llu bytes, data/pages=%lld%%", - n_pages, sum_data, - ((sum_data * 100)/ page_size)/n_pages); - - - if (level == 0 && n_pages_extern) { - putc('\n', stdout); - /* also scan blob pages*/ - fprintf(stdout, " external pages: "); - - fprintf(stdout, "pages=%llu, data=%llu bytes, data/pages=%lld%%", - n_pages_extern, sum_data_extern, - ((sum_data_extern * 100)/ page_size)/n_pages_extern); - } - - putc('\n', stdout); - - if (level > 0) { - xtrabackup_stats_level(index, level - 1); - } - - return(TRUE); -} - -static void -xtrabackup_stats_func(int argc, char **argv) -{ - ulint n; - - /* cd to datadir */ - - if (my_setwd(mysql_real_data_home,MYF(MY_WME))) - { - msg("xtrabackup: cannot my_setwd %s\n", mysql_real_data_home); - exit(EXIT_FAILURE); - } - msg("xtrabackup: cd to %s\n", mysql_real_data_home); - encryption_plugin_prepare_init(argc, argv); - mysql_data_home= mysql_data_home_buff; - mysql_data_home[0]=FN_CURLIB; // all paths are relative from here - mysql_data_home[1]=0; - - srv_n_purge_threads = 1; - /* set read only */ - srv_read_only_mode = TRUE; - - /* initialize components */ - if(innodb_init_param()) - exit(EXIT_FAILURE); - - /* Check if the log files have been created, otherwise innodb_init() - will crash when called with srv_read_only == TRUE */ - for (n = 0; n < srv_n_log_files; n++) { - char logname[FN_REFLEN]; - ibool exists; - os_file_type_t type; - - snprintf(logname, sizeof(logname), "%s%c%s%lu", - srv_log_group_home_dir, SRV_PATH_SEPARATOR, - "ib_logfile", (ulong) n); - srv_normalize_path_for_win(logname); - - if (!os_file_status(logname, &exists, &type) || !exists || - type != OS_FILE_TYPE_FILE) { - msg("xtrabackup: Error: " - "Cannot find log file %s.\n", logname); - msg("xtrabackup: Error: " - "to use the statistics feature, you need a " - "clean copy of the database including " - "correctly sized log files, so you need to " - "execute with --prepare twice to use this " - "functionality on a backup.\n"); - exit(EXIT_FAILURE); - } - } - - msg("xtrabackup: Starting 'read-only' InnoDB instance to gather " - "index statistics.\n" - "xtrabackup: Using %lld bytes for buffer pool (set by " - "--use-memory parameter)\n", xtrabackup_use_memory); - - if(innodb_init()) - exit(EXIT_FAILURE); - - xb_filters_init(); - - fprintf(stdout, "\n\n\n"); - - /* gather stats */ - - { - dict_table_t* sys_tables; - dict_index_t* sys_index; - dict_table_t* table; - btr_pcur_t pcur; - rec_t* rec; - byte* field; - ulint len; - mtr_t mtr; - - /* Enlarge the fatal semaphore wait timeout during the InnoDB table - monitor printout */ - - os_increment_counter_by_amount(server_mutex, - srv_fatal_semaphore_wait_threshold, - 72000); - - mutex_enter(&(dict_sys->mutex)); - - mtr_start(&mtr); - - sys_tables = dict_table_get_low("SYS_TABLES"); - sys_index = UT_LIST_GET_FIRST(sys_tables->indexes); - - btr_pcur_open_at_index_side(TRUE, sys_index, BTR_SEARCH_LEAF, &pcur, - TRUE, 0, &mtr); -loop: - btr_pcur_move_to_next_user_rec(&pcur, &mtr); - - rec = btr_pcur_get_rec(&pcur); - - if (!btr_pcur_is_on_user_rec(&pcur)) - { - /* end of index */ - - btr_pcur_close(&pcur); - mtr_commit(&mtr); - - mutex_exit(&(dict_sys->mutex)); - - /* Restore the fatal semaphore wait timeout */ - os_increment_counter_by_amount(server_mutex, - srv_fatal_semaphore_wait_threshold, - -72000); - - goto end; - } - - field = rec_get_nth_field_old(rec, 0, &len); - - if (!rec_get_deleted_flag(rec, 0)) { - - /* We found one */ - - char* table_name = mem_strdupl((char*) field, len); - - btr_pcur_store_position(&pcur, &mtr); - - mtr_commit(&mtr); - - table = dict_table_get_low(table_name); - mem_free(table_name); - - if (table && check_if_skip_table(table->name)) - goto skip; - - - if (table == NULL) { - fputs("InnoDB: Failed to load table ", stderr); - ut_print_namel(stderr, NULL, TRUE, (char*) field, len); - putc('\n', stderr); - } else { - dict_index_t* index; - - /* The table definition was corrupt if there - is no index */ - - if (dict_table_get_first_index(table)) { - dict_stats_update_transient(table); - } - - //dict_table_print_low(table); - - index = UT_LIST_GET_FIRST(table->indexes); - while (index != NULL) { -{ - ib_int64_t n_vals; - - if (index->n_user_defined_cols > 0) { - n_vals = index->stat_n_diff_key_vals[ - index->n_user_defined_cols]; - } else { - n_vals = index->stat_n_diff_key_vals[1]; - } - - fprintf(stdout, - " table: %s, index: %s, space id: %lu, root page: %lu" - ", zip size: %lu" - "\n estimated statistics in dictionary:\n" - " key vals: %lu, leaf pages: %lu, size pages: %lu\n" - " real statistics:\n", - table->name, index->name, - (ulong) index->space, - (ulong) index->page, - (ulong) fil_space_get_zip_size(index->space), - (ulong) n_vals, - (ulong) index->stat_n_leaf_pages, - (ulong) index->stat_index_size); - - { - mtr_t local_mtr; - page_t* root; - ulint page_level; - - mtr_start(&local_mtr); - - mtr_x_lock(&(index->lock), &local_mtr); - root = btr_root_get(index, &local_mtr); - page_level = btr_page_get_level(root, &local_mtr); - - xtrabackup_stats_level(index, page_level); - - mtr_commit(&local_mtr); - } - - putc('\n', stdout); -} - index = UT_LIST_GET_NEXT(indexes, index); - } - } - -skip: - mtr_start(&mtr); - - btr_pcur_restore_position(BTR_SEARCH_LEAF, &pcur, &mtr); - } - - goto loop; - } - -end: - putc('\n', stdout); - - fflush(stdout); - - xb_filters_free(); - - /* shutdown InnoDB */ - innodb_end(); -} - /* ================= prepare ================= */ static my_bool @@ -6045,74 +5452,6 @@ xb_export_cfg_write( } -/********************************************************************//** -Searches archived log files in archived log directory. The min and max -LSN's of found files as well as archived log file size are stored in -xtrabackup_arch_first_file_lsn, xtrabackup_arch_last_file_lsn and -xtrabackup_arch_file_size respectively. -@return true on success -*/ -static -bool -xtrabackup_arch_search_files( -/*=========================*/ - ib_uint64_t start_lsn) /*!< in: filter out log files - witch does not contain data - with lsn < start_lsn */ -{ - os_file_dir_t dir; - os_file_stat_t fileinfo; - ut_ad(innobase_log_arch_dir); - - dir = os_file_opendir(innobase_log_arch_dir, FALSE); - if (!dir) { - msg("xtrabackup: error: cannot open archived log directory %s\n", - innobase_log_arch_dir); - return false; - } - - while(!os_file_readdir_next_file(innobase_log_arch_dir, - dir, - &fileinfo) ) { - lsn_t log_file_lsn; - char* log_str_end_lsn_ptr; - - if (strncmp(fileinfo.name, - IB_ARCHIVED_LOGS_PREFIX, - sizeof(IB_ARCHIVED_LOGS_PREFIX) - 1)) { - continue; - } - - log_file_lsn = strtoll(fileinfo.name + - sizeof(IB_ARCHIVED_LOGS_PREFIX) - 1, - &log_str_end_lsn_ptr, 10); - - if (*log_str_end_lsn_ptr) { - continue; - } - - if (log_file_lsn + (fileinfo.size - LOG_FILE_HDR_SIZE) < start_lsn) { - continue; - } - - if (!xtrabackup_arch_first_file_lsn || - log_file_lsn < xtrabackup_arch_first_file_lsn) { - xtrabackup_arch_first_file_lsn = log_file_lsn; - } - if (log_file_lsn > xtrabackup_arch_last_file_lsn) { - xtrabackup_arch_last_file_lsn = log_file_lsn; - } - - //TODO: find the more suitable way to extract archived log file - //size - if (fileinfo.size > (ib_int64_t)xtrabackup_arch_file_size) { - xtrabackup_arch_file_size = fileinfo.size; - } - } - - return xtrabackup_arch_first_file_lsn != 0; -} - static void innodb_free_param() @@ -6178,8 +5517,7 @@ xtrabackup_prepare_func(int argc, char ** argv) xtrabackup_target_dir[1]=0; /* - read metadata of target, we don't need metadata reading in the case - archived logs applying + read metadata of target */ sprintf(metadata_path, "%s/%s", xtrabackup_target_dir, XTRABACKUP_METADATA_FILENAME); @@ -6190,38 +5528,33 @@ xtrabackup_prepare_func(int argc, char ** argv) exit(EXIT_FAILURE); } - if (!innobase_log_arch_dir) - { - if (!strcmp(metadata_type, "full-backuped")) { - msg("xtrabackup: This target seems to be not prepared " - "yet.\n"); - } else if (!strcmp(metadata_type, "log-applied")) { - msg("xtrabackup: This target seems to be already " - "prepared with --apply-log-only.\n"); - goto skip_check; - } else if (!strcmp(metadata_type, "full-prepared")) { - msg("xtrabackup: This target seems to be already " - "prepared.\n"); - } else { - msg("xtrabackup: This target seems not to have correct " - "metadata...\n"); - exit(EXIT_FAILURE); - } + if (!strcmp(metadata_type, "full-backuped")) { + msg("xtrabackup: This target seems to be not prepared yet.\n"); + } else if (!strcmp(metadata_type, "log-applied")) { + msg("xtrabackup: This target seems to be already " + "prepared with --apply-log-only.\n"); + goto skip_check; + } else if (!strcmp(metadata_type, "full-prepared")) { + msg("xtrabackup: This target seems to be already prepared.\n"); + } else { + msg("xtrabackup: This target seems not to have correct " + "metadata...\n"); + exit(EXIT_FAILURE); + } - if (xtrabackup_incremental) { - msg("xtrabackup: error: applying incremental backup " - "needs target prepared with --apply-log-only.\n"); - exit(EXIT_FAILURE); - } + if (xtrabackup_incremental) { + msg("xtrabackup: error: applying incremental backup " + "needs target prepared with --apply-log-only.\n"); + exit(EXIT_FAILURE); + } skip_check: - if (xtrabackup_incremental - && metadata_to_lsn != incremental_lsn) { - msg("xtrabackup: error: This incremental backup seems " - "not to be proper for the target.\n" - "xtrabackup: Check 'to_lsn' of the target and " - "'from_lsn' of the incremental.\n"); - exit(EXIT_FAILURE); - } + if (xtrabackup_incremental + && metadata_to_lsn != incremental_lsn) { + msg("xtrabackup: error: This incremental backup seems " + "not to be proper for the target.\n" + "xtrabackup: Check 'to_lsn' of the target and " + "'from_lsn' of the incremental.\n"); + exit(EXIT_FAILURE); } /* Create logfiles for recovery from 'xtrabackup_logfile', before start InnoDB */ @@ -6244,7 +5577,7 @@ skip_check: xb_filters_init(); - if(!innobase_log_arch_dir && xtrabackup_init_temp_log()) + if (xtrabackup_init_temp_log()) goto error_cleanup; if(innodb_init_param()) { @@ -6253,7 +5586,7 @@ skip_check: xb_normalize_init_values(); - if (xtrabackup_incremental || innobase_log_arch_dir) { + if (xtrabackup_incremental) { err = xb_data_files_init(); if (err != DB_SUCCESS) { msg("xtrabackup: error: xb_data_files_init() failed " @@ -6270,7 +5603,7 @@ skip_check: goto error_cleanup; } } - if (xtrabackup_incremental || innobase_log_arch_dir) { + if (xtrabackup_incremental) { xb_data_files_close(); } if (xtrabackup_incremental) { @@ -6306,53 +5639,6 @@ skip_check: srv_n_write_io_threads = 4; } - if (innobase_log_arch_dir) { - srv_arch_dir = innobase_log_arch_dir; - srv_archive_recovery = true; - if (xtrabackup_archived_to_lsn) { - if (xtrabackup_archived_to_lsn < metadata_last_lsn) { - msg("xtrabackup: warning: logs applying lsn " - "limit " UINT64PF " is " - "less than metadata last-lsn " UINT64PF - " and will be set to metadata last-lsn value\n", - xtrabackup_archived_to_lsn, - metadata_last_lsn); - xtrabackup_archived_to_lsn = metadata_last_lsn; - } - if (xtrabackup_archived_to_lsn < flushed_lsn) { - msg("xtrabackup: error: logs applying " - "lsn limit " UINT64PF " is less than " - "min_flushed_lsn " UINT64PF - ", there is nothing to do\n", - xtrabackup_archived_to_lsn, - flushed_lsn); - goto error_cleanup; - } - } - srv_archive_recovery_limit_lsn= xtrabackup_archived_to_lsn; - /* - Unfinished transactions are not rolled back during log applying - as they can be finished at the firther files applyings. - */ - xtrabackup_apply_log_only = srv_apply_log_only = true; - - if (!xtrabackup_arch_search_files(flushed_lsn)) { - goto error_cleanup; - } - - /* - Check if last log file last lsn is big enough to overlap - last scanned lsn read from metadata. - */ - if (xtrabackup_arch_last_file_lsn + - xtrabackup_arch_file_size - - LOG_FILE_HDR_SIZE < metadata_last_lsn) { - msg("xtrabackup: error: there are no enough archived logs " - "to apply\n"); - goto error_cleanup; - } - } - msg("xtrabackup: Starting InnoDB instance for recovery.\n" "xtrabackup: Using %lld bytes for buffer pool " "(set by --use-memory parameter)\n", xtrabackup_use_memory); @@ -6581,9 +5867,6 @@ next_node: exit(EXIT_FAILURE); } - if (innobase_log_arch_dir) - srv_start_lsn = log_sys->lsn = recv_sys->recovered_lsn; - /* Check whether the log is applied enough or not. */ if ((xtrabackup_incremental && srv_start_lsn < incremental_to_lsn) @@ -6756,8 +6039,6 @@ xb_init() if (opt_decompress) { mixed_options[n_mixed_options++] = "--decompress"; - } else if (opt_decrypt) { - mixed_options[n_mixed_options++] = "--decrypt"; } if (xtrabackup_copy_back) { @@ -7220,24 +6501,11 @@ int main(int argc, char **argv) innobase_file_per_table = TRUE; } - if (!xtrabackup_prepare && - (innobase_log_arch_dir || xtrabackup_archived_to_lsn)) { - - /* Default my.cnf can contain innobase_log_arch_dir option set - for server, reset it to allow backup. */ - innobase_log_arch_dir= NULL; - xtrabackup_archived_to_lsn= 0; - msg("xtrabackup: warning: " - "as --innodb-log-arch-dir and --to-archived-lsn can be used " - "only with --prepare they will be reset\n"); - } - /* cannot execute both for now */ { int num = 0; if (xtrabackup_backup) num++; - if (xtrabackup_stats) num++; if (xtrabackup_prepare) num++; if (xtrabackup_copy_back) num++; if (xtrabackup_move_back) num++; @@ -7258,10 +6526,6 @@ int main(int argc, char **argv) if (xtrabackup_backup) xtrabackup_backup_func(); - /* --stats */ - if (xtrabackup_stats) - xtrabackup_stats_func(argc_server,server_defaults); - /* --prepare */ if (xtrabackup_prepare) { xtrabackup_prepare_func(argc_server, server_defaults); diff --git a/extra/mariabackup/xtrabackup.h b/extra/mariabackup/xtrabackup.h index 371466aad2c..eafc848fd43 100644 --- a/extra/mariabackup/xtrabackup.h +++ b/extra/mariabackup/xtrabackup.h @@ -81,7 +81,6 @@ extern char *xtrabackup_tables_exclude; extern char *xtrabackup_databases_exclude; extern ibool xtrabackup_compress; -extern ibool xtrabackup_encrypt; extern my_bool xtrabackup_backup; extern my_bool xtrabackup_prepare; @@ -92,15 +91,10 @@ extern my_bool xtrabackup_decrypt_decompress; extern char *innobase_data_file_path; extern char *innobase_doublewrite_file; -extern char *xtrabackup_encrypt_key; -extern char *xtrabackup_encrypt_key_file; extern longlong innobase_log_file_size; extern long innobase_log_files_in_group; extern longlong innobase_page_size; -extern const char *xtrabackup_encrypt_algo_names[]; -extern TYPELIB xtrabackup_encrypt_algo_typelib; - extern int xtrabackup_parallel; extern my_bool xb_close_files; @@ -113,9 +107,6 @@ extern "C"{ #ifdef __cplusplus } #endif -extern ulong xtrabackup_encrypt_algo; -extern uint xtrabackup_encrypt_threads; -extern ulonglong xtrabackup_encrypt_chunk_size; extern my_bool xtrabackup_export; extern char *xtrabackup_incremental_basedir; extern char *xtrabackup_extra_lsndir; @@ -158,8 +149,6 @@ extern TYPELIB query_type_typelib; extern ulong opt_lock_wait_query_type; extern ulong opt_kill_long_query_type; -extern ulong opt_decrypt_algo; - extern uint opt_kill_long_queries_timeout; extern uint opt_lock_wait_timeout; extern uint opt_lock_wait_threshold; @@ -167,7 +156,6 @@ extern uint opt_debug_sleep_before_unlock; extern uint opt_safe_slave_backup_timeout; extern const char *opt_history; -extern my_bool opt_decrypt; enum binlog_info_enum { BINLOG_INFO_OFF, BINLOG_INFO_LOCKLESS, BINLOG_INFO_ON, BINLOG_INFO_AUTO}; diff --git a/mysql-test/r/func_group.result b/mysql-test/r/func_group.result index df953630a8b..3340328cad6 100644 --- a/mysql-test/r/func_group.result +++ b/mysql-test/r/func_group.result @@ -2448,6 +2448,15 @@ DROP TABLE t1; # End of 10.1 tests # # +# MDEV-13064: assertion `n < m_size' fails in Item::split_sum_func2() +# +create table t1 (i int) engine=MyISAM; +insert into t1 value (1),(2); +select count(*)+sleep(0) from t1; +count(*)+sleep(0) +2 +drop table t1; +# # Start of 10.3 tests # # diff --git a/mysql-test/r/mdl_sync.result b/mysql-test/r/mdl_sync.result index 1e285650c77..3880fc5ef91 100644 --- a/mysql-test/r/mdl_sync.result +++ b/mysql-test/r/mdl_sync.result @@ -3049,3 +3049,21 @@ SET DEBUG_SYNC= 'RESET'; disconnect con1; disconnect con2; disconnect con3; +# +# MDEV-12620 - set lock_wait_timeout = 1;flush tables with read lock; +# lock not released after timeout +# +CREATE TABLE t1(a INT) ENGINE=InnoDB; +SET debug_sync='open_tables_after_open_and_process_table SIGNAL ready WAIT_FOR go'; +SELECT * FROM t1; +connect con1,localhost,root,,; +SET debug_sync='now WAIT_FOR ready'; +SET lock_wait_timeout=1; +FLUSH TABLES WITH READ LOCK; +ERROR HY000: Lock wait timeout exceeded; try restarting transaction +SET debug_sync='now SIGNAL go'; +connection default; +a +SET debug_sync='RESET'; +DROP TABLE t1; +disconnect con1; diff --git a/mysql-test/suite/innodb/t/log_file_size.test b/mysql-test/suite/innodb/t/log_file_size.test index ae6442a7994..14cc889e34b 100644 --- a/mysql-test/suite/innodb/t/log_file_size.test +++ b/mysql-test/suite/innodb/t/log_file_size.test @@ -1,5 +1,5 @@ # Test resizing the InnoDB redo log. ---source include/have_innodb.inc +--source include/innodb_page_size_small.inc # Embedded server tests do not support restarting --source include/not_embedded.inc # DBUG_EXECUTE_IF is needed @@ -24,6 +24,12 @@ call mtr.add_suppression("InnoDB: Unable to open .*ib_logfile0. to check native FLUSH TABLES; --enable_query_log +--let $restart_parameters= --innodb-thread-concurrency=1 --innodb-log-file-size=1m --innodb-log-files-in-group=2 +--source include/restart_mysqld.inc + +--let $restart_parameters= --innodb-thread-concurrency=100 --innodb-log-file-size=10M --innodb-log-files-in-group=2 +--source include/restart_mysqld.inc + CREATE TABLE t1(a INT PRIMARY KEY) ENGINE=InnoDB; BEGIN; INSERT INTO t1 VALUES (42); diff --git a/mysql-test/suite/mariabackup/full_backup.test b/mysql-test/suite/mariabackup/full_backup.test index a79f54c67e4..f987410cf76 100644 --- a/mysql-test/suite/mariabackup/full_backup.test +++ b/mysql-test/suite/mariabackup/full_backup.test @@ -13,7 +13,6 @@ INSERT INTO t VALUES(2); echo # xtrabackup prepare; --disable_result_log exec $XTRABACKUP --prepare --target-dir=$targetdir; -exec $XTRABACKUP --defaults-file=$targetdir/backup-my.cnf --stats --datadir=$targetdir; -- source include/restart_and_restore.inc --enable_result_log diff --git a/mysql-test/suite/mariabackup/incremental_encrypted.test b/mysql-test/suite/mariabackup/incremental_encrypted.test index c379b3c8165..04f4ba5d18a 100644 --- a/mysql-test/suite/mariabackup/incremental_encrypted.test +++ b/mysql-test/suite/mariabackup/incremental_encrypted.test @@ -27,11 +27,6 @@ echo # Prepare full backup, apply incremental one; exec $XTRABACKUP --prepare --apply-log-only --target-dir=$basedir; exec $XTRABACKUP --prepare --target-dir=$basedir --incremental-dir=$incremental_dir; -# stats also can support encryption, but needs plugin-load and plugin variables, they are stored in backup-my.cnf -# We need to prepare again to create log files though. -exec $XTRABACKUP --prepare --target-dir=$basedir; -exec $XTRABACKUP --defaults-file=$basedir/backup-my.cnf --stats --datadir=$basedir; - echo # Restore and check results; let $targetdir=$basedir; -- source include/restart_and_restore.inc diff --git a/mysql-test/suite/mariabackup/xb_file_key_management.test b/mysql-test/suite/mariabackup/xb_file_key_management.test index bc975a7cdc4..be0b15eb387 100644 --- a/mysql-test/suite/mariabackup/xb_file_key_management.test +++ b/mysql-test/suite/mariabackup/xb_file_key_management.test @@ -19,7 +19,6 @@ echo # xtrabackup prepare; --disable_result_log exec $XTRABACKUP --prepare --target-dir=$targetdir; -exec $XTRABACKUP --defaults-file=$targetdir/backup-my.cnf --stats --datadir=$targetdir ; -- source include/restart_and_restore.inc --enable_result_log diff --git a/mysql-test/suite/plugins/r/show_all_plugins.result b/mysql-test/suite/plugins/r/show_all_plugins.result index 4471011b660..dd6cbfce4c4 100644 --- a/mysql-test/suite/plugins/r/show_all_plugins.result +++ b/mysql-test/suite/plugins/r/show_all_plugins.result @@ -26,7 +26,7 @@ three_attempts NOT INSTALLED AUTHENTICATION dialog_examples.so GPL two_questions NOT INSTALLED AUTHENTICATION dialog_examples.so GPL show status like '%libraries%'; Variable_name Value -Opened_plugin_libraries 8 +Opened_plugin_libraries 7 show plugins soname where library = 'ha_example.so'; Name Status Type Library License EXAMPLE NOT INSTALLED STORAGE ENGINE ha_example.so GPL diff --git a/mysql-test/suite/sys_vars/r/sysvars_innodb.result b/mysql-test/suite/sys_vars/r/sysvars_innodb.result index 1cd91a3a61b..b970292ff51 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_innodb.result +++ b/mysql-test/suite/sys_vars/r/sysvars_innodb.result @@ -1778,8 +1778,8 @@ DEFAULT_VALUE 50331648 VARIABLE_SCOPE GLOBAL VARIABLE_TYPE BIGINT VARIABLE_COMMENT Size of each log file in a log group. -NUMERIC_MIN_VALUE 4194304 -NUMERIC_MAX_VALUE 9223372036854775807 +NUMERIC_MIN_VALUE 1048576 +NUMERIC_MAX_VALUE 549755813888 NUMERIC_BLOCK_SIZE 1048576 ENUM_VALUE_LIST NULL READ_ONLY YES diff --git a/mysql-test/t/func_group.test b/mysql-test/t/func_group.test index 4928203e603..d847f7cd5f7 100644 --- a/mysql-test/t/func_group.test +++ b/mysql-test/t/func_group.test @@ -1692,6 +1692,17 @@ DROP TABLE t1; --echo # End of 10.1 tests --echo # +--echo # +--echo # MDEV-13064: assertion `n < m_size' fails in Item::split_sum_func2() +--echo # + +create table t1 (i int) engine=MyISAM; +insert into t1 value (1),(2); + +select count(*)+sleep(0) from t1; + +drop table t1; + --echo # --echo # Start of 10.3 tests --echo # diff --git a/mysql-test/t/mdl_sync.test b/mysql-test/t/mdl_sync.test index 4aa191d3dfc..fbecd6bf547 100644 --- a/mysql-test/t/mdl_sync.test +++ b/mysql-test/t/mdl_sync.test @@ -4075,6 +4075,30 @@ disconnect con2; disconnect con3; +--echo # +--echo # MDEV-12620 - set lock_wait_timeout = 1;flush tables with read lock; +--echo # lock not released after timeout +--echo # +CREATE TABLE t1(a INT) ENGINE=InnoDB; +SET debug_sync='open_tables_after_open_and_process_table SIGNAL ready WAIT_FOR go'; +send SELECT * FROM t1; + +connect (con1,localhost,root,,); +SET debug_sync='now WAIT_FOR ready'; +# lock_wait_timeout should be 0 in 10.3, so that we don't have to wait at all +SET lock_wait_timeout=1; +--error ER_LOCK_WAIT_TIMEOUT +FLUSH TABLES WITH READ LOCK; +SET debug_sync='now SIGNAL go'; + +connection default; +reap; +SET debug_sync='RESET'; +DROP TABLE t1; + +disconnect con1; + + # Check that all connections opened by test cases in this file are really # gone so execution of other tests won't be affected by their presence. --source include/wait_until_count_sessions.inc diff --git a/sql/filesort.cc b/sql/filesort.cc index 21b45e78c6e..8ca6cd04cda 100644 --- a/sql/filesort.cc +++ b/sql/filesort.cc @@ -182,12 +182,6 @@ SORT_INFO *filesort(THD *thd, TABLE *table, Filesort *filesort, outfile= &sort->io_cache; - /* - Release InnoDB's adaptive hash index latch (if holding) before - running a sort. - */ - ha_release_temporary_latches(thd); - my_b_clear(&tempfile); my_b_clear(&buffpek_pointers); buffpek=0; diff --git a/sql/handler.cc b/sql/handler.cc index 66f0bb19482..c74675ed113 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -2019,44 +2019,6 @@ commit_checkpoint_notify_ha(handlerton *hton, void *cookie) } -/** - @details - This function should be called when MySQL sends rows of a SELECT result set - or the EOF mark to the client. It releases a possible adaptive hash index - S-latch held by thd in InnoDB and also releases a possible InnoDB query - FIFO ticket to enter InnoDB. To save CPU time, InnoDB allows a thd to - keep them over several calls of the InnoDB handler interface when a join - is executed. But when we let the control to pass to the client they have - to be released because if the application program uses mysql_use_result(), - it may deadlock on the S-latch if the application on another connection - performs another SQL query. In MySQL-4.1 this is even more important because - there a connection can have several SELECT queries open at the same time. - - @param thd the thread handle of the current connection - - @return - always 0 -*/ - -int ha_release_temporary_latches(THD *thd) -{ - Ha_trx_info *info; - - /* - Note that below we assume that only transactional storage engines - may need release_temporary_latches(). If this will ever become false, - we could iterate on thd->open_tables instead (and remove duplicates - as if (!seen[hton->slot]) { seen[hton->slot]=1; ... }). - */ - for (info= thd->transaction.stmt.ha_list; info; info= info->next()) - { - handlerton *hton= info->ht(); - if (hton && hton->release_temporary_latches) - hton->release_temporary_latches(hton, thd); - } - return 0; -} - /** Check if all storage engines used in transaction agree that after rollback to savepoint it is safe to release MDL locks acquired after diff --git a/sql/handler.h b/sql/handler.h index 7966351d02d..daa6cdd1658 100644 --- a/sql/handler.h +++ b/sql/handler.h @@ -1235,7 +1235,6 @@ struct handlerton enum_binlog_command binlog_command, const char *query, uint query_length, const char *db, const char *table_name); - int (*release_temporary_latches)(handlerton *hton, THD *thd); /* Get log status. @@ -4352,9 +4351,6 @@ int ha_change_key_cache_param(KEY_CACHE *key_cache); int ha_repartition_key_cache(KEY_CACHE *key_cache); int ha_change_key_cache(KEY_CACHE *old_key_cache, KEY_CACHE *new_key_cache); -/* report to InnoDB that control passes to the client */ -int ha_release_temporary_latches(THD *thd); - /* transactions: interface to handlerton functions */ int ha_start_consistent_snapshot(THD *thd); int ha_commit_or_rollback_by_xid(XID *xid, bool commit); diff --git a/sql/item.cc b/sql/item.cc index cfe5f137e75..173444ad580 100644 --- a/sql/item.cc +++ b/sql/item.cc @@ -561,8 +561,7 @@ Item::Item(THD *thd): command => we should check thd->lex->current_select on zero (thd->lex can be uninitialised) */ - if (thd->lex->current_select && - thd->stmt_arena->is_stmt_prepare_or_first_sp_execute()) + if (thd->lex->current_select) { enum_parsing_place place= thd->lex->current_select->parsing_place; @@ -5764,7 +5763,7 @@ bool Item_field::fix_fields(THD *thd, Item **reference) SELECT_LEX *select= thd->lex->current_select; thd->change_item_tree(reference, - select->parsing_place == IN_GROUP_BY && + select->context_analysis_place == IN_GROUP_BY && alias_name_used ? *rf->ref : rf); /* diff --git a/sql/item_windowfunc.cc b/sql/item_windowfunc.cc index 6ab903a81bb..a216930cad6 100644 --- a/sql/item_windowfunc.cc +++ b/sql/item_windowfunc.cc @@ -71,7 +71,7 @@ Item_window_func::fix_fields(THD *thd, Item **ref) { DBUG_ASSERT(fixed == 0); - enum_parsing_place place= thd->lex->current_select->parsing_place; + enum_parsing_place place= thd->lex->current_select->context_analysis_place; if (!(place == SELECT_LIST || place == IN_ORDER_BY)) { diff --git a/sql/sql_cache.cc b/sql/sql_cache.cc index b2927c89fda..89afef4eaf7 100644 --- a/sql/sql_cache.cc +++ b/sql/sql_cache.cc @@ -1,5 +1,5 @@ /* Copyright (c) 2000, 2013, Oracle and/or its affiliates. - Copyright (c) 2010, 2013, Monty Program Ab + Copyright (c) 2010, 2017, MariaDB Corporation This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -1465,12 +1465,6 @@ def_week_frmt: %lu, in_trans: %d, autocommit: %d", (int)flags.in_trans, (int)flags.autocommit)); - /* - Make InnoDB to release the adaptive hash index latch before - acquiring the query cache mutex. - */ - ha_release_temporary_latches(thd); - /* A table- or a full flush operation can potentially take a long time to finish. We choose not to wait for them and skip caching statements diff --git a/sql/sql_class.cc b/sql/sql_class.cc index 69c1770da39..740d5648214 100644 --- a/sql/sql_class.cc +++ b/sql/sql_class.cc @@ -2818,13 +2818,6 @@ int select_send::send_data(List &items) if (thd->killed == ABORT_QUERY) DBUG_RETURN(FALSE); - /* - We may be passing the control from mysqld to the client: release the - InnoDB adaptive hash S-latch to avoid thread deadlocks if it was reserved - by thd - */ - ha_release_temporary_latches(thd); - protocol->prepare_for_resend(); if (protocol->send_result_set_row(&items)) { @@ -2843,13 +2836,6 @@ int select_send::send_data(List &items) bool select_send::send_eof() { - /* - We may be passing the control from mysqld to the client: release the - InnoDB adaptive hash S-latch to avoid thread deadlocks if it was reserved - by thd - */ - ha_release_temporary_latches(thd); - /* Don't send EOF if we're in error condition (which implies we've already sent or are sending an error) diff --git a/sql/sql_insert.cc b/sql/sql_insert.cc index 2dcffa08999..16ab4ee650a 100644 --- a/sql/sql_insert.cc +++ b/sql/sql_insert.cc @@ -1,6 +1,6 @@ /* Copyright (c) 2000, 2016, Oracle and/or its affiliates. - Copyright (c) 2010, 2016, MariaDB + Copyright (c) 2010, 2017, MariaDB This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -3753,9 +3753,6 @@ int select_insert::send_data(List &values) } } - // Release latches in case bulk insert takes a long time - ha_release_temporary_latches(thd); - error= write_record(thd, table, &info); table->auto_increment_field_not_null= FALSE; diff --git a/sql/sql_lex.h b/sql/sql_lex.h index 84ab8df0864..4cf5bccb5c0 100644 --- a/sql/sql_lex.h +++ b/sql/sql_lex.h @@ -911,6 +911,7 @@ public: /* reserved for exists 2 in */ uint select_n_reserved; enum_parsing_place parsing_place; /* where we are parsing expression */ + enum_parsing_place context_analysis_place; /* where we are in prepare */ bool with_sum_func; /* sum function indicator */ ulong table_join_options; diff --git a/sql/sql_reload.cc b/sql/sql_reload.cc index f54c0b88110..c01ad90f5d2 100644 --- a/sql/sql_reload.cc +++ b/sql/sql_reload.cc @@ -253,7 +253,8 @@ bool reload_acl_and_cache(THD *thd, unsigned long long options, NOTE: my_error() has been already called by reopen_tables() within close_cached_tables(). */ - result= 1; + thd->global_read_lock.unlock_global_read_lock(thd); + return 1; } if (thd->global_read_lock.make_global_read_lock_block_commit(thd)) // Killed diff --git a/sql/sql_select.cc b/sql/sql_select.cc index b8297af627b..8e977334593 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -648,17 +648,15 @@ setup_without_group(THD *thd, Ref_ptr_array ref_pointer_array, thd->lex->allow_sum_func|= (nesting_map)1 << select->nest_level; - save_place= thd->lex->current_select->parsing_place; - thd->lex->current_select->parsing_place= IN_ORDER_BY; + save_place= thd->lex->current_select->context_analysis_place; + thd->lex->current_select->context_analysis_place= IN_ORDER_BY; res= res || setup_order(thd, ref_pointer_array, tables, fields, all_fields, order); - thd->lex->current_select->parsing_place= save_place; - thd->lex->allow_sum_func&= ~((nesting_map)1 << select->nest_level); - save_place= thd->lex->current_select->parsing_place; - thd->lex->current_select->parsing_place= IN_GROUP_BY; + thd->lex->allow_sum_func&= ~((nesting_map)1 << select->nest_level); + thd->lex->current_select->context_analysis_place= IN_GROUP_BY; res= res || setup_group(thd, ref_pointer_array, tables, fields, all_fields, group, hidden_group_fields); - thd->lex->current_select->parsing_place= save_place; + thd->lex->current_select->context_analysis_place= save_place; thd->lex->allow_sum_func|= (nesting_map)1 << select->nest_level; res= res || setup_windows(thd, ref_pointer_array, tables, fields, all_fields, win_specs, win_funcs); @@ -712,6 +710,7 @@ JOIN::prepare(TABLE_LIST *tables_init, if (select_lex->handle_derived(thd->lex, DT_PREPARE)) DBUG_RETURN(1); + thd->lex->current_select->context_analysis_place= NO_MATTER; thd->lex->current_select->is_item_list_lookup= 1; /* If we have already executed SELECT, then it have not sense to prevent @@ -801,12 +800,13 @@ JOIN::prepare(TABLE_LIST *tables_init, ref_ptrs= ref_ptr_array_slice(0); - enum_parsing_place save_place= thd->lex->current_select->parsing_place; - thd->lex->current_select->parsing_place= SELECT_LIST; + enum_parsing_place save_place= + thd->lex->current_select->context_analysis_place; + thd->lex->current_select->context_analysis_place= SELECT_LIST; if (setup_fields(thd, ref_ptrs, fields_list, MARK_COLUMNS_READ, &all_fields, 1)) DBUG_RETURN(-1); - thd->lex->current_select->parsing_place= save_place; + thd->lex->current_select->context_analysis_place= save_place; if (setup_without_group(thd, ref_ptrs, tables_list, select_lex->leaf_tables, fields_list, @@ -22213,14 +22213,16 @@ int setup_order(THD *thd, Ref_ptr_array ref_pointer_array, TABLE_LIST *tables, List &fields, List &all_fields, ORDER *order, bool from_window_spec) { - enum_parsing_place parsing_place= thd->lex->current_select->parsing_place; + enum_parsing_place context_analysis_place= + thd->lex->current_select->context_analysis_place; thd->where="order clause"; for (; order; order=order->next) { if (find_order_in_list(thd, ref_pointer_array, tables, order, fields, all_fields, FALSE, from_window_spec)) return 1; - if ((*order->item)->with_window_func && parsing_place != IN_ORDER_BY) + if ((*order->item)->with_window_func && + context_analysis_place != IN_ORDER_BY) { my_error(ER_WINDOW_FUNCTION_IN_WINDOW_SPEC, MYF(0)); return 1; @@ -22262,7 +22264,8 @@ setup_group(THD *thd, Ref_ptr_array ref_pointer_array, TABLE_LIST *tables, List &fields, List &all_fields, ORDER *order, bool *hidden_group_fields, bool from_window_spec) { - enum_parsing_place parsing_place= thd->lex->current_select->parsing_place; + enum_parsing_place context_analysis_place= + thd->lex->current_select->context_analysis_place; *hidden_group_fields=0; ORDER *ord; @@ -22278,14 +22281,14 @@ setup_group(THD *thd, Ref_ptr_array ref_pointer_array, TABLE_LIST *tables, all_fields, TRUE, from_window_spec)) return 1; (*ord->item)->marker= UNDEF_POS; /* Mark found */ - if ((*ord->item)->with_sum_func && parsing_place == IN_GROUP_BY) + if ((*ord->item)->with_sum_func && context_analysis_place == IN_GROUP_BY) { my_error(ER_WRONG_GROUP_FIELD, MYF(0), (*ord->item)->full_name()); return 1; } if ((*ord->item)->with_window_func) { - if (parsing_place == IN_GROUP_BY) + if (context_analysis_place == IN_GROUP_BY) my_error(ER_WRONG_PLACEMENT_OF_WINDOW_FUNCTION, MYF(0)); else my_error(ER_WINDOW_FUNCTION_IN_WINDOW_SPEC, MYF(0)); diff --git a/storage/innobase/btr/btr0bulk.cc b/storage/innobase/btr/btr0bulk.cc index 9ff3bc5f6d1..8a954f9d3c3 100644 --- a/storage/innobase/btr/btr0bulk.cc +++ b/storage/innobase/btr/btr0bulk.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2014, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -991,11 +992,7 @@ BtrBulk::finish(dberr_t err) ut_ad(err == DB_SUCCESS); } -#ifdef UNIV_DEBUG - dict_sync_check check(true); - - ut_ad(!sync_check_iterate(check)); -#endif /* UNIV_DEBUG */ + ut_ad(!sync_check_iterate(dict_sync_check())); ut_ad(err != DB_SUCCESS || btr_validate_index(m_index, NULL, false)); return(err); diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index 05ab07229e3..791939f32dd 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -1870,15 +1870,8 @@ buf_flush_batch( counts */ { ut_ad(flush_type == BUF_FLUSH_LRU || flush_type == BUF_FLUSH_LIST); - -#ifdef UNIV_DEBUG - { - dict_sync_check check(true); - - ut_ad(flush_type != BUF_FLUSH_LIST - || !sync_check_iterate(check)); - } -#endif /* UNIV_DEBUG */ + ut_ad(flush_type == BUF_FLUSH_LRU + || !sync_check_iterate(dict_sync_check())); buf_pool_mutex_enter(buf_pool); diff --git a/storage/innobase/dict/dict0dict.cc b/storage/innobase/dict/dict0dict.cc index 63507e0a2c9..df36a320616 100644 --- a/storage/innobase/dict/dict0dict.cc +++ b/storage/innobase/dict/dict0dict.cc @@ -6114,14 +6114,7 @@ dict_set_corrupted( ut_ad(mutex_own(&dict_sys->mutex)); ut_ad(!dict_table_is_comp(dict_sys->sys_tables)); ut_ad(!dict_table_is_comp(dict_sys->sys_indexes)); - -#ifdef UNIV_DEBUG - { - dict_sync_check check(true); - - ut_ad(!sync_check_iterate(check)); - } -#endif /* UNIV_DEBUG */ + ut_ad(!sync_check_iterate(dict_sync_check())); /* Mark the table as corrupted only if the clustered index is corrupted */ diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index b4e2983dbd3..b0a65fa2c13 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -1869,6 +1869,8 @@ void innobase_srv_conc_exit_innodb( row_prebuilt_t* prebuilt) { + ut_ad(!sync_check_iterate(sync_check())); + #ifdef WITH_WSREP if (wsrep_on(prebuilt->trx->mysql_thd) && wsrep_thd_is_BF(prebuilt->trx->mysql_thd, FALSE)) { @@ -1877,13 +1879,6 @@ innobase_srv_conc_exit_innodb( #endif /* WITH_WSREP */ trx_t* trx = prebuilt->trx; -#ifdef BTR_CUR_HASH_ADAPT -# ifdef UNIV_DEBUG - btrsea_sync_check check(trx->has_search_latch); - - ut_ad(!sync_check_iterate(check)); -# endif /* UNIV_DEBUG */ -#endif /* BTR_CUR_HASH_ADAPT */ /* This is to avoid making an unnecessary function call. */ if (trx->declared_to_be_inside_innodb @@ -1901,13 +1896,7 @@ innobase_srv_conc_force_exit_innodb( /*================================*/ trx_t* trx) /*!< in: transaction handle */ { -#ifdef BTR_CUR_HASH_ADAPT -# ifdef UNIV_DEBUG - btrsea_sync_check check(trx->has_search_latch); - - ut_ad(!sync_check_iterate(check)); -# endif /* UNIV_DEBUG */ -#endif /* BTR_CUR_HASH_ADAPT */ + ut_ad(!sync_check_iterate(sync_check())); /* This is to avoid making an unnecessary function call. */ if (trx->declared_to_be_inside_innodb) { @@ -1994,13 +1983,7 @@ const char* thd_innodb_tmpdir( THD* thd) { -#ifdef BTR_CUR_HASH_ADAPT -# ifdef UNIV_DEBUG - trx_t* trx = thd_to_trx(thd); - btrsea_sync_check check(trx->has_search_latch); - ut_ad(!sync_check_iterate(check)); -# endif /* UNIV_DEBUG */ -#endif /* BTR_CUR_HASH_ADAPT */ + ut_ad(!sync_check_iterate(sync_check())); const char* tmp_dir = THDVAR(thd, tmpdir); @@ -2035,28 +2018,6 @@ thd_to_trx_id( } #endif /* WITH_WSREP */ -/********************************************************************//** -Call this function when mysqld passes control to the client. That is to -avoid deadlocks on the adaptive hash S-latch possibly held by thd. For more -documentation, see handler.cc. -@return 0 */ -inline -int -innobase_release_temporary_latches( -/*===============================*/ - handlerton* hton, /*!< in: handlerton */ - THD* thd) /*!< in: MySQL thread */ -{ - DBUG_ASSERT(hton == innodb_hton_ptr); - - if (!srv_was_started) { - } else if (trx_t* trx __attribute__((unused))= thd_to_trx(thd)) { - trx_assert_no_search_latch(trx); - } - - return(0); -} - /********************************************************************//** Increments innobase_active_counter and every INNOBASE_WAKE_INTERVALth time calls srv_active_wake_master_thread. This function should be used @@ -3384,7 +3345,6 @@ innobase_query_caching_of_table_permitted( return(false); } - trx_assert_no_search_latch(trx); innobase_srv_conc_force_exit_innodb(trx); if (!thd_test_options(thd, OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN) @@ -3692,8 +3652,6 @@ ha_innobase::init_table_handle_for_HANDLER(void) /* Initialize the m_prebuilt struct much like it would be inited in external_lock */ - trx_assert_no_search_latch(m_prebuilt->trx); - innobase_srv_conc_force_exit_innodb(m_prebuilt->trx); /* If the transaction is not started yet, start it */ @@ -3913,9 +3871,6 @@ innobase_init( innobase_hton->flags = HTON_SUPPORTS_EXTENDED_KEYS | HTON_SUPPORTS_FOREIGN_KEYS; - innobase_hton->release_temporary_latches = - innobase_release_temporary_latches; - #ifdef MYSQL_REPLACE_TRX_IN_THD innobase_hton->replace_native_transaction_in_thd = innodb_replace_trx_in_thd; @@ -4808,12 +4763,6 @@ innobase_commit_ordered( trx = check_trx_exists(thd); TrxInInnoDB trx_in_innodb(trx); - /* Since we will reserve the kernel mutex, we must not be holding the - search system latch, or we will disobey the latching order. But we - already released it in innobase_xa_prepare() (if not before), so just - have an assert here.*/ - trx_assert_no_search_latch(trx); - if (!trx_is_registered_for_2pc(trx) && trx_is_started(trx)) { /* We cannot throw error here; instead we will catch this error again in innobase_commit() and report it from there. */ @@ -5025,12 +4974,6 @@ innobase_rollback_trx( DBUG_ENTER("innobase_rollback_trx"); DBUG_PRINT("trans", ("aborting transaction")); - /* Release a possible FIFO ticket and search latch. Since we will - reserve the trx_sys->mutex, we have to release the search system - latch first to obey the latching order. */ - - trx_assert_no_search_latch(trx); - innobase_srv_conc_force_exit_innodb(trx); /* If we had reserved the auto-inc lock for some table (if @@ -6540,37 +6483,19 @@ initialize_auto_increment(dict_table_t* table, const Field* field) dict_table_autoinc_unlock(table); } -/*****************************************************************//** -Creates and opens a handle to a table which already exists in an InnoDB -database. -@return 1 if error, 0 if success */ - +/** Open an InnoDB table +@param[in] name table name +@return error code +@retval 0 on success */ int -ha_innobase::open( -/*==============*/ - const char* name, /*!< in: table name */ - int mode, /*!< in: not used */ - uint test_if_locked) /*!< in: not used */ +ha_innobase::open(const char* name, int, uint) { dict_table_t* ib_table; char norm_name[FN_REFLEN]; - THD* thd; dict_err_ignore_t ignore_err = DICT_ERR_IGNORE_NONE; DBUG_ENTER("ha_innobase::open"); - UT_NOT_USED(mode); - UT_NOT_USED(test_if_locked); - - thd = ha_thd(); - - /* Under some cases MySQL seems to call this function while - holding search latch(es). This breaks the latching order as - we acquire dict_sys->mutex below and leads to a deadlock. */ - if (thd != NULL) { - innobase_release_temporary_latches(ht, thd); - } - normalize_table_name(norm_name, name); m_user_thd = NULL; @@ -6585,6 +6510,7 @@ ha_innobase::open( m_upd_buf_size = 0; char* is_part = is_partition(norm_name); + THD* thd = ha_thd(); /* Check whether FOREIGN_KEY_CHECKS is set to 0. If so, the table can be opened even if some FK indexes are missing. If not, the table @@ -7055,12 +6981,6 @@ ha_innobase::close() { DBUG_ENTER("ha_innobase::close"); - THD* thd = ha_thd(); - - if (thd != NULL) { - innobase_release_temporary_latches(ht, thd); - } - row_prebuilt_free(m_prebuilt, FALSE); if (m_upd_buf != NULL) { @@ -13156,10 +13076,6 @@ create_table_info_t::initialize() parent_trx = check_trx_exists(m_thd); - /* In case MySQL calls this in the middle of a SELECT query, release - possible adaptive hash latch to avoid deadlocks of threads */ - - trx_assert_no_search_latch(parent_trx); DBUG_RETURN(0); } @@ -14012,12 +13928,6 @@ innobase_drop_database( THD* thd = current_thd; - /* In case MySQL calls this in the middle of a SELECT - query, release possible adaptive hash latch to avoid - deadlocks of threads */ - - trx_assert_no_search_latch(check_trx_exists(thd)); - ulint len = 0; char* ptr = strend(path) - 2; @@ -14751,12 +14661,7 @@ ha_innobase::info_low( update_thd(ha_thd()); - /* In case MySQL calls this in the middle of a SELECT query, release - possible adaptive hash latch to avoid deadlocks of threads */ - - m_prebuilt->trx->op_info = (char*)"returning various info to MariaDB"; - - trx_assert_no_search_latch(m_prebuilt->trx); + m_prebuilt->trx->op_info = "returning various info to MariaDB"; ib_table = m_prebuilt->table; DBUG_ASSERT(ib_table->n_ref_count > 0); @@ -15603,12 +15508,7 @@ ha_innobase::update_table_comment( update_thd(ha_thd()); - m_prebuilt->trx->op_info = (char*)"returning table comment"; - - /* In case MySQL calls this in the middle of a SELECT query, release - possible adaptive hash latch to avoid deadlocks of threads */ - - trx_assert_no_search_latch(m_prebuilt->trx); + m_prebuilt->trx->op_info = "returning table comment"; #define SSTR( x ) reinterpret_cast< std::ostringstream & >( \ ( std::ostringstream() << std::dec << x ) ).str() @@ -15664,22 +15564,14 @@ ha_innobase::get_foreign_key_create_info(void) update_thd(ha_thd()); - m_prebuilt->trx->op_info = (char*)"getting info on foreign keys"; - - /* In case MySQL calls this in the middle of a SELECT query, - release possible adaptive hash latch to avoid - deadlocks of threads */ - - trx_assert_no_search_latch(m_prebuilt->trx); - - + m_prebuilt->trx->op_info = "getting info on foreign keys"; /* Output the data to a temporary string */ std::string str = dict_print_info_on_foreign_keys( TRUE, m_prebuilt->trx, m_prebuilt->table); - m_prebuilt->trx->op_info = (char*)""; + m_prebuilt->trx->op_info = ""; /* Allocate buffer for the string */ char* fk_str = (char*) my_malloc(str.length() + 1, MYF(0)); @@ -16682,8 +16574,6 @@ innodb_show_status( trx_t* trx = check_trx_exists(thd); - trx_assert_no_search_latch(trx); - innobase_srv_conc_force_exit_innodb(trx); TrxInInnoDB trx_in_innodb(trx); @@ -17999,12 +17889,6 @@ innobase_xa_prepare( thd_get_xid(thd, (MYSQL_XID*) trx->xid); - /* Release a possible FIFO ticket and search latch. Since we will - reserve the trx_sys->mutex, we have to release the search system - latch first to obey the latching order. */ - - trx_assert_no_search_latch(trx); - innobase_srv_conc_force_exit_innodb(trx); TrxInInnoDB trx_in_innodb(trx); @@ -21333,7 +21217,7 @@ static MYSQL_SYSVAR_LONG(log_buffer_size, innobase_log_buffer_size, static MYSQL_SYSVAR_LONGLONG(log_file_size, innobase_log_file_size, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, "Size of each log file in a log group.", - NULL, NULL, 48*1024*1024L, 4*1024*1024L, LLONG_MAX, 1024*1024L); + NULL, NULL, 48 << 20, 1 << 20, 512ULL << 30, 1 << 20); static MYSQL_SYSVAR_ULONG(log_files_in_group, srv_n_log_files, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, diff --git a/storage/innobase/handler/ha_innodb.h b/storage/innobase/handler/ha_innodb.h index 717781fd65c..f9ad2007a5f 100644 --- a/storage/innobase/handler/ha_innodb.h +++ b/storage/innobase/handler/ha_innodb.h @@ -906,18 +906,6 @@ innodb_base_col_setup_for_stored( /** whether this is a computed virtual column */ #define innobase_is_v_fld(field) ((field)->vcol_info && !(field)->stored_in_db()) -/** Release temporary latches. -Call this function when mysqld passes control to the client. That is to -avoid deadlocks on the adaptive hash S-latch possibly held by thd. For more -documentation, see handler.cc. -@param[in] hton Handlerton. -@param[in] thd MySQL thread. -@return 0 */ -int -innobase_release_temporary_latches( - handlerton* hton, - THD* thd); - /** Always normalize table name to lower case on Windows */ #ifdef _WIN32 #define normalize_table_name(norm_name, name) \ diff --git a/storage/innobase/handler/ha_innopart.cc b/storage/innobase/handler/ha_innopart.cc index 1041871805e..fb6f4b89a41 100644 --- a/storage/innobase/handler/ha_innopart.cc +++ b/storage/innobase/handler/ha_innopart.cc @@ -932,7 +932,6 @@ ha_innopart::open( { dict_table_t* ib_table; char norm_name[FN_REFLEN]; - THD* thd; DBUG_ENTER("ha_innopart::open"); @@ -942,16 +941,11 @@ ha_innopart::open( ut_ad(table->part_info != NULL); m_part_info = table->part_info; } - thd = ha_thd(); /* Under some cases MySQL seems to call this function while holding search latch(es). This breaks the latching order as we acquire dict_sys->mutex below and leads to a deadlock. */ - if (thd != NULL) { - innobase_release_temporary_latches(ht, thd); - } - normalize_table_name(norm_name, name); m_user_thd = NULL; @@ -1017,6 +1011,7 @@ share_error: MONITOR_INC(MONITOR_TABLE_OPEN); bool no_tablespace; + THD* thd = ha_thd(); /* TODO: Should we do this check for every partition during ::open()? */ /* TODO: refactor this in ha_innobase so it can increase code reuse. */ @@ -1372,15 +1367,8 @@ void ha_innopart::clear_ins_upd_nodes() int ha_innopart::close() { - THD* thd; - DBUG_ENTER("ha_innopart::close"); - thd = ha_thd(); - if (thd != NULL) { - innobase_release_temporary_latches(ht, thd); - } - ut_ad(m_pcur_parts == NULL); ut_ad(m_clust_pcur_parts == NULL); close_partitioning(); @@ -3032,11 +3020,6 @@ ha_innopart::records_in_range( m_prebuilt->trx->op_info = (char*)"estimating records in index range"; - /* In case MySQL calls this in the middle of a SELECT query, release - possible adaptive hash latch to avoid deadlocks of threads. */ - - trx_assert_no_search_latch(m_prebuilt->trx); - active_index = keynr; key = table->key_info + active_index; @@ -3171,11 +3154,6 @@ ha_innopart::estimate_rows_upper_bound() m_prebuilt->trx->op_info = "calculating upper bound for table rows"; - /* In case MySQL calls this in the middle of a SELECT query, release - possible adaptive hash latch to avoid deadlocks of threads. */ - - trx_assert_no_search_latch(m_prebuilt->trx); - for (uint i = m_part_info->get_first_used_partition(); i < m_tot_parts; i = m_part_info->get_next_used_partition(i)) { @@ -3287,12 +3265,7 @@ ha_innopart::info_low( update_thd(ha_thd()); - /* In case MySQL calls this in the middle of a SELECT query, release - possible adaptive hash latch to avoid deadlocks of threads. */ - - m_prebuilt->trx->op_info = (char*)"returning various info to MySQL"; - - trx_assert_no_search_latch(m_prebuilt->trx); + m_prebuilt->trx->op_info = "returning various info to MySQL"; ut_ad(m_part_share->get_table_part(0)->n_ref_count > 0); diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index 416dd231e10..1eec22874d1 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -591,7 +591,6 @@ ha_innobase::check_if_supported_inplace_alter( } update_thd(); - trx_assert_no_search_latch(m_prebuilt->trx); /* Change on engine specific table options require rebuild of the table */ diff --git a/storage/innobase/handler/i_s.cc b/storage/innobase/handler/i_s.cc index 36ecc0b8a29..8f04f9d15ee 100644 --- a/storage/innobase/handler/i_s.cc +++ b/storage/innobase/handler/i_s.cc @@ -699,8 +699,7 @@ fill_innodb_trx_from_cache( #ifdef BTR_CUR_HASH_ADAPT /* trx_adaptive_hash_latched */ - OK(fields[IDX_TRX_ADAPTIVE_HASH_LATCHED]->store( - row->trx_has_search_latch, true)); + OK(fields[IDX_TRX_ADAPTIVE_HASH_LATCHED]->store(0, true)); #endif /* BTR_CUR_HASH_ADAPT */ /* trx_is_read_only*/ diff --git a/storage/innobase/include/log0log.ic b/storage/innobase/include/log0log.ic index 82a94265776..1e09c263975 100644 --- a/storage/innobase/include/log0log.ic +++ b/storage/innobase/include/log0log.ic @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2015, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -479,11 +480,11 @@ void log_free_check(void) /*================*/ { -#ifdef UNIV_DEBUG /* During row_log_table_apply(), this function will be called while we are holding some latches. This is OK, as long as we are not holding any latches on buffer blocks. */ +#ifdef UNIV_DEBUG static const latch_level_t latches[] = { SYNC_DICT, /* dict_sys->mutex during commit_try_rebuild() */ @@ -491,13 +492,12 @@ log_free_check(void) commit_try_rebuild() */ SYNC_INDEX_TREE /* index->lock */ }; - - sync_allowed_latches check( - latches, latches + sizeof(latches)/sizeof(*latches)); - - ut_ad(!sync_check_iterate(check)); #endif /* UNIV_DEBUG */ + ut_ad(!sync_check_iterate( + sync_allowed_latches(latches, + latches + UT_ARR_SIZE(latches)))); + if (log_sys->check_flush_or_checkpoint) { log_check_margins(); diff --git a/storage/innobase/include/sync0debug.h b/storage/innobase/include/sync0debug.h index ba697b70e13..ecc742918f0 100644 --- a/storage/innobase/include/sync0debug.h +++ b/storage/innobase/include/sync0debug.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2013, 2015, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -84,10 +84,10 @@ sync_check_find(latch_level_t level); /** Checks that the level array for the current thread is empty. Terminate iteration if the functor returns true. -@param[in,out] functor called for each element. -@return true if the functor returns true */ +@param[in] functor called for each element. +@return true if the functor returns true for any element */ bool -sync_check_iterate(sync_check_functor_t& functor); +sync_check_iterate(const sync_check_functor_t& functor); /** Acquires the debug mutex. We cannot use the mutex defined in sync0sync, because the debug mutex is also acquired in sync0arr while holding the OS diff --git a/storage/innobase/include/sync0types.h b/storage/innobase/include/sync0types.h index ec4503daa64..8a68f3d0cc6 100644 --- a/storage/innobase/include/sync0types.h +++ b/storage/innobase/include/sync0types.h @@ -1078,108 +1078,43 @@ struct latch_t { /** Subclass this to iterate over a thread's acquired latch levels. */ struct sync_check_functor_t { - virtual ~sync_check_functor_t() { } - virtual bool operator()(const latch_level_t) = 0; - virtual bool result() const = 0; + virtual bool operator()(const latch_level_t) const = 0; }; -#ifdef BTR_CUR_HASH_ADAPT -/** Functor to check whether the calling thread owns the btr search mutex. */ -struct btrsea_sync_check : public sync_check_functor_t { - - /** Constructor - @param[in] has_search_latch true if owns the latch */ - explicit btrsea_sync_check(bool has_search_latch) - : - m_result(), - m_has_search_latch(has_search_latch) { } - - /** Destructor */ - virtual ~btrsea_sync_check() { } - - /** Called for every latch owned by the calling thread. - @param[in] level Level of the existing latch - @return true if the predicate check is successful */ - virtual bool operator()(const latch_level_t level) - { - /* If calling thread doesn't hold search latch then - check if there are latch level exception provided. */ - - if (!m_has_search_latch - && (level != SYNC_SEARCH_SYS - && level != SYNC_FTS_CACHE)) { - - m_result = true; - - return(m_result); - } - - return(false); - } - - /** @return result from the check */ - virtual bool result() const - { - return(m_result); - } - -private: - /** True if all OK */ - bool m_result; - - /** If the caller owns the search latch */ - const bool m_has_search_latch; -}; -#endif /* BTR_CUR_HASH_ADAPT */ - -/** Functor to check for dictionay latching constraints. */ -struct dict_sync_check : public sync_check_functor_t { - - /** Constructor - @param[in] dict_mutex_allow true if the dict mutex - is allowed */ - explicit dict_sync_check(bool dict_mutex_allowed) - : - m_result(), - m_dict_mutex_allowed(dict_mutex_allowed) { } - - /** Destructor */ - virtual ~dict_sync_check() { } - +/** Check that no latch is being held. +@tparam some_allowed whether some latches are allowed to be held */ +template +struct sync_checker : public sync_check_functor_t +{ /** Check the latching constraints - @param[in] level The level held by the thread */ - virtual bool operator()(const latch_level_t level) + @param[in] level The level held by the thread + @return whether a latch violation was detected */ + bool operator()(const latch_level_t level) const { - if (!m_dict_mutex_allowed - || (level != SYNC_DICT - && level != SYNC_DICT_OPERATION - && level != SYNC_FTS_CACHE - /* This only happens in recv_apply_hashed_log_recs. */ - && level != SYNC_RECV_WRITER - && level != SYNC_NO_ORDER_CHECK)) { - - m_result = true; - - return(true); + if (some_allowed) { + switch (level) { + case SYNC_RECV_WRITER: + /* This only happens in + recv_apply_hashed_log_recs. */ + case SYNC_DICT: + case SYNC_DICT_OPERATION: + case SYNC_FTS_CACHE: + case SYNC_NO_ORDER_CHECK: + return(false); + default: + return(true); + } } - return(false); + return(true); } - - /** @return the result of the check */ - virtual bool result() const - { - return(m_result); - } - -private: - /** True if all OK */ - bool m_result; - - /** True if it is OK to hold the dict mutex */ - const bool m_dict_mutex_allowed; }; +/** The strict latch checker (no InnoDB latches may be held) */ +typedef struct sync_checker sync_check; +/** The sloppy latch checker (can hold InnoDB dictionary or SQL latches) */ +typedef struct sync_checker dict_sync_check; + /** Functor to check for given latching constraints. */ struct sync_allowed_latches : public sync_check_functor_t { @@ -1189,9 +1124,7 @@ struct sync_allowed_latches : public sync_check_functor_t { sync_allowed_latches( const latch_level_t* from, const latch_level_t* to) - : - m_result(), - m_latches(from, to) { } + : begin(from), end(to) { } /** Checks whether the given latch_t violates the latch constraint. This object maintains a list of allowed latch levels, and if the given @@ -1199,41 +1132,17 @@ struct sync_allowed_latches : public sync_check_functor_t { then it is a violation. @param[in] latch The latch level to check - @return true if there is a latch ordering violation */ - virtual bool operator()(const latch_level_t level) + @return true if there is a latch violation */ + bool operator()(const latch_level_t level) const { - for (latches_t::const_iterator it = m_latches.begin(); - it != m_latches.end(); - ++it) { - - if (level == *it) { - - m_result = false; - - /* No violation */ - return(false); - } - } - - return(true); - } - - /** @return the result of the check */ - virtual bool result() const - { - return(m_result); + return(std::find(begin, end, level) == end); } private: - /** Save the result of validation check here - True if all OK */ - bool m_result; - - typedef std::vector > - latches_t; - - /** List of latch levels that are allowed to be held */ - latches_t m_latches; + /** First element in an array of allowed latch levels */ + const latch_level_t* const begin; + /** First element after the end of the array of allowed latch levels */ + const latch_level_t* const end; }; /** Get the latch id from a latch name. diff --git a/storage/innobase/include/trx0i_s.h b/storage/innobase/include/trx0i_s.h index 17a297527af..e02c5d88a29 100644 --- a/storage/innobase/include/trx0i_s.h +++ b/storage/innobase/include/trx0i_s.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2007, 2015, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -162,10 +163,6 @@ struct i_s_trx_row_t { /*!< check_foreigns in trx_t */ const char* trx_foreign_key_error; /*!< detailed_error in trx_t */ -#ifdef BTR_CUR_HASH_ADAPT - ibool trx_has_search_latch; - /*!< has_search_latch in trx_t */ -#endif /* BTR_CUR_HASH_ADAPT */ ulint trx_is_read_only; /*!< trx_t::read_only */ ulint trx_is_autocommit_non_locking; diff --git a/storage/innobase/include/trx0trx.h b/storage/innobase/include/trx0trx.h index 6274dec9f9d..b2d4952318c 100644 --- a/storage/innobase/include/trx0trx.h +++ b/storage/innobase/include/trx0trx.h @@ -58,15 +58,6 @@ class FlushObserver; /** Dummy session used currently in MySQL interface */ extern sess_t* trx_dummy_sess; -#ifdef BTR_CUR_HASH_ADAPT -/** Assert that the transaction is not holding the adaptive hash index latch. -@param[in] trx transaction */ -# define trx_assert_no_search_latch(trx) \ - ut_ad(!trx->has_search_latch) -#else /* BTR_CUR_HASH_ADAPT */ -# define trx_assert_no_search_latch(trx) -#endif - /** Set flush observer for the transaction @param[in/out] trx transaction struct @param[in] observer flush observer */ @@ -1072,11 +1063,6 @@ struct trx_t { flush the log in trx_commit_complete_for_mysql() */ ulint duplicates; /*!< TRX_DUP_IGNORE | TRX_DUP_REPLACE */ -#ifdef BTR_CUR_HASH_ADAPT - bool has_search_latch; - /*!< TRUE if this trx has latched the - search system latch in S-mode */ -#endif /* BTR_CUR_HASH_ADAPT */ trx_dict_op_t dict_operation; /**< @see enum trx_dict_op_t */ /* Fields protected by the srv_conc_mutex. */ @@ -1508,17 +1494,11 @@ private: } /* Avoid excessive mutex acquire/release */ - if (++trx->in_depth > 1) { + if (trx->in_depth++) { /* The transaction is already inside InnoDB. */ - ut_ad(trx->in_depth > 1); return; } - /* Only the owning thread should release the latch. */ - - ut_ad(trx->in_depth == 1); - trx_assert_no_search_latch(trx); - trx_mutex_enter(trx); wait(trx); @@ -1543,16 +1523,10 @@ private: ut_ad(trx->in_depth > 0); - if (--trx->in_depth > 0) { - ut_ad(trx->in_depth); + if (--trx->in_depth) { return; } - /* Only the owning thread should release the latch. */ - - ut_ad(trx->in_depth == 0); - trx_assert_no_search_latch(trx); - trx_mutex_enter(trx); ut_ad((trx->in_innodb & TRX_FORCE_ROLLBACK_MASK) > 0); diff --git a/storage/innobase/log/log0log.cc b/storage/innobase/log/log0log.cc index d8e4ba6fe98..4f924e064a8 100644 --- a/storage/innobase/log/log0log.cc +++ b/storage/innobase/log/log0log.cc @@ -677,14 +677,13 @@ log_set_capacity() { lsn_t margin; ulint free; - bool success = true; - lsn_t smallest_capacity; - log_mutex_enter(); - - smallest_capacity = log_group_get_capacity(&log_sys->log); + lsn_t smallest_capacity = ((srv_log_file_size_requested + << srv_page_size_shift) + - LOG_FILE_HDR_SIZE) + * srv_n_log_files; /* Add extra safety */ - smallest_capacity = smallest_capacity - smallest_capacity / 10; + smallest_capacity -= smallest_capacity / 10; /* For each OS thread we must reserve so much free space in the smallest log group that it can accommodate the log entries produced @@ -694,15 +693,20 @@ log_set_capacity() free = LOG_CHECKPOINT_FREE_PER_THREAD * (10 + srv_thread_concurrency) + LOG_CHECKPOINT_EXTRA_FREE; if (free >= smallest_capacity / 2) { - success = false; - - goto failure; - } else { - margin = smallest_capacity - free; + ib::error() << "Cannot continue operation. ib_logfiles are too" + " small for innodb_thread_concurrency=" + << srv_thread_concurrency << ". The combined size of" + " ib_logfiles should be bigger than" + " 200 kB * innodb_thread_concurrency. " + << INNODB_PARAMETERS_MSG; + return(false); } + margin = smallest_capacity - free; margin = margin - margin / 10; /* Add still some extra safety */ + log_mutex_enter(); + log_sys->log_group_capacity = smallest_capacity; log_sys->max_modified_age_async = margin @@ -714,19 +718,9 @@ log_set_capacity() / LOG_POOL_CHECKPOINT_RATIO_ASYNC; log_sys->max_checkpoint_age = margin; -failure: log_mutex_exit(); - if (!success) { - ib::error() << "Cannot continue operation. ib_logfiles are too" - " small for innodb_thread_concurrency=" - << srv_thread_concurrency << ". The combined size of" - " ib_logfiles should be bigger than" - " 200 kB * innodb_thread_concurrency. " - << INNODB_PARAMETERS_MSG; - } - - return(success); + return(true); } /** Initializes the redo logging subsystem. */ diff --git a/storage/innobase/row/row0sel.cc b/storage/innobase/row/row0sel.cc index 3bae9bd25ac..fd0205ad153 100644 --- a/storage/innobase/row/row0sel.cc +++ b/storage/innobase/row/row0sel.cc @@ -2231,16 +2231,7 @@ stop_for_a_while: btr_pcur_store_position(&(plan->pcur), &mtr); mtr_commit(&mtr); - -#ifdef BTR_CUR_HASH_ADAPT -# ifdef UNIV_DEBUG - { - btrsea_sync_check check(true); - - ut_ad(!sync_check_iterate(check)); - } -# endif /* UNIV_DEBUG */ -#endif /* BTR_CUR_HASH_ADAPT */ + ut_ad(!sync_check_iterate(sync_check())); err = DB_SUCCESS; goto func_exit; @@ -2258,14 +2249,7 @@ commit_mtr_for_a_while: mtr_commit(&mtr); mtr_has_extra_clust_latch = FALSE; - -#ifdef UNIV_DEBUG - { - dict_sync_check check(true); - - ut_ad(!sync_check_iterate(check)); - } -#endif /* UNIV_DEBUG */ + ut_ad(!sync_check_iterate(dict_sync_check())); goto table_loop; @@ -2280,20 +2264,13 @@ lock_wait_or_error: mtr_commit(&mtr); -#ifdef UNIV_DEBUG - { - dict_sync_check check(true); - - ut_ad(!sync_check_iterate(check)); - } -#endif /* UNIV_DEBUG */ - func_exit: #ifdef BTR_CUR_HASH_ADAPT if (search_latch_locked) { btr_search_s_unlock(index); } #endif /* BTR_CUR_HASH_ADAPT */ + ut_ad(!sync_check_iterate(dict_sync_check())); if (heap != NULL) { mem_heap_free(heap); @@ -3041,7 +3018,6 @@ row_sel_store_mysql_field_func( mem_heap_t* heap; /* Copy an externally stored field to a temporary heap */ - trx_assert_no_search_latch(prebuilt->trx); ut_ad(field_no == templ->clust_rec_field_no); ut_ad(templ->type != DATA_POINT); @@ -3928,11 +3904,7 @@ row_sel_try_search_shortcut_for_mysql( ut_ad(!prebuilt->templ_contains_blob); btr_pcur_open_with_no_init(index, search_tuple, PAGE_CUR_GE, - BTR_SEARCH_LEAF, pcur, - (trx->has_search_latch) - ? RW_S_LATCH - : 0, - mtr); + BTR_SEARCH_LEAF, pcur, RW_S_LATCH, mtr); rec = btr_pcur_get_rec(pcur); if (!page_rec_is_user_rec(rec)) { @@ -4190,14 +4162,7 @@ row_search_mvcc( DBUG_RETURN(DB_END_OF_INDEX); } -#ifdef BTR_CUR_HASH_ADAPT -# ifdef UNIV_DEBUG - { - btrsea_sync_check check(trx->has_search_latch); - ut_ad(!sync_check_iterate(check)); - } -# endif /* UNIV_DEBUG */ -#endif /* BTR_CUR_HASH_ADAPT */ + ut_ad(!sync_check_iterate(sync_check())); if (dict_table_is_discarded(prebuilt->table)) { @@ -4222,8 +4187,6 @@ row_search_mvcc( && (prebuilt->read_just_key || prebuilt->m_read_virtual_key); - trx_assert_no_search_latch(trx); - /* Reset the new record lock info if srv_locks_unsafe_for_binlog is set or session is using a READ COMMITED isolation level. Then we are able to remove the record locks set here on an individual @@ -4378,9 +4341,7 @@ row_search_mvcc( and if we try that, we can deadlock on the adaptive hash index semaphore! */ - trx_assert_no_search_latch(trx); rw_lock_s_lock(btr_get_search_latch(index)); - trx->has_search_latch = true; switch (row_sel_try_search_shortcut_for_mysql( &rec, prebuilt, &offsets, &heap, @@ -4435,7 +4396,6 @@ row_search_mvcc( err = DB_SUCCESS; rw_lock_s_unlock(btr_get_search_latch(index)); - trx->has_search_latch = false; goto func_exit; @@ -4446,7 +4406,6 @@ row_search_mvcc( err = DB_RECORD_NOT_FOUND; rw_lock_s_unlock(btr_get_search_latch(index)); - trx->has_search_latch = false; /* NOTE that we do NOT store the cursor position */ @@ -4464,7 +4423,6 @@ row_search_mvcc( mtr_start(&mtr); rw_lock_s_unlock(btr_get_search_latch(index)); - trx->has_search_latch = false; } } #endif /* BTR_CUR_HASH_ADAPT */ @@ -4472,8 +4430,6 @@ row_search_mvcc( /*-------------------------------------------------------------*/ /* PHASE 3: Open or restore index cursor position */ - trx_assert_no_search_latch(trx); - spatial_search = dict_index_is_spatial(index) && mode >= PAGE_CUR_CONTAIN; @@ -5787,15 +5743,7 @@ func_exit: } } -#ifdef BTR_CUR_HASH_ADAPT -# ifdef UNIV_DEBUG - { - btrsea_sync_check check(trx->has_search_latch); - - ut_ad(!sync_check_iterate(check)); - } -# endif /* UNIV_DEBUG */ -#endif /* BTR_CUR_HASH_ADAPT */ + ut_ad(!sync_check_iterate(sync_check())); DEBUG_SYNC_C("innodb_row_search_for_mysql_exit"); diff --git a/storage/innobase/srv/srv0conc.cc b/storage/innobase/srv/srv0conc.cc index bf8a326a633..9f589b57d9c 100644 --- a/storage/innobase/srv/srv0conc.cc +++ b/storage/innobase/srv/srv0conc.cc @@ -197,11 +197,6 @@ srv_conc_enter_innodb_with_atomics( (void) my_atomic_addlint( &srv_conc.n_waiting, 1); - /* Release possible search system latch this - thread has */ - - trx_assert_no_search_latch(trx); - thd_wait_begin(trx->mysql_thd, THD_WAIT_USER_LOCK); notified_mysql = TRUE; @@ -257,15 +252,7 @@ srv_conc_enter_innodb( { trx_t* trx = prebuilt->trx; -#ifdef BTR_CUR_HASH_ADAPT -# ifdef UNIV_DEBUG - { - btrsea_sync_check check(trx->has_search_latch); - - ut_ad(!sync_check_iterate(check)); - } -# endif /* UNIV_DEBUG */ -#endif /* BTR_CUR_HASH_ADAPT */ + ut_ad(!sync_check_iterate(sync_check())); srv_conc_enter_innodb_with_atomics(trx); } @@ -279,15 +266,7 @@ srv_conc_force_enter_innodb( trx_t* trx) /*!< in: transaction object associated with the thread */ { -#ifdef BTR_CUR_HASH_ADAPT -# ifdef UNIV_DEBUG - { - btrsea_sync_check check(trx->has_search_latch); - - ut_ad(!sync_check_iterate(check)); - } -# endif /* UNIV_DEBUG */ -#endif /* BTR_CUR_HASH_ADAPT */ + ut_ad(!sync_check_iterate(sync_check())); if (!srv_thread_concurrency) { @@ -320,15 +299,7 @@ srv_conc_force_exit_innodb( srv_conc_exit_innodb_with_atomics(trx); -#ifdef BTR_CUR_HASH_ADAPT -# ifdef UNIV_DEBUG - { - btrsea_sync_check check(trx->has_search_latch); - - ut_ad(!sync_check_iterate(check)); - } -# endif /* UNIV_DEBUG */ -#endif /* BTR_CUR_HASH_ADAPT */ + ut_ad(!sync_check_iterate(sync_check())); } /*********************************************************************//** diff --git a/storage/innobase/sync/sync0debug.cc b/storage/innobase/sync/sync0debug.cc index 11743e14be2..e43e87eb14e 100644 --- a/storage/innobase/sync/sync0debug.cc +++ b/storage/innobase/sync/sync0debug.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2014, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -298,29 +298,23 @@ struct LatchDebug { } /** Iterate over a thread's latches. - @param[in,out] functor The callback + @param[in] functor The callback @return true if the functor returns true. */ - bool for_each(sync_check_functor_t& functor) + bool for_each(const sync_check_functor_t& functor) UNIV_NOTHROW { - const Latches* latches = thread_latches(); + if (const Latches* latches = thread_latches()) { + Latches::const_iterator end = latches->end(); + for (Latches::const_iterator it = latches->begin(); + it != end; ++it) { - if (latches == 0) { - return(functor.result()); - } - - Latches::const_iterator end = latches->end(); - - for (Latches::const_iterator it = latches->begin(); - it != end; - ++it) { - - if (functor(it->m_level)) { - break; + if (functor(it->m_level)) { + return(true); + } } } - return(functor.result()); + return(false); } /** Removes a latch from the thread level array if it is found there. @@ -1215,13 +1209,12 @@ sync_check_find(latch_level_t level) /** Iterate over the thread's latches. @param[in,out] functor called for each element. -@return false if the sync debug hasn't been initialised -@return the value returned by the functor */ +@return true if the functor returns true for any element */ bool -sync_check_iterate(sync_check_functor_t& functor) +sync_check_iterate(const sync_check_functor_t& functor) { - if (LatchDebug::instance() != NULL) { - return(LatchDebug::instance()->for_each(functor)); + if (LatchDebug* debug = LatchDebug::instance()) { + return(debug->for_each(functor)); } return(false); diff --git a/storage/innobase/trx/trx0i_s.cc b/storage/innobase/trx/trx0i_s.cc index fcc50b8c76d..7854ad2ab5a 100644 --- a/storage/innobase/trx/trx0i_s.cc +++ b/storage/innobase/trx/trx0i_s.cc @@ -589,10 +589,6 @@ thd_done: row->trx_foreign_key_error = NULL; } -#ifdef BTR_CUR_HASH_ADAPT - row->trx_has_search_latch = (ibool) trx->has_search_latch; -#endif /* BTR_CUR_HASH_ADAPT */ - row->trx_is_read_only = trx->read_only; row->trx_is_autocommit_non_locking = trx_is_autocommit_non_locking(trx); diff --git a/storage/innobase/trx/trx0trx.cc b/storage/innobase/trx/trx0trx.cc index 315697403c4..7fdbd808a60 100644 --- a/storage/innobase/trx/trx0trx.cc +++ b/storage/innobase/trx/trx0trx.cc @@ -272,8 +272,6 @@ struct TrxFactory { ut_a(trx->lock.wait_lock == NULL); ut_a(trx->lock.wait_thr == NULL); - - trx_assert_no_search_latch(trx); ut_a(trx->dict_operation_lock_mode == 0); if (trx->lock.lock_heap != NULL) { @@ -341,9 +339,6 @@ struct TrxFactory { ut_a(trx->lock.wait_thr == NULL); ut_a(trx->lock.wait_lock == NULL); - - trx_assert_no_search_latch(trx); - ut_a(trx->dict_operation_lock_mode == 0); ut_a(UT_LIST_GET_LEN(trx->lock.trx_locks) == 0); @@ -2413,13 +2408,6 @@ state_ok: (ulong) n_rec_locks); } -#ifdef BTR_CUR_HASH_ADAPT - if (trx->has_search_latch) { - newline = TRUE; - fputs(", holds adaptive hash latch", f); - } -#endif /* BTR_CUR_HASH_ADAPT */ - if (trx->undo_no != 0) { newline = TRUE; fprintf(f, ", undo log entries " TRX_ID_FMT, trx->undo_no); @@ -2551,11 +2539,6 @@ state_ok: fprintf(f, "que state %lu ", (ulong) trx->lock.que_state); } - if (trx->has_search_latch) { - newline = TRUE; - fputs(", holds adaptive hash latch", f); - } - if (trx->undo_no != 0) { newline = TRUE; fprintf(f, ", undo log entries " TRX_ID_FMT, trx->undo_no); diff --git a/storage/myisam/ha_myisam.cc b/storage/myisam/ha_myisam.cc index 86f3bbb9bc5..871ee77b92b 100644 --- a/storage/myisam/ha_myisam.cc +++ b/storage/myisam/ha_myisam.cc @@ -1183,9 +1183,6 @@ int ha_myisam::repair(THD *thd, HA_CHECK ¶m, bool do_optimize) share->state.dupp_key= MI_MAX_KEY; strmov(fixed_name,file->filename); - // Release latches since this can take a long time - ha_release_temporary_latches(thd); - /* Don't lock tables if we have used LOCK TABLE or if we come from enable_index() diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index ba426dd3d3b..6097f87b43d 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -2122,39 +2122,6 @@ ha_innobase::is_fake_change_enabled(THD* thd) return(trx && UNIV_UNLIKELY(trx->fake_changes)); } -/********************************************************************//** -In XtraDB it is impossible for a transaction to own a search latch outside of -InnoDB code, so there is nothing to release on demand. We keep this function to -simplify maintenance. -@return 0 */ -static -int -innobase_release_temporary_latches( -/*===============================*/ - handlerton* hton MY_ATTRIBUTE((unused)), /*!< in: handlerton */ - THD* thd MY_ATTRIBUTE((unused))) /*!< in: MySQL thread */ -{ -#ifdef UNIV_DEBUG - DBUG_ASSERT(hton == innodb_hton_ptr); - - if (!innodb_inited || thd == NULL) { - - return(0); - } - - trx_t* trx = thd_to_trx(thd); - - if (trx != NULL) { -#ifdef UNIV_SYNC_DEBUG - ut_ad(!btr_search_own_any()); -#endif - trx_search_latch_release_if_reserved(trx); - } -#endif - - return(0); -} - /********************************************************************//** Increments innobase_active_counter and every INNOBASE_WAKE_INTERVALth time calls srv_active_wake_master_thread. This function should be used @@ -3813,9 +3780,6 @@ innobase_init( innobase_hton->flags = HTON_SUPPORTS_EXTENDED_KEYS | HTON_SUPPORTS_FOREIGN_KEYS; - innobase_hton->release_temporary_latches = - innobase_release_temporary_latches; - innobase_hton->kill_query = innobase_kill_connection; if (srv_file_per_table) @@ -6262,9 +6226,6 @@ ha_innobase::open( thd = ha_thd(); - /* No-op in XtraDB */ - innobase_release_temporary_latches(ht, thd); - normalize_table_name(norm_name, name); user_thd = NULL; @@ -6724,9 +6685,6 @@ ha_innobase::close() thd = ha_thd(); - /* No-op in XtraDB */ - innobase_release_temporary_latches(ht, thd); - row_prebuilt_free(prebuilt, FALSE); if (upd_buf != NULL) { diff --git a/storage/xtradb/log/log0log.cc b/storage/xtradb/log/log0log.cc index 3252cd793c9..833f3240369 100644 --- a/storage/xtradb/log/log0log.cc +++ b/storage/xtradb/log/log0log.cc @@ -860,43 +860,16 @@ ibool log_calc_max_ages(void) /*===================*/ { - log_group_t* group; lsn_t margin; ulint free; - ibool success = TRUE; - lsn_t smallest_capacity; - lsn_t archive_margin; - lsn_t smallest_archive_margin; - mutex_enter(&(log_sys->mutex)); - - group = UT_LIST_GET_FIRST(log_sys->log_groups); - - ut_ad(group); - - smallest_capacity = LSN_MAX; - smallest_archive_margin = LSN_MAX; - - while (group) { - if (log_group_get_capacity(group) < smallest_capacity) { - - smallest_capacity = log_group_get_capacity(group); - } - - archive_margin = log_group_get_capacity(group) - - (group->file_size - LOG_FILE_HDR_SIZE) - - LOG_ARCHIVE_EXTRA_MARGIN; - - if (archive_margin < smallest_archive_margin) { - - smallest_archive_margin = archive_margin; - } - - group = UT_LIST_GET_NEXT(log_groups, group); - } + lsn_t smallest_capacity = ((srv_log_file_size_requested + << srv_page_size_shift) + - LOG_FILE_HDR_SIZE) + * srv_n_log_files; /* Add extra safety */ - smallest_capacity = smallest_capacity - smallest_capacity / 10; + smallest_capacity -= smallest_capacity / 10; /* For each OS thread we must reserve so much free space in the smallest log group that it can accommodate the log entries produced @@ -906,15 +879,16 @@ log_calc_max_ages(void) free = LOG_CHECKPOINT_FREE_PER_THREAD * (10 + srv_thread_concurrency) + LOG_CHECKPOINT_EXTRA_FREE; if (free >= smallest_capacity / 2) { - success = FALSE; - - goto failure; - } else { - margin = smallest_capacity - free; + ib_logf(IB_LOG_LEVEL_FATAL, + "The combined size of ib_logfiles" + " should be bigger than\n" + "InnoDB: 200 kB * innodb_thread_concurrency."); } - + margin = smallest_capacity - free; margin = margin - margin / 10; /* Add still some extra safety */ + mutex_enter(&log_sys->mutex); + log_sys->log_group_capacity = smallest_capacity; log_sys->max_modified_age_async = margin @@ -927,22 +901,17 @@ log_calc_max_ages(void) log_sys->max_checkpoint_age = margin; #ifdef UNIV_LOG_ARCHIVE - log_sys->max_archived_lsn_age = smallest_archive_margin; + lsn_t archive_margin = smallest_capacity + - (srv_log_file_size_requested - LOG_FILE_HDR_SIZE) + - LOG_ARCHIVE_EXTRA_MARGIN; + log_sys->max_archived_lsn_age = archive_margin; - log_sys->max_archived_lsn_age_async = smallest_archive_margin - - smallest_archive_margin / LOG_ARCHIVE_RATIO_ASYNC; + log_sys->max_archived_lsn_age_async = archive_margin + - archive_margin / LOG_ARCHIVE_RATIO_ASYNC; #endif /* UNIV_LOG_ARCHIVE */ -failure: - mutex_exit(&(log_sys->mutex)); + mutex_exit(&log_sys->mutex); - if (!success) { - ib_logf(IB_LOG_LEVEL_FATAL, - "The combined size of ib_logfiles" - " should be bigger than\n" - "InnoDB: 200 kB * innodb_thread_concurrency."); - } - - return(success); + return(true); } /******************************************************//**