From b4a4d82396dce58496a9d0741fec505452d753f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Wed, 1 Apr 2015 11:50:21 +0300 Subject: [PATCH 1/6] InnoDB/XtraDB Encryption cleanup. Step 1: -- Remove page encryption from dictionary (per table encryption will be handled by storing crypt_data to page 0) -- Remove encryption/compression from os0file and all functions before that (compression will be added to buf0buf.cc) -- Use same CRYPT_SCHEME_1 for all encryption methods -- Do some code cleanups to confort InnoDB coding style --- extra/innochecksum.cc | 2 +- storage/innobase/buf/buf0buf.cc | 8 +- storage/innobase/buf/buf0dblwr.cc | 44 +- storage/innobase/buf/buf0flu.cc | 8 +- storage/innobase/buf/buf0rea.cc | 4 +- storage/innobase/fil/fil0crypt.cc | 996 ++++++++++-------- storage/innobase/fil/fil0fil.cc | 159 +-- storage/innobase/fil/fil0pagecompress.cc | 179 +--- storage/innobase/fsp/fsp0fsp.cc | 1 + storage/innobase/handler/ha_innodb.cc | 11 +- storage/innobase/handler/i_s.cc | 2 + storage/innobase/include/dict0dict.h | 4 +- storage/innobase/include/dict0dict.ic | 36 +- storage/innobase/include/dict0pagecompress.ic | 26 - storage/innobase/include/fil0crypt.h | 394 +++++++ storage/innobase/include/fil0crypt.ic | 69 ++ storage/innobase/include/fil0fil.h | 306 +----- storage/innobase/include/fil0fil.ic | 108 ++ storage/innobase/include/fil0pageencryption.h | 79 -- storage/innobase/include/fsp0fsp.h | 41 +- storage/innobase/include/fsp0pageencryption.h | 66 -- .../innobase/include/fsp0pageencryption.ic | 167 --- storage/innobase/include/os0file.h | 69 +- storage/innobase/include/os0file.ic | 23 +- storage/innobase/log/log0log.cc | 10 +- storage/innobase/log/log0recv.cc | 5 +- storage/innobase/os/os0file.cc | 438 +------- storage/innobase/row/row0log.cc | 4 +- storage/innobase/row/row0merge.cc | 2 +- storage/innobase/row/row0mysql.cc | 1 + storage/innobase/srv/srv0srv.cc | 4 +- storage/innobase/srv/srv0start.cc | 3 +- storage/xtradb/buf/buf0buf.cc | 9 +- storage/xtradb/buf/buf0dblwr.cc | 48 +- storage/xtradb/buf/buf0flu.cc | 8 +- storage/xtradb/buf/buf0rea.cc | 4 +- storage/xtradb/fil/fil0crypt.cc | 996 ++++++++++-------- storage/xtradb/fil/fil0fil.cc | 161 +-- storage/xtradb/fil/fil0pagecompress.cc | 185 +--- storage/xtradb/fsp/fsp0fsp.cc | 1 + storage/xtradb/handler/ha_innodb.cc | 11 +- storage/xtradb/handler/i_s.cc | 2 + storage/xtradb/include/dict0dict.h | 4 +- storage/xtradb/include/dict0dict.ic | 37 +- storage/xtradb/include/dict0pagecompress.ic | 26 - storage/xtradb/include/fil0crypt.h | 394 +++++++ storage/xtradb/include/fil0crypt.ic | 69 ++ storage/xtradb/include/fil0fil.h | 309 +----- storage/xtradb/include/fil0fil.ic | 108 ++ storage/xtradb/include/fil0pageencryption.h | 79 -- storage/xtradb/include/fsp0fsp.h | 44 +- storage/xtradb/include/fsp0pageencryption.h | 66 -- storage/xtradb/include/fsp0pageencryption.ic | 168 --- storage/xtradb/include/os0file.h | 73 +- storage/xtradb/include/os0file.ic | 23 +- storage/xtradb/log/log0log.cc | 16 +- storage/xtradb/log/log0online.cc | 2 +- storage/xtradb/log/log0recv.cc | 9 +- storage/xtradb/os/os0file.cc | 420 +------- storage/xtradb/row/row0log.cc | 4 +- storage/xtradb/row/row0merge.cc | 2 +- storage/xtradb/row/row0mysql.cc | 1 + storage/xtradb/srv/srv0srv.cc | 4 +- storage/xtradb/srv/srv0start.cc | 3 +- 64 files changed, 2524 insertions(+), 4031 deletions(-) create mode 100644 storage/innobase/include/fil0crypt.h create mode 100644 storage/innobase/include/fil0crypt.ic create mode 100644 storage/innobase/include/fil0fil.ic delete mode 100644 storage/innobase/include/fil0pageencryption.h delete mode 100644 storage/innobase/include/fsp0pageencryption.h delete mode 100644 storage/innobase/include/fsp0pageencryption.ic create mode 100644 storage/xtradb/include/fil0crypt.h create mode 100644 storage/xtradb/include/fil0crypt.ic create mode 100644 storage/xtradb/include/fil0fil.ic delete mode 100644 storage/xtradb/include/fil0pageencryption.h delete mode 100644 storage/xtradb/include/fsp0pageencryption.h delete mode 100644 storage/xtradb/include/fsp0pageencryption.ic diff --git a/extra/innochecksum.cc b/extra/innochecksum.cc index 1f323b9d730..34bc4a199d5 100644 --- a/extra/innochecksum.cc +++ b/extra/innochecksum.cc @@ -52,10 +52,10 @@ The parts not included are excluded by #ifndef UNIV_INNOCHECKSUM. */ #include "ut0ut.h" #include "ut0byte.h" #include "mach0data.h" -#include "fsp0types.h" #include "rem0rec.h" #include "buf0checksum.h" /* buf_calc_page_*() */ #include "fil0fil.h" /* FIL_* */ +#include "fsp0types.h" #include "page0page.h" /* PAGE_* */ #include "page0zip.h" /* page_zip_*() */ #include "trx0undo.h" /* TRX_* */ diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 59cbe633dec..b2758e1253f 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -40,6 +40,7 @@ Created 11/5/1995 Heikki Tuuri #include "mem0mem.h" #include "btr0btr.h" #include "fil0fil.h" +#include "fil0crypt.h" #ifndef UNIV_HOTBACKUP #include "buf0buddy.h" #include "lock0lock.h" @@ -54,7 +55,6 @@ Created 11/5/1995 Heikki Tuuri #include "page0zip.h" #include "srv0mon.h" #include "buf0checksum.h" -#include "fil0pageencryption.h" #include "fil0pagecompress.h" #include "ut0byte.h" #include @@ -504,7 +504,7 @@ buf_page_is_corrupted( ulint zip_size) /*!< in: size of compressed page; 0 for uncompressed pages */ { - ulint page_encrypted = fil_page_is_compressed_encrypted(read_buf) || fil_page_is_encrypted(read_buf); + ulint page_encrypted = fil_page_is_encrypted(read_buf); ulint checksum_field1; ulint checksum_field2; ibool crc32_inited = FALSE; @@ -5763,7 +5763,7 @@ buf_page_decrypt_after_read( unsigned key_version = mach_read_from_4(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); - bool page_compressed_encrypted = fil_page_is_compressed_encrypted(dst_frame); + bool page_compressed = fil_page_is_compressed(dst_frame); if (key_version == 0) { /* the page we read is unencrypted */ @@ -5801,7 +5801,7 @@ buf_page_decrypt_after_read( /* decompress from dst_frame to comp_buf and then copy to buffer pool */ - if (page_compressed_encrypted) { + if (page_compressed) { if (bpage->comp_buf_free == NULL) { bpage->comp_buf_free = (byte *)malloc(UNIV_PAGE_SIZE*2); // TODO: is 4k aligment enough ? diff --git a/storage/innobase/buf/buf0dblwr.cc b/storage/innobase/buf/buf0dblwr.cc index 477bcfe0ee3..53ef6ecb316 100644 --- a/storage/innobase/buf/buf0dblwr.cc +++ b/storage/innobase/buf/buf0dblwr.cc @@ -36,6 +36,8 @@ Created 2011/12/19 #include "srv0srv.h" #include "page0zip.h" #include "trx0sys.h" +#include "fil0fil.h" +#include "fil0crypt.h" #ifndef UNIV_HOTBACKUP @@ -385,7 +387,7 @@ buf_dblwr_init_or_load_pages( /* Read the trx sys header to check if we are using the doublewrite buffer */ off_t trx_sys_page = TRX_SYS_PAGE_NO * UNIV_PAGE_SIZE; - os_file_read(file, read_buf, trx_sys_page, UNIV_PAGE_SIZE, FALSE); + os_file_read(file, read_buf, trx_sys_page, UNIV_PAGE_SIZE); doublewrite = read_buf + TRX_SYS_DOUBLEWRITE; @@ -430,9 +432,9 @@ buf_dblwr_init_or_load_pages( block_bytes = TRX_SYS_DOUBLEWRITE_BLOCK_SIZE * UNIV_PAGE_SIZE; - os_file_read(file, buf, block1 * UNIV_PAGE_SIZE, block_bytes, FALSE); + os_file_read(file, buf, block1 * UNIV_PAGE_SIZE, block_bytes); os_file_read(file, buf + block_bytes, block2 * UNIV_PAGE_SIZE, - block_bytes, FALSE); + block_bytes); /* Check if any of these pages is half-written in data files, in the intended position */ @@ -530,9 +532,7 @@ buf_dblwr_process() zip_size ? zip_size : UNIV_PAGE_SIZE, read_buf, NULL, - 0, - 0, - false); + 0); if (fil_space_verify_crypt_checksum(read_buf, zip_size)) { /* page is encrypted and checksum is OK */ @@ -593,9 +593,7 @@ buf_dblwr_process() zip_size ? zip_size : UNIV_PAGE_SIZE, page, NULL, - 0, - 0, - false); + 0); ib_logf(IB_LOG_LEVEL_INFO, "Recovered the page from" @@ -620,9 +618,7 @@ buf_dblwr_process() zip_size ? zip_size : UNIV_PAGE_SIZE, page, NULL, - 0, - 0, - false); + 0); } } } @@ -644,9 +640,9 @@ buf_dblwr_process() memset(buf, 0, bytes); fil_io(OS_FILE_WRITE, true, TRX_SYS_SPACE, 0, - buf_dblwr->block1, 0, bytes, buf, NULL, NULL, 0, false); + buf_dblwr->block1, 0, bytes, buf, NULL, NULL); fil_io(OS_FILE_WRITE, true, TRX_SYS_SPACE, 0, - buf_dblwr->block2, 0, bytes, buf, NULL, NULL, 0, false); + buf_dblwr->block2, 0, bytes, buf, NULL, NULL); ut_free(unaligned_buf); } @@ -860,9 +856,7 @@ buf_dblwr_write_block_to_datafile( buf_page_get_zip_size(bpage), frame, (void*) bpage, - 0, - bpage->newest_modification, - bpage->encrypt_later); + 0); return; } @@ -881,9 +875,7 @@ buf_dblwr_write_block_to_datafile( UNIV_PAGE_SIZE, frame, (void*) block, - (ulint *)&bpage->write_size, - bpage->newest_modification, - bpage->encrypt_later); + (ulint *)&bpage->write_size); } /********************************************************************//** @@ -977,7 +969,7 @@ try_again: fil_io(OS_FILE_WRITE, true, TRX_SYS_SPACE, 0, buf_dblwr->block1, 0, len, - (void*) write_buf, NULL, 0, 0, false); + (void*) write_buf, NULL, 0); if (buf_dblwr->first_free <= TRX_SYS_DOUBLEWRITE_BLOCK_SIZE) { /* No unwritten pages in the second block. */ @@ -993,7 +985,7 @@ try_again: fil_io(OS_FILE_WRITE, true, TRX_SYS_SPACE, 0, buf_dblwr->block2, 0, len, - (void*) write_buf, NULL, 0, 0, false); + (void*) write_buf, NULL, 0); flush: /* increment the doublewrite flushed pages counter */ @@ -1230,9 +1222,7 @@ retry: UNIV_PAGE_SIZE, (void*) (buf_dblwr->write_buf + UNIV_PAGE_SIZE * i), NULL, - 0, - bpage->newest_modification, - bpage->encrypt_later); + 0); } else { /* It is a regular page. Write it directly to the doublewrite buffer */ @@ -1244,9 +1234,7 @@ retry: UNIV_PAGE_SIZE, frame, NULL, - 0, - bpage->newest_modification, - bpage->encrypt_later); + 0); } /* Now flush the doublewrite buffer data to disk */ diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index 9bd7a7c007d..b8a34558867 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -921,9 +921,7 @@ buf_flush_write_block_low( zip_size ? zip_size : UNIV_PAGE_SIZE, frame, bpage, - &bpage->write_size, - bpage->newest_modification, - bpage->encrypt_later); + &bpage->write_size); } else { /* InnoDB uses doublewrite buffer and doublewrite buffer @@ -943,9 +941,7 @@ buf_flush_write_block_low( zip_size ? zip_size : UNIV_PAGE_SIZE, frame, bpage, - &bpage->write_size, - bpage->newest_modification, - bpage->encrypt_later); + &bpage->write_size); } else if (flush_type == BUF_FLUSH_SINGLE_PAGE) { buf_dblwr_write_single_page(bpage, sync); } else { diff --git a/storage/innobase/buf/buf0rea.cc b/storage/innobase/buf/buf0rea.cc index e91a5da6621..5917af7ca9d 100644 --- a/storage/innobase/buf/buf0rea.cc +++ b/storage/innobase/buf/buf0rea.cc @@ -186,7 +186,7 @@ buf_read_page_low( *err = fil_io(OS_FILE_READ | wake_later | ignore_nonexistent_pages, sync, space, zip_size, offset, 0, zip_size, - frame, bpage, &bpage->write_size, 0, false); + frame, bpage, &bpage->write_size); } else { ut_a(buf_page_get_state(bpage) == BUF_BLOCK_FILE_PAGE); @@ -194,7 +194,7 @@ buf_read_page_low( | ignore_nonexistent_pages, sync, space, 0, offset, 0, UNIV_PAGE_SIZE, frame, bpage, - &bpage->write_size, 0, false); + &bpage->write_size); } if (sync) { diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index 9f8315a8016..aab0b1f5568 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -24,6 +24,7 @@ Modified Jan Lindström jan.lindstrom@mariadb.com *******************************************************/ #include "fil0fil.h" +#include "fil0crypt.h" #include "srv0srv.h" #include "srv0start.h" #include "mach0data.h" @@ -35,14 +36,13 @@ Modified Jan Lindström jan.lindstrom@mariadb.com #include "btr0scrub.h" #include "fsp0fsp.h" #include "fil0pagecompress.h" -#include "fil0pageencryption.h" #include "ha_prototypes.h" // IB_LOG_ #include #include #include - +#include "fil0crypt.h" /** Mutex for keys */ UNIV_INTERN ib_mutex_t fil_crypt_key_mutex; @@ -119,67 +119,40 @@ static const unsigned char CRYPT_MAGIC[MAGIC_SZ] = { static const unsigned char EMPTY_PATTERN[MAGIC_SZ] = { 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 }; -/** - * CRYPT_SCHEME_UNENCRYPTED - * - * Used as intermediate state when convering a space from unencrypted - * to encrypted - */ -#define CRYPT_SCHEME_UNENCRYPTED 0 - -/** - * CRYPT_SCHEME_1 - * - * L = AES_ECB(KEY, IV) - * CRYPT(PAGE) = AES_CRT(KEY=L, IV=C, PAGE) - */ -#define CRYPT_SCHEME_1 1 -#define CRYPT_SCHEME_1_IV_LEN 16 -// cached L given key_version -struct key_struct +/****************************************************************** +Map used AES method to crypt scheme +@return used AES crypt scheme */ +UNIV_INTERN +uint +fil_crypt_get_aes_method( +/*=====================*/ + uint aes_method) { - uint key_version; - byte key[CRYPT_SCHEME_1_IV_LEN]; -}; + switch (aes_method) { + case MY_AES_ALGORITHM_NONE: + return (uint) CRYPT_SCHEME_1_UNENCRYPTED; + break; + case MY_AES_ALGORITHM_CTR: + return (uint) CRYPT_SCHEME_1_CTR; + break; + case MY_AES_ALGORITHM_CBC: + return (uint) CRYPT_SCHEME_1_CBC; + break; + default: + ib_logf(IB_LOG_LEVEL_FATAL, + "Current AES method %d not supported.\n", aes_method); + ut_error; + } -struct fil_space_rotate_state_t -{ - time_t start_time; // time when rotation started - ulint active_threads; // active threads in space - ulint next_offset; // next "free" offset - ulint max_offset; // max offset needing to be rotated - uint min_key_version_found; // min key version found but not rotated - lsn_t end_lsn; // max lsn created when rotating this space - bool starting; // initial write of IV - bool flushing; // space is being flushed at end of rotate - struct { - bool is_active; // is scrubbing active in this space - time_t last_scrub_completed; // when was last scrub completed - } scrubbing; -}; - -struct fil_space_crypt_struct -{ - ulint type; // CRYPT_SCHEME - uint keyserver_requests; // no of key requests to key server - uint key_count; // No of initalized key-structs - key_struct keys[3]; // cached L = AES_ECB(KEY, IV) - uint min_key_version; // min key version for this space - ulint page0_offset; // byte offset on page 0 for crypt data - - ib_mutex_t mutex; // mutex protecting following variables - bool closing; // is tablespace being closed - fil_space_rotate_state_t rotate_state; - - uint iv_length; // length of IV - byte iv[1]; // IV-data -}; + return (uint) CRYPT_SCHEME_1_UNENCRYPTED; +} /********************************************************************* Init space crypt */ UNIV_INTERN void fil_space_crypt_init() +/*==================*/ { mutex_create(fil_crypt_key_mutex_key, &fil_crypt_key_mutex, SYNC_NO_ORDER_CHECK); @@ -196,6 +169,7 @@ Cleanup space crypt */ UNIV_INTERN void fil_space_crypt_cleanup() +/*=====================*/ { os_event_free(fil_crypt_throttle_sleep_event); } @@ -204,31 +178,35 @@ fil_space_crypt_cleanup() Get key bytes for a space/key-version */ static void -fil_crypt_get_key(byte *dst, uint* key_length, - fil_space_crypt_t* crypt_data, uint version, bool page_encrypted) +fil_crypt_get_key( +/*==============*/ + byte* dst, /*mutex); - if (!page_encrypted) { - // Check if we already have key - for (uint i = 0; i < crypt_data->key_count; i++) { - if (crypt_data->keys[i].key_version == version) { - memcpy(dst, crypt_data->keys[i].key, - sizeof(crypt_data->keys[i].key)); - mutex_exit(&crypt_data->mutex); - return; - } + // Check if we already have key + for (uint i = 0; i < crypt_data->key_count; i++) { + if (crypt_data->keys[i].key_version == version) { + memcpy(dst, crypt_data->keys[i].key, + crypt_data->keys[i].key_length); + *key_length = crypt_data->keys[i].key_length; + mutex_exit(&crypt_data->mutex); + return; } - // Not found! - crypt_data->keyserver_requests++; + } - // Rotate keys to make room for a new - for (uint i = 1; i < array_elements(crypt_data->keys); i++) { - crypt_data->keys[i] = crypt_data->keys[i - 1]; - } - } + // Not found! + crypt_data->keyserver_requests++; + + // Rotate keys to make room for a new + for (uint i = 1; i < array_elements(crypt_data->keys); i++) { + crypt_data->keys[i] = crypt_data->keys[i - 1]; + } if (has_encryption_key(version)) { int rc; @@ -247,51 +225,45 @@ fil_crypt_get_key(byte *dst, uint* key_length, ut_error; } + /* Now compute L by encrypting IV using this key. Note + that we use random IV from crypt data. */ + const unsigned char* src = crypt_data->iv; + const int srclen = crypt_data->iv_length; + unsigned char* buf = crypt_data->keys[0].key; + uint32 buflen = CRYPT_SCHEME_1_IV_LEN; - // do ctr key initialization - if (current_aes_dynamic_method == MY_AES_ALGORITHM_CTR) - { - /* Now compute L by encrypting IV using this key. Note - that we use random IV from crypt data. */ - const unsigned char* src = crypt_data->iv; - const int srclen = crypt_data->iv_length; - unsigned char* buf = page_encrypted ? keybuf : crypt_data->keys[0].key; - uint32 buflen = page_encrypted ? *key_length : sizeof(crypt_data->keys[0].key); + /* We use AES_ECB to encryp IV */ + my_aes_encrypt_dynamic_type func = get_aes_encrypt_func(MY_AES_ALGORITHM_ECB); - // call ecb explicit - my_aes_encrypt_dynamic_type func = get_aes_encrypt_func(MY_AES_ALGORITHM_ECB); - int rc = (*func)(src, srclen, - buf, &buflen, - (unsigned char*)keybuf, *key_length, - NULL, 0, - 1); + int rc = (*func)(src, /* Data to be encrypted = IV */ + srclen, /* data length */ + buf, /* Output buffer */ + &buflen, /* Output buffer */ + keybuf, /* Key */ + *key_length, /* Key length */ + NULL, /* AES_ECB does not use IV */ + 0, /* IV-length */ + 1); /* NoPadding */ - if (rc != AES_OK) { - ib_logf(IB_LOG_LEVEL_FATAL, - "Unable to encrypt key-block " - " src: %p srclen: %d buf: %p buflen: %d." - " return-code: %d. Can't continue!\n", - src, srclen, buf, buflen, rc); - ut_error; - } - - if (!page_encrypted) { - crypt_data->keys[0].key_version = version; - crypt_data->key_count++; - - if (crypt_data->key_count > array_elements(crypt_data->keys)) { - crypt_data->key_count = array_elements(crypt_data->keys); - } - } - - // set the key size to the aes block size because this encrypted data is the key - *key_length = MY_AES_BLOCK_SIZE; - memcpy(dst, buf, buflen); - } else { - // otherwise keybuf contains the right key - memcpy(dst, keybuf, *key_length); + if (rc != AES_OK) { + ib_logf(IB_LOG_LEVEL_FATAL, + "Unable to encrypt key-block " + " src: %p srclen: %d buf: %p buflen: %d." + " return-code: %d. Can't continue!\n", + src, srclen, buf, buflen, rc); + ut_error; } + crypt_data->keys[0].key_version = version; + crypt_data->key_count++; + *key_length = buflen; + crypt_data->keys[0].key_length = buflen; + + if (crypt_data->key_count > array_elements(crypt_data->keys)) { + crypt_data->key_count = array_elements(crypt_data->keys); + } + + memcpy(dst, buf, buflen); mutex_exit(&crypt_data->mutex); } @@ -299,8 +271,12 @@ fil_crypt_get_key(byte *dst, uint* key_length, Get key bytes for a space/latest(key-version) */ static inline void -fil_crypt_get_latest_key(byte *dst, uint* key_length, - fil_space_crypt_t* crypt_data, uint *version) +fil_crypt_get_latest_key( +/*=====================*/ + byte* dst, /*!< out: Key */ + uint* key_length, /*!< out: Key length */ + fil_space_crypt_t* crypt_data, /*!< in: crypt data */ + uint* version) /*!< in: Key version */ { if (srv_encrypt_tables) { // used for key rotation - get the next key id from the key provider @@ -312,29 +288,33 @@ fil_crypt_get_latest_key(byte *dst, uint* key_length, } } - return fil_crypt_get_key(dst, key_length, crypt_data, *version, srv_encrypt_tables == FALSE); + return fil_crypt_get_key(dst, key_length, crypt_data, *version); } /****************************************************************** -Create a fil_space_crypt_t object */ +Create a fil_space_crypt_t object +@return crypt object */ UNIV_INTERN fil_space_crypt_t* fil_space_create_crypt_data() +/*=========================*/ { const uint iv_length = CRYPT_SCHEME_1_IV_LEN; const uint sz = sizeof(fil_space_crypt_t) + iv_length; fil_space_crypt_t* crypt_data = static_cast(malloc(sz)); + uint aes_method = current_aes_dynamic_method; + memset(crypt_data, 0, sz); if (srv_encrypt_tables == FALSE) { - crypt_data->type = CRYPT_SCHEME_UNENCRYPTED; crypt_data->min_key_version = 0; } else { - crypt_data->type = CRYPT_SCHEME_1; crypt_data->min_key_version = get_latest_encryption_key_version(); } + /* Set up the current AES method */ + crypt_data->type = (fil_crypt_method_t)fil_crypt_get_aes_method(aes_method); mutex_create(fil_crypt_data_mutex_key, &crypt_data->mutex, SYNC_NO_ORDER_CHECK); crypt_data->iv_length = iv_length; @@ -346,13 +326,18 @@ fil_space_create_crypt_data() Compare two crypt objects */ UNIV_INTERN int -fil_space_crypt_compare(const fil_space_crypt_t* crypt_data1, - const fil_space_crypt_t* crypt_data2) +fil_space_crypt_compare( +/*====================*/ + const fil_space_crypt_t* crypt_data1,/*!< in: Crypt data */ + const fil_space_crypt_t* crypt_data2)/*!< in: Crypt data */ { - ut_a(crypt_data1->type == CRYPT_SCHEME_UNENCRYPTED || - crypt_data1->type == CRYPT_SCHEME_1); - ut_a(crypt_data2->type == CRYPT_SCHEME_UNENCRYPTED || - crypt_data2->type == CRYPT_SCHEME_1); + ut_a(crypt_data1->type == CRYPT_SCHEME_1_UNENCRYPTED || + crypt_data1->type == CRYPT_SCHEME_1_CTR || + crypt_data1->type == CRYPT_SCHEME_1_CBC); + + ut_a(crypt_data2->type == CRYPT_SCHEME_1_UNENCRYPTED || + crypt_data2->type == CRYPT_SCHEME_1_CTR || + crypt_data1->type == CRYPT_SCHEME_1_CBC); ut_a(crypt_data1->iv_length == CRYPT_SCHEME_1_IV_LEN); ut_a(crypt_data2->iv_length == CRYPT_SCHEME_1_IV_LEN); @@ -365,10 +350,15 @@ fil_space_crypt_compare(const fil_space_crypt_t* crypt_data1, } /****************************************************************** -Read crypt data from a page (0) */ +Read crypt data from a page (0) +@return crypt data from page 0. */ UNIV_INTERN fil_space_crypt_t* -fil_space_read_crypt_data(ulint space, const byte* page, ulint offset) +fil_space_read_crypt_data( +/*======================*/ + ulint space, /*!< in: file space id*/ + const byte* page, /*!< in: page 0 */ + ulint offset) /*!< in: offset */ { if (memcmp(page + offset, EMPTY_PATTERN, MAGIC_SZ) == 0) { /* crypt is not stored */ @@ -393,8 +383,10 @@ fil_space_read_crypt_data(ulint space, const byte* page, ulint offset) ulint type = mach_read_from_1(page + offset + MAGIC_SZ + 0); - if (! (type == CRYPT_SCHEME_UNENCRYPTED || - type == CRYPT_SCHEME_1)) { + if (! (type == CRYPT_SCHEME_1_UNENCRYPTED || + type == CRYPT_SCHEME_1_CTR || + type == CRYPT_SCHEME_1_CBC)) { + ib_logf(IB_LOG_LEVEL_ERROR, "Found non sensible crypt scheme: %lu for space %lu " " offset: %lu bytes: " @@ -449,7 +441,9 @@ fil_space_read_crypt_data(ulint space, const byte* page, ulint offset) Free a crypt data object */ UNIV_INTERN void -fil_space_destroy_crypt_data(fil_space_crypt_t **crypt_data) +fil_space_destroy_crypt_data( +/*=========================*/ + fil_space_crypt_t **crypt_data) /*!< out: crypt data */ { if (crypt_data != NULL && (*crypt_data) != NULL) { /* lock (and unlock) mutex to make sure no one has it locked @@ -466,10 +460,14 @@ fil_space_destroy_crypt_data(fil_space_crypt_t **crypt_data) Write crypt data to a page (0) */ static void -fil_space_write_crypt_data_low(fil_space_crypt_t *crypt_data, - ulint type, - byte* page, ulint offset, - ulint maxsize, mtr_t* mtr) +fil_space_write_crypt_data_low( +/*===========================*/ + fil_space_crypt_t* crypt_data, /* 0 && offset < UNIV_PAGE_SIZE); ulint space_id = mach_read_from_4( @@ -492,10 +490,8 @@ fil_space_write_crypt_data_low(fil_space_crypt_t *crypt_data, mlog_write_ulint(page + offset + MAGIC_SZ + 2 + len, min_key_version, MLOG_4BYTES, mtr); - DBUG_EXECUTE_IF("ib_file_crypt_redo_crash_1", - ut_error;); - byte* log_ptr = mlog_open(mtr, 11 + 12 + len); + if (log_ptr != NULL) { log_ptr = mlog_write_initial_log_record_fast( page, @@ -515,25 +511,26 @@ fil_space_write_crypt_data_low(fil_space_crypt_t *crypt_data, mlog_catenate_string(mtr, crypt_data->iv, len); } - - DBUG_EXECUTE_IF("ib_file_crypt_redo_crash_2", - ut_error;); } /****************************************************************** Write crypt data to a page (0) */ UNIV_INTERN void -fil_space_write_crypt_data(ulint space, byte* page, ulint offset, - ulint maxsize, mtr_t* mtr) +fil_space_write_crypt_data( +/*=======================*/ + ulint space, /*mutex); - memcpy(iv, crypt_data->iv, crypt_data->iv_length); - mutex_exit(&crypt_data->mutex); - } + /* For AES CTR create counter and AES CBS IV */ + mach_write_to_4(iv + 0, space); + ulint space_offset = mach_read_from_4(src_frame + FIL_PAGE_OFFSET); + mach_write_to_4(iv + 4, space_offset); + mach_write_to_8(iv + 8, lsn); ibool page_compressed = (mach_read_from_2(src_frame+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED); - ibool page_encrypted = fil_space_is_page_encrypted(space); - ulint compression_alg = mach_read_from_8(src_frame+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); + ulint compression_algo = page_compressed ? mach_read_from_8(src_frame+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION) : 0; - // copy page header + /* FIL page header is not encrypted */ memcpy(dst_frame, src_frame, FIL_PAGE_DATA); - if (page_encrypted && !page_compressed) { - // key id - mach_write_to_2(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, - key_version); - // original page type - mach_write_to_2(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 2, - orig_page_type); - // new page type - mach_write_to_2(dst_frame+FIL_PAGE_TYPE, FIL_PAGE_PAGE_ENCRYPTED); - } else { - // store key version - mach_write_to_4(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, - key_version); - } + /* Store key version */ + mach_write_to_4(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, key_version); - // encrypt page data + /* Calculate the start offset in a page */ ulint unencrypted_bytes = FIL_PAGE_DATA + FIL_PAGE_DATA_END; ulint srclen = page_size - unencrypted_bytes; const byte* src = src_frame + FIL_PAGE_DATA; byte* dst = dst_frame + FIL_PAGE_DATA; - uint32 dstlen; + uint32 dstlen=0; + /* For page compressed tables we encrypt only the actual compressed + payload. Note that first two bytes of page data is actual payload + size and that should not be encrypted. */ if (page_compressed) { - srclen = page_size - FIL_PAGE_DATA; + ulint payload = mach_read_from_2(src_frame + FIL_PAGE_DATA); + mach_write_to_2(dst_frame + FIL_PAGE_DATA, payload); + srclen = payload; + src+=2; + dst+=2; } - int rc = (* my_aes_encrypt_dynamic)(src, srclen, - dst, &dstlen, - (unsigned char*)key, key_length, - (unsigned char*)iv, sizeof(iv), - 1); + /* Get encryption method */ + my_aes_encrypt_dynamic_type func = get_aes_encrypt_func((enum_my_aes_encryption_algorithm)fil_crypt_map_aes_method(aes_method)); + + /* Encrypt the data */ + int rc = (*func)(src, /* Original page */ + srclen, /* Page length */ + dst, /* Output buffer */ + &dstlen, /* Output length */ + key, /* Encryption key */ + key_length, /* Key length */ + iv, /* IV */ + sizeof(iv), /* IV length */ + 1); /* Use noPadding */ if (! ((rc == AES_OK) && ((ulint) dstlen == srclen))) { ib_logf(IB_LOG_LEVEL_FATAL, @@ -735,18 +767,164 @@ fil_space_encrypt(ulint space, ulint offset, lsn_t lsn, ut_error; } + /* Set up the checksum */ + mach_write_to_4(dst_frame+FIL_PAGE_SPACE_OR_CHKSUM, BUF_NO_CHECKSUM_MAGIC); + + /* For compressed tables we do not store the FIL header because + the whole page is not stored to the disk. In compressed tables only + the FIL header + compressed (and now encrypted) payload alligned + to sector boundary is written. */ if (!page_compressed) { - // copy page trailer + /* FIL page trailer is also not encrypted */ memcpy(dst_frame + page_size - FIL_PAGE_DATA_END, src_frame + page_size - FIL_PAGE_DATA_END, FIL_PAGE_DATA_END); + } - /* handle post encryption checksum */ + /* Store AES encryption method */ + mach_write_to_2(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4, + aes_method); + /* Store compression algorithm (for page compresed tables) or 0 */ + mach_write_to_2(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 6, + compression_algo); +} + +/********************************************************************* +Check if extra buffer shall be allocated for decrypting after read +@return true if fil space has encryption data. */ +UNIV_INTERN +bool +fil_space_check_encryption_read( +/*=============================*/ + ulint space) /*!< in: tablespace id */ +{ + fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space); + + if (crypt_data == NULL) { + return false; + } + + if (crypt_data->type == CRYPT_SCHEME_UNENCRYPTED) { + return false; + } + + return true; +} + +/****************************************************************** +Decrypt a page +@return true if page was encrypted */ +UNIV_INTERN +bool +fil_space_decrypt( +/*==============*/ + fil_space_crypt_t* crypt_data, /*!< in: crypt data */ + const byte* src_frame, /*!< in: input buffer */ + ulint page_size, /*!< in: page size */ + byte* dst_frame) /*!< out: output buffer */ +{ + ulint page_type = mach_read_from_2(src_frame+FIL_PAGE_TYPE); + uint key_version = mach_read_from_4(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); + bool page_compressed = (page_type == FIL_PAGE_PAGE_COMPRESSED); + ulint compression_algo = mach_read_from_2(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 6); + uint aes_method = 0; + + /* Page is not encrypted if key_version is 0 */ + if (key_version == 0) { + //TODO: is this really needed ? + memcpy(dst_frame, src_frame, page_size); + return false; /* page not decrypted */ + } + + /* read space & offset & lsn */ + ulint space = mach_read_from_4( + src_frame + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); + ulint offset = mach_read_from_4( + src_frame + FIL_PAGE_OFFSET); + ib_uint64_t lsn = mach_read_from_8(src_frame + FIL_PAGE_LSN); + + /* Copy FIL page header, it is not encrypted */ + memcpy(dst_frame, src_frame, FIL_PAGE_DATA); + + /* Get key */ + byte key[MY_AES_MAX_KEY_LENGTH]; + uint key_length; + unsigned char iv[MY_AES_BLOCK_SIZE]; + fil_crypt_get_key(key, &key_length, crypt_data, key_version); + aes_method = mach_read_from_2(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4); + + /* Verify used AES method */ + if ( aes_method != CRYPT_SCHEME_1_CTR && + aes_method != CRYPT_SCHEME_1_CBC) { + ib_logf(IB_LOG_LEVEL_FATAL, + "Unable to decrypt data-block " + " aes_method: %d unknown!\n", + aes_method); + ut_error; + } + + /* Create counter used as IV */ + mach_write_to_4(iv + 0, space); + mach_write_to_4(iv + 4, offset); + mach_write_to_8(iv + 8, lsn); + + /* Calculate the offset where decryption starts */ + const byte* src = src_frame + FIL_PAGE_DATA; + byte* dst = dst_frame + FIL_PAGE_DATA; + uint32 dstlen=0; + ulint srclen = page_size - (FIL_PAGE_DATA + FIL_PAGE_DATA_END); + + /* For page compressed tables we decrypt only the actual compressed + payload. Note that first two bytes of page data is actual payload + size and that should not be decrypted. */ + if (page_compressed) { + ulint compressed_len = mach_read_from_2(src_frame + FIL_PAGE_DATA); + src+=2; + dst+=2; + mach_write_to_2(dst_frame + FIL_PAGE_DATA, compressed_len); + srclen = compressed_len; + } + + /* Get AES method */ + my_aes_encrypt_dynamic_type func = get_aes_decrypt_func((enum_my_aes_encryption_algorithm)fil_crypt_map_aes_method(aes_method)); + + /* Decrypt the data */ + int rc = (*func)(src, /* Data to be encrypted */ + srclen, /* data length */ + dst, /* Output buffer */ + &dstlen, /* Output buffer */ + key, /* Key */ + key_length, /* Key length */ + iv, /* IV */ + sizeof(iv), /* IV-length */ + 1); /* NoPadding */ + + if (! ((rc == AES_OK) && ((ulint) dstlen == srclen))) { + ib_logf(IB_LOG_LEVEL_FATAL, + "Unable to decrypt data-block " + " src: %p srclen: %ld buf: %p buflen: %d." + " return-code: %d. Can't continue!\n", + src, (long)srclen, + dst, dstlen, rc); + ut_error; + } + + /* For compressed tables we do not store the FIL header because + the whole page is not stored to the disk. In compressed tables only + the FIL header + compressed (and now encrypted) payload alligned + to sector boundary is written. */ + if (!page_compressed) { + /* Copy FIL trailer */ + memcpy(dst_frame + page_size - FIL_PAGE_DATA_END, + src_frame + page_size - FIL_PAGE_DATA_END, + FIL_PAGE_DATA_END); + + /* handle post decryption checksum */ ib_uint32_t checksum = 0; srv_checksum_algorithm_t algorithm = static_cast(srv_checksum_algorithm); - if (zip_size == 0) { + if (page_size == UNIV_PAGE_SIZE) { switch (algorithm) { case SRV_CHECKSUM_ALGORITHM_CRC32: case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32: @@ -765,173 +943,23 @@ fil_space_encrypt(ulint space, ulint offset, lsn_t lsn, * if new enum is added and not handled here */ } } else { - checksum = page_zip_calc_checksum(dst_frame, zip_size, + checksum = page_zip_calc_checksum(dst_frame, page_size, algorithm); } - // store the post-encryption checksum after the key-version - mach_write_to_4(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4, - checksum); - } else { - /* Page compressed and encrypted tables have different - FIL_HEADER */ - ulint page_len = log10((double)page_size)/log10((double)2); - /* Set up the correct page type */ - mach_write_to_2(dst_frame+FIL_PAGE_TYPE, FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED); - /* Set up the compression algorithm */ - mach_write_to_2(dst_frame+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION+4, orig_page_type); - /* Set up the compressed size */ - mach_write_to_1(dst_frame+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION+6, page_len); - /* Set up the compression method */ - mach_write_to_1(dst_frame+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION+7, compression_alg); + mach_write_to_4(dst_frame + FIL_PAGE_SPACE_OR_CHKSUM, checksum); + + fprintf(stderr, "KUKKU %ld:%lu\n", mach_read_from_4(dst_frame + FIL_PAGE_SPACE_OR_CHKSUM), + mach_read_from_4(dst_frame + FIL_PAGE_END_LSN_OLD_CHKSUM)); } -} - -/********************************************************************* -Check if extra buffer shall be allocated for decrypting after read */ -UNIV_INTERN -bool -fil_space_check_encryption_read( -/*==============================*/ - ulint space) /*!< in: tablespace id */ -{ - fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space); - - if (crypt_data == NULL) { - return false; - } - - if (crypt_data->type == CRYPT_SCHEME_UNENCRYPTED) { - return false; - } - - return true; -} - -/****************************************************************** -Decrypt a page */ -UNIV_INTERN -bool -fil_space_decrypt(fil_space_crypt_t* crypt_data, - const byte* src_frame, ulint page_size, byte* dst_frame) -{ - ulint page_type = mach_read_from_2(src_frame+FIL_PAGE_TYPE); - // key version - uint key_version; - bool page_encrypted = (page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED - || page_type == FIL_PAGE_PAGE_ENCRYPTED); - - bool page_compressed = (page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED - || page_type == FIL_PAGE_PAGE_COMPRESSED); - - ulint orig_page_type=0; - - if (page_type == FIL_PAGE_PAGE_ENCRYPTED) { - key_version = mach_read_from_2( - src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); - orig_page_type = mach_read_from_2( - src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 2); - } else { - key_version = mach_read_from_4( - src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); - } - - if (key_version == 0 && !page_encrypted) { - //TODO: is this really needed ? - memcpy(dst_frame, src_frame, page_size); - return false; /* page not decrypted */ - } - - // read space & offset & lsn - ulint space = mach_read_from_4( - src_frame + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); - ulint offset = mach_read_from_4( - src_frame + FIL_PAGE_OFFSET); - ib_uint64_t lsn = mach_read_from_8(src_frame + FIL_PAGE_LSN); - - // copy page header - memcpy(dst_frame, src_frame, FIL_PAGE_DATA); - - if (page_type == FIL_PAGE_PAGE_ENCRYPTED) { - // orig page type - mach_write_to_2(dst_frame+FIL_PAGE_TYPE, orig_page_type); - } - - // get key - byte key[MY_AES_MAX_KEY_LENGTH]; - uint key_length; - fil_crypt_get_key(key, &key_length, crypt_data, key_version, page_encrypted); - - // get the iv - unsigned char iv[MY_AES_BLOCK_SIZE]; - - if (current_aes_dynamic_method == MY_AES_ALGORITHM_CTR) { - // create counter block - - mach_write_to_4(iv + 0, space); - mach_write_to_4(iv + 4, offset); - mach_write_to_8(iv + 8, lsn); - } else { - // Get random IV from crypt_data - mutex_enter(&crypt_data->mutex); - memcpy(iv, crypt_data->iv, crypt_data->iv_length); - mutex_exit(&crypt_data->mutex); - } - - const byte* src = src_frame + FIL_PAGE_DATA; - byte* dst = dst_frame + FIL_PAGE_DATA; - uint32 dstlen; - ulint srclen = page_size - (FIL_PAGE_DATA + FIL_PAGE_DATA_END); - ulint compressed_len; - ulint compression_method; + /* Clear key-version & aes_method from dst */ + memset(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, 0, 8); + /* For page compressed tables store compression algorithm back */ if (page_compressed) { - orig_page_type = mach_read_from_2(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION+4); - compressed_len = mach_read_from_1(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION+6); - compression_method = mach_read_from_1(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION+7); - } - - if (page_encrypted && !page_compressed) { - orig_page_type = mach_read_from_2(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION+2); - } - - if (page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { - srclen = pow((double)2, (double)((int)compressed_len)) - FIL_PAGE_DATA; - } - - int rc = (* my_aes_decrypt_dynamic)(src, srclen, - dst, &dstlen, - (unsigned char*)key, key_length, - (unsigned char*)iv, sizeof(iv), - 1); - - if (! ((rc == AES_OK) && ((ulint) dstlen == srclen))) { - ib_logf(IB_LOG_LEVEL_FATAL, - "Unable to decrypt data-block " - " src: %p srclen: %ld buf: %p buflen: %d." - " return-code: %d. Can't continue!\n", - src, (long)srclen, - dst, dstlen, rc); - ut_error; - } - - if (page_type != FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { - // copy page trailer - memcpy(dst_frame + page_size - FIL_PAGE_DATA_END, - src_frame + page_size - FIL_PAGE_DATA_END, - FIL_PAGE_DATA_END); - - // clear key-version & crypt-checksum from dst - memset(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, 0, 8); - } else { - /* For page compressed tables we set up the FIL_HEADER again */ - /* setting original page type */ - mach_write_to_2(dst_frame + FIL_PAGE_TYPE, orig_page_type); - /* page_compression uses BUF_NO_CHECKSUM_MAGIC as checksum */ - mach_write_to_4(dst_frame + FIL_PAGE_SPACE_OR_CHKSUM, BUF_NO_CHECKSUM_MAGIC); - /* Set up the flush lsn to be compression algorithm */ - mach_write_to_8(dst_frame+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, compression_method); + mach_write_to_8(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, + compression_algo); } return true; /* page was decrypted */ @@ -941,8 +969,12 @@ fil_space_decrypt(fil_space_crypt_t* crypt_data, Decrypt a page */ UNIV_INTERN void -fil_space_decrypt(ulint space, - const byte* src_frame, ulint page_size, byte* dst_frame) +fil_space_decrypt( +/*==============*/ + ulint space, /*!< in: Fil space id */ + const byte* src_frame, /*!< in: input buffer */ + ulint page_size, /*!< in: page size */ + byte* dst_frame) /*!< out: output buffer */ { fil_space_decrypt(fil_space_get_crypt_data(space), src_frame, page_size, dst_frame); @@ -953,8 +985,13 @@ Verify checksum for a page (iff it's encrypted) NOTE: currently this function can only be run in single threaded mode as it modifies srv_checksum_algorithm (temporarily) @return true if page is encrypted AND OK, false otherwise */ +UNIV_INTERN bool -fil_space_verify_crypt_checksum(const byte* src_frame, ulint zip_size) +fil_space_verify_crypt_checksum( +/*============================*/ + const byte* src_frame, /*!< in: page the verify */ + ulint zip_size) /*!< in: compressed size if + row_format compressed */ { // key version uint key_version = mach_read_from_4( @@ -1041,7 +1078,8 @@ struct key_state_t { Copy global key state */ static void fil_crypt_get_key_state( - key_state_t *new_state) +/*====================*/ + key_state_t *new_state) /*!< out: key state */ { if (srv_encrypt_tables == TRUE) { new_state->key_version = get_latest_encryption_key_version(); @@ -1054,9 +1092,13 @@ fil_crypt_get_key_state( } /*********************************************************************** -Check if a key needs rotation given a key_state */ +Check if a key needs rotation given a key_state +@return true if key needs rotation, false if not */ static bool -fil_crypt_needs_rotation(uint key_version, const key_state_t *key_state) +fil_crypt_needs_rotation( +/*=====================*/ + uint key_version, /*!< in: Key version */ + const key_state_t* key_state) /*!< in: Key state */ { // TODO(jonaso): Add support for rotating encrypted => unencrypted @@ -1081,9 +1123,13 @@ fil_crypt_needs_rotation(uint key_version, const key_state_t *key_state) } /*********************************************************************** -Check if a space is closing (i.e just before drop) */ -UNIV_INTERN bool -fil_crypt_is_closing(ulint space) +Check if a space is closing (i.e just before drop) +@return true if space is closing, false if not. */ +UNIV_INTERN +bool +fil_crypt_is_closing( +/*=================*/ + ulint space) /*!< in: FIL space id */ { bool closing; fil_space_crypt_t *crypt_data = fil_space_get_crypt_data(space); @@ -1097,16 +1143,22 @@ fil_crypt_is_closing(ulint space) Start encrypting a space @return true if a pending op (fil_inc_pending_ops/fil_decr_pending_ops) is held */ -static bool -fil_crypt_start_encrypting_space(ulint space, bool *recheck) { +static +bool +fil_crypt_start_encrypting_space( +/*=============================*/ + ulint space, /*!< in: FIL space id */ + bool* recheck)/*!< out: true if recheck needed */ +{ /* we have a pending op when entering function */ bool pending_op = true; + uint aes_method = current_aes_dynamic_method; mutex_enter(&fil_crypt_threads_mutex); fil_space_crypt_t *crypt_data = fil_space_get_crypt_data(space); - ibool page_encrypted = fil_space_is_page_encrypted(space); + ibool page_encrypted = (crypt_data != NULL); /*If spage is not encrypted and encryption is not enabled, then do not continue encrypting the space. */ @@ -1184,7 +1236,7 @@ fil_crypt_start_encrypting_space(ulint space, bool *recheck) { /* 4 - write crypt data to page 0 */ fil_space_write_crypt_data_low(crypt_data, - CRYPT_SCHEME_1, + fil_crypt_get_aes_method(aes_method), frame, crypt_data->page0_offset, maxsize, &mtr); @@ -1232,7 +1284,7 @@ fil_crypt_start_encrypting_space(ulint space, bool *recheck) { /* 5 - publish crypt data */ mutex_enter(&fil_crypt_threads_mutex); mutex_enter(&crypt_data->mutex); - crypt_data->type = CRYPT_SCHEME_1; + crypt_data->type = fil_crypt_get_aes_method(aes_method); ut_a(crypt_data->rotate_state.active_threads == 1); crypt_data->rotate_state.active_threads = 0; crypt_data->rotate_state.starting = false; @@ -1257,10 +1309,14 @@ fil_crypt_start_encrypting_space(ulint space, bool *recheck) { } /*********************************************************************** -Check if space needs rotation given a key_state */ -static bool -fil_crypt_space_needs_rotation(uint space, const key_state_t *key_state, - bool *recheck) +Check if space needs rotation given a key_state +@return true if space needs key rotation */ +static +bool +fil_crypt_space_needs_rotation( + uint space, /*!< in: FIL space id */ + const key_state_t* key_state, /*!< in: Key state */ + bool* recheck) /*!< out: needs recheck ? */ { if (fil_space_get_type(space) != FIL_TABLESPACE) { return false; @@ -1374,7 +1430,9 @@ struct rotate_thread_t { /*********************************************************************** Update global statistics with thread statistics */ static void -fil_crypt_update_total_stat(rotate_thread_t *state) +fil_crypt_update_total_stat( +/*========================*/ + rotate_thread_t *state) /*!< in: Key rotation status */ { mutex_enter(&crypt_stat_mutex); crypt_stat.pages_read_from_cache += @@ -1397,9 +1455,13 @@ fil_crypt_update_total_stat(rotate_thread_t *state) /*********************************************************************** Allocate iops to thread from global setting, -used before starting to rotate a space */ -static bool -fil_crypt_alloc_iops(rotate_thread_t *state) +used before starting to rotate a space. +@return true if allocation succeeded, false if failed */ +static +bool +fil_crypt_alloc_iops( +/*=================*/ + rotate_thread_t *state) /*!< in: Key rotation status */ { ut_ad(state->allocated_iops == 0); @@ -1429,8 +1491,11 @@ fil_crypt_alloc_iops(rotate_thread_t *state) /*********************************************************************** Reallocate iops to thread, used when inside a space */ -static void -fil_crypt_realloc_iops(rotate_thread_t *state) +static +void +fil_crypt_realloc_iops( +/*========================*/ + rotate_thread_t *state) /*!< in: Key rotation status */ { ut_a(state->allocated_iops > 0); @@ -1519,8 +1584,11 @@ fil_crypt_realloc_iops(rotate_thread_t *state) /*********************************************************************** Return allocated iops to global */ -static void -fil_crypt_return_iops(rotate_thread_t *state) +static +void +fil_crypt_return_iops( +/*========================*/ + rotate_thread_t *state) /*!< in: Key rotation status */ { if (state->allocated_iops > 0) { uint iops = state->allocated_iops; @@ -1544,11 +1612,14 @@ fil_crypt_return_iops(rotate_thread_t *state) /*********************************************************************** Search for a space needing rotation */ +UNIV_INTERN bool fil_crypt_find_space_to_rotate( - const key_state_t *key_state, - rotate_thread_t *state, - bool *recheck) +/*===========================*/ + const key_state_t* key_state, /*!< in: Key state */ + rotate_thread_t* state, /*!< in: Key rotation state */ + bool* recheck) /*!< out: true if recheck + needed */ { /* we need iops to start rotating */ while (!state->should_shutdown() && !fil_crypt_alloc_iops(state)) { @@ -1591,8 +1662,9 @@ Start rotating a space */ static void fil_crypt_start_rotate_space( - const key_state_t *key_state, - rotate_thread_t *state) +/*=========================*/ + const key_state_t* key_state, /*!< in: Key state */ + rotate_thread_t* state) /*!< in: Key rotation state */ { ulint space = state->space; fil_space_crypt_t *crypt_data = fil_space_get_crypt_data(space); @@ -1629,12 +1701,14 @@ fil_crypt_start_rotate_space( } /*********************************************************************** -Search for batch of pages needing rotation */ +Search for batch of pages needing rotation +@return true if page needing key rotation found, false if not found */ static bool fil_crypt_find_page_to_rotate( - const key_state_t *key_state, - rotate_thread_t *state) +/*==========================*/ + const key_state_t* key_state, /*!< in: Key state */ + rotate_thread_t* state) /*!< in: Key rotation state */ { ulint batch = srv_alloc_time * state->allocated_iops; ulint space = state->space; @@ -1665,9 +1739,15 @@ fil_crypt_find_page_to_rotate( } /*********************************************************************** -Check if a page is uninitialized (doesn't need to be rotated) */ -static bool -fil_crypt_is_page_uninitialized(const byte* frame, uint zip_size) +Check if a page is uninitialized (doesn't need to be rotated) +@return true if page is uninitialized, false if not.*/ +static +bool +fil_crypt_is_page_uninitialized( +/*============================*/ + const byte *frame, /*!< in: Page */ + uint zip_size) /*!< in: compressed size if + row_format compressed */ { if (zip_size) { ulint stored_checksum = mach_read_from_4( @@ -1696,15 +1776,20 @@ fil_crypt_is_page_uninitialized(const byte* frame, uint zip_size) sleeptime_ms, __FILE__, __LINE__) /*********************************************************************** -Get a page and compute sleep time */ +Get a page and compute sleep time +@return page */ static buf_block_t* -fil_crypt_get_page_throttle_func(rotate_thread_t *state, - ulint space, uint zip_size, ulint offset, - mtr_t *mtr, - ulint *sleeptime_ms, - const char *file, - ulint line) +fil_crypt_get_page_throttle_func( + rotate_thread_t* state, /*!< in/out: Key rotation state */ + ulint space, /*!< in: FIL space id */ + uint zip_size, /*!< in: compressed size if + row_format compressed */ + ulint offset, /*!< in: page offsett */ + mtr_t* mtr, /*!< in/out: minitransaction */ + ulint* sleeptime_ms, /*!< out: sleep time */ + const char* file, /*!< in: file name */ + ulint line) /*!< in: file line */ { buf_block_t* block = buf_page_try_get_func(space, offset, RW_X_LATCH, true, @@ -1753,17 +1838,22 @@ Get block and allocation status note: innodb locks fil_space_latch and then block when allocating page but locks block and then fil_space_latch when freeing page. +@return block */ static buf_block_t* btr_scrub_get_block_and_allocation_status( - rotate_thread_t *state, - ulint space, - ulint zip_size, - ulint offset, - mtr_t *mtr, +/*======================================*/ + rotate_thread_t* state, /*!< in/out: Key rotation state */ + ulint space, /*!< in: FIL space id */ + uint zip_size, /*!< in: compressed size if + row_format compressed */ + ulint offset, /*!< in: page offsett */ + mtr_t* mtr, /*!< in/out: minitransaction + */ btr_scrub_page_allocation_status_t *allocation_status, - ulint *sleeptime_ms) + /*!< in/out: allocation status */ + ulint* sleeptime_ms) /*!< out: sleep time */ { mtr_t local_mtr; buf_block_t *block = NULL; @@ -1807,8 +1897,9 @@ Rotate one page */ static void fil_crypt_rotate_page( - const key_state_t *key_state, - rotate_thread_t *state) +/*===================*/ + const key_state_t* key_state, /*!< in: Key state */ + rotate_thread_t* state) /*!< in: Key rotation state */ { ulint space = state->space; ulint offset = state->offset; @@ -1960,8 +2051,9 @@ Rotate a batch of pages */ static void fil_crypt_rotate_pages( - const key_state_t *key_state, - rotate_thread_t *state) +/*===================*/ + const key_state_t* key_state, /*!< in: Key state */ + rotate_thread_t* state) /*!< in: Key rotation state */ { ulint space = state->space; ulint end = state->offset + state->batch; @@ -1989,7 +2081,10 @@ fil_crypt_rotate_pages( Flush rotated pages and then update page 0 */ static void -fil_crypt_flush_space(rotate_thread_t *state, ulint space) +fil_crypt_flush_space( +/*==================*/ + rotate_thread_t* state, /*!< in: Key rotation state */ + ulint space) /*!< in: FIL space id */ { fil_space_crypt_t *crypt_data = fil_space_get_crypt_data(space); @@ -2033,9 +2128,10 @@ fil_crypt_flush_space(rotate_thread_t *state, ulint space) RW_X_LATCH, NULL, BUF_GET, __FILE__, __LINE__, &mtr); byte* frame = buf_block_get_frame(block); - ulint maxsize = 0; + ulint maxsize; crypt_data->page0_offset = fsp_header_get_crypt_offset(zip_size, &maxsize); + fil_space_write_crypt_data(space, frame, crypt_data->page0_offset, ULINT_MAX, &mtr); @@ -2048,8 +2144,9 @@ Complete rotating a space */ static void fil_crypt_complete_rotate_space( - const key_state_t *key_state, - rotate_thread_t *state) +/*============================*/ + const key_state_t* key_state, /*!< in: Key state */ + rotate_thread_t* state) /*!< in: Key rotation state */ { ulint space = state->space; fil_space_crypt_t *crypt_data = fil_space_get_crypt_data(space); @@ -2123,7 +2220,7 @@ A thread which monitors global key state and rotates tablespaces accordingly extern "C" UNIV_INTERN os_thread_ret_t DECLARE_THREAD(fil_crypt_thread)( -/*===============================*/ +/*=============================*/ void* arg __attribute__((unused))) /*!< in: a dummy parameter required * by os_thread_create */ { @@ -2240,13 +2337,19 @@ DECLARE_THREAD(fil_crypt_thread)( Adjust thread count for key rotation */ UNIV_INTERN void -fil_crypt_set_thread_cnt(uint new_cnt) +fil_crypt_set_thread_cnt( +/*=====================*/ + uint new_cnt) /*!< in: New key rotation thread count */ { if (new_cnt > srv_n_fil_crypt_threads) { uint add = new_cnt - srv_n_fil_crypt_threads; srv_n_fil_crypt_threads = new_cnt; for (uint i = 0; i < add; i++) { - os_thread_create(fil_crypt_thread, NULL, NULL); + os_thread_id_t rotation_thread_id; + os_thread_create(fil_crypt_thread, NULL, &rotation_thread_id); + ib_logf(IB_LOG_LEVEL_INFO, + "Creating #%d thread id %lu total threads %du\n", + i, os_thread_pf(rotation_thread_id), new_cnt); } } else if (new_cnt < srv_n_fil_crypt_threads) { srv_n_fil_crypt_threads = new_cnt; @@ -2263,7 +2366,9 @@ fil_crypt_set_thread_cnt(uint new_cnt) Adjust max key age */ UNIV_INTERN void -fil_crypt_set_rotate_key_age(uint val) +fil_crypt_set_rotate_key_age( +/*=========================*/ + uint val) /*!< in: New max key age */ { srv_fil_crypt_rotate_key_age = val; os_event_set(fil_crypt_threads_event); @@ -2273,7 +2378,9 @@ fil_crypt_set_rotate_key_age(uint val) Adjust rotation iops */ UNIV_INTERN void -fil_crypt_set_rotation_iops(uint val) +fil_crypt_set_rotation_iops( +/*========================*/ + uint val) /*!< in: New iops setting */ { srv_n_fil_crypt_iops = val; os_event_set(fil_crypt_threads_event); @@ -2284,6 +2391,7 @@ Init threads for key rotation */ UNIV_INTERN void fil_crypt_threads_init() +/*====================*/ { fil_crypt_event = os_event_create(); fil_crypt_threads_event = os_event_create(); @@ -2300,6 +2408,7 @@ End threads for key rotation */ UNIV_INTERN void fil_crypt_threads_end() +/*===================*/ { /* stop threads */ fil_crypt_set_thread_cnt(0); @@ -2309,7 +2418,9 @@ fil_crypt_threads_end() Clean up key rotation threads resources */ UNIV_INTERN void -fil_crypt_threads_cleanup() { +fil_crypt_threads_cleanup() +/*=======================*/ +{ os_event_free(fil_crypt_event); os_event_free(fil_crypt_threads_event); } @@ -2319,7 +2430,8 @@ Mark a space as closing */ UNIV_INTERN void fil_space_crypt_mark_space_closing( - ulint space) +/*===============================*/ + ulint space) /*!< in: Space id */ { mutex_enter(&fil_crypt_threads_mutex); fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space); @@ -2340,7 +2452,8 @@ Wait for crypt threads to stop accessing space */ UNIV_INTERN void fil_space_crypt_close_tablespace( - ulint space) +/*=============================*/ + ulint space) /*!< in: Space id */ { mutex_enter(&fil_crypt_threads_mutex); fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space); @@ -2352,11 +2465,14 @@ fil_space_crypt_close_tablespace( uint start = time(0); uint last = start; + mutex_enter(&crypt_data->mutex); mutex_exit(&fil_crypt_threads_mutex); crypt_data->closing = true; + uint cnt = crypt_data->rotate_state.active_threads; bool flushing = crypt_data->rotate_state.flushing; + while (cnt > 0 || flushing) { mutex_exit(&crypt_data->mutex); /* release dict mutex so that scrub threads can release their @@ -2386,11 +2502,12 @@ fil_space_crypt_close_tablespace( /********************************************************************* Get crypt status for a space (used by information_schema) return 0 if crypt data present */ +UNIV_INTERN int fil_space_crypt_get_status( -/*==================*/ - ulint id, /*!< in: space id */ - struct fil_space_crypt_status_t* status) /*!< out: status */ +/*=======================*/ + ulint id, /*!< in: space id */ + struct fil_space_crypt_status_t* status) /*!< out: status */ { fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(id); @@ -2400,6 +2517,7 @@ fil_space_crypt_get_status( mutex_enter(&crypt_data->mutex); status->keyserver_requests = crypt_data->keyserver_requests; status->min_key_version = crypt_data->min_key_version; + if (crypt_data->rotate_state.active_threads > 0 || crypt_data->rotate_state.flushing) { status->rotating = true; @@ -2427,8 +2545,11 @@ fil_space_crypt_get_status( /********************************************************************* Return crypt statistics */ +UNIV_INTERN void -fil_crypt_total_stat(fil_crypt_stat_t *stat) +fil_crypt_total_stat( +/*=================*/ + fil_crypt_stat_t *stat) /*!< out: Crypt statistics */ { mutex_enter(&crypt_stat_mutex); *stat = crypt_stat; @@ -2438,11 +2559,12 @@ fil_crypt_total_stat(fil_crypt_stat_t *stat) /********************************************************************* Get scrub status for a space (used by information_schema) return 0 if data found */ +UNIV_INTERN int fil_space_get_scrub_status( -/*==================*/ - ulint id, /*!< in: space id */ - struct fil_space_scrub_status_t* status) /*!< out: status */ +/*=======================*/ + ulint id, /*!< in: space id */ + struct fil_space_scrub_status_t* status) /*!< out: status */ { fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(id); memset(status, 0, sizeof(*status)); diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 8b0a788b7b9..9d37de6d63b 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -27,8 +27,7 @@ Created 10/25/1995 Heikki Tuuri #include "fil0fil.h" #include "fil0pagecompress.h" #include "fsp0pagecompress.h" -#include "fil0pageencryption.h" -#include "fsp0pageencryption.h" +#include "fil0crypt.h" #include #include @@ -284,7 +283,7 @@ fil_read( actual page size does not decrease. */ { return(fil_io(OS_FILE_READ, sync, space_id, zip_size, block_offset, - byte_offset, len, buf, message, write_size, 0, false)); + byte_offset, len, buf, message, write_size)); } /********************************************************************//** @@ -311,18 +310,16 @@ fil_write( this must be appropriately aligned */ void* message, /*!< in: message for aio handler if non-sync aio used, else ignored */ - ulint* write_size, /*!< in/out: Actual write size initialized + ulint* write_size) /*!< in/out: Actual write size initialized after fist successfull trim operation for this page and if initialized we do not trim again if actual page size does not decrease. */ - lsn_t lsn, /*!< in: lsn of the newest modification */ - bool encrypt_later) /*!< in: encrypt later ? */ { ut_ad(!srv_read_only_mode); return(fil_io(OS_FILE_WRITE, sync, space_id, zip_size, block_offset, - byte_offset, len, buf, message, write_size, lsn, encrypt_later)); + byte_offset, len, buf, message, write_size)); } /*******************************************************************//** @@ -648,10 +645,9 @@ fil_node_open_file( set */ page = static_cast(ut_align(buf2, UNIV_PAGE_SIZE)); - success = os_file_read(node->handle, page, 0, UNIV_PAGE_SIZE, - space->flags); + success = os_file_read(node->handle, page, 0, UNIV_PAGE_SIZE); - if (fil_page_encryption_status(page)) { + if (fil_page_is_encrypted(page)) { /* if page is (still) encrypted, write an error and return. * Otherwise the server would crash if decrypting is not possible. * This may be the case, if the key file could not be @@ -1156,21 +1152,6 @@ fil_space_create( ut_a(fil_system); - if (fsp_flags_is_page_encrypted(flags)) { - if (!has_encryption_key(fsp_flags_get_page_encryption_key(flags))) { - /* by returning here it should be avoided that - * the server crashes, if someone tries to access an - * encrypted table and the encryption key is not available. - * The the table is treaded as non-existent. - */ - ib_logf(IB_LOG_LEVEL_WARN, - "Tablespace '%s' can not be opened, because " - " encryption key can not be found (space id: %lu, key %lu)\n" - , name, (ulong) id, fsp_flags_get_page_encryption_key(flags)); - return (FALSE); - } - } - /* Look for a matching tablespace and if found free it. */ do { mutex_enter(&fil_system->mutex); @@ -1836,7 +1817,7 @@ fil_write_lsn_and_arch_no_to_file( lsn); err = fil_write(TRUE, space, 0, sum_of_sizes, 0, - UNIV_PAGE_SIZE, buf, NULL, 0, lsn, false); + UNIV_PAGE_SIZE, buf, NULL, 0); } mem_free(buf1); @@ -1927,13 +1908,10 @@ fil_check_first_page( or the encryption key is not available, the check for reading the first page should intentionally fail with "can not decrypt" message. */ - page_is_encrypted = fil_page_encryption_status(page); - if (page_is_encrypted == PAGE_ENCRYPTION_KEY_MISSING && page_is_encrypted) { - page_is_encrypted = 1; - } else { - page_is_encrypted = 0; + page_is_encrypted = fil_page_encryption_status(page, space_id); + if (!page_is_encrypted) { if (UNIV_PAGE_SIZE != fsp_flags_get_page_size(flags)) { - fprintf(stderr, + fprintf(stderr, "InnoDB: Error: Current page size %lu != " " page size on page %lu\n", UNIV_PAGE_SIZE, fsp_flags_get_page_size(flags)); @@ -1963,7 +1941,7 @@ fil_check_first_page( /* this error message is interpreted by the calling method, which is * executed if the server starts in recovery mode. */ - return(MSG_CANNOT_DECRYPT); + return(FIL_MSG_CANNOT_DECRYPT); } } @@ -2016,10 +1994,7 @@ fil_read_first_page( page = static_cast(ut_align(buf, UNIV_PAGE_SIZE)); - os_file_read(data_file, page, 0, UNIV_PAGE_SIZE, - orig_space_id != ULINT_UNDEFINED ? - fil_space_is_page_compressed(orig_space_id) : - FALSE); + os_file_read(data_file, page, 0, UNIV_PAGE_SIZE); /* The FSP_HEADER on page 0 is only valid for the first file in a tablespace. So if this is not the first datafile, leave @@ -4082,8 +4057,7 @@ fil_user_tablespace_find_space_id( for (ulint j = 0; j < page_count; ++j) { - st = os_file_read(fsp->file, page, (j* page_size), page_size, - fsp_flags_is_page_compressed(fsp->flags)); + st = os_file_read(fsp->file, page, (j* page_size), page_size); if (!st) { ib_logf(IB_LOG_LEVEL_INFO, @@ -4229,7 +4203,7 @@ check_first_page: "%s in tablespace %s (table %s)", check_msg, fsp->filepath, tablename); fsp->success = FALSE; - if (strncmp(check_msg, MSG_CANNOT_DECRYPT, strlen(check_msg))==0) { + if (strncmp(check_msg, FIL_MSG_CANNOT_DECRYPT, strlen(check_msg))==0) { /* by returning here, it should be avoided, that the server crashes, * if started in recovery mode and can not decrypt tables, if * the key file can not be read. @@ -5221,7 +5195,7 @@ retry: success = os_aio(OS_FILE_WRITE, OS_AIO_SYNC, node->name, node->handle, buf, offset, page_size * n_pages, - node, NULL, 0, FALSE, 0, 0, 0, 0, false); + node, NULL, 0); #endif /* UNIV_HOTBACKUP */ @@ -5602,13 +5576,11 @@ fil_io( appropriately aligned */ void* message, /*!< in: message for aio handler if non-sync aio used, else ignored */ - ulint* write_size, /*!< in/out: Actual write size initialized + ulint* write_size) /*!< in/out: Actual write size initialized after fist successfull trim operation for this page and if initialized we do not trim again if actual page size does not decrease. */ - lsn_t lsn, /*!< in: lsn of the newest modification */ - bool encrypt_later) /*!< in: encrypt later ? */ { ulint mode; fil_space_t* space; @@ -5618,10 +5590,6 @@ fil_io( ulint wake_later; os_offset_t offset; ibool ignore_nonexistent_pages; - ibool page_compressed = FALSE; - ulint page_compression_level = 0; - ibool page_encrypted; - ulint page_encryption_key; is_log = type & OS_FILE_LOG; type = type & ~OS_FILE_LOG; @@ -5805,11 +5773,6 @@ fil_io( ut_a(byte_offset % OS_FILE_LOG_BLOCK_SIZE == 0); ut_a((len % OS_FILE_LOG_BLOCK_SIZE) == 0); - page_compressed = fsp_flags_is_page_compressed(space->flags); - page_compression_level = fsp_flags_get_page_compression_level(space->flags); - page_encrypted = fsp_flags_is_page_encrypted(space->flags); - page_encryption_key = fsp_flags_get_page_encryption_key(space->flags); - #ifdef UNIV_HOTBACKUP /* In mysqlbackup do normal i/o, not aio */ if (type == OS_FILE_READ) { @@ -5831,13 +5794,7 @@ fil_io( len, node, message, - write_size, - page_compressed, - page_compression_level, - page_encrypted, - page_encryption_key, - lsn, - encrypt_later); + write_size); #endif /* UNIV_HOTBACKUP */ @@ -6387,9 +6344,7 @@ fil_iterate( readptr = iter.crypt_io_buffer; } - if (!os_file_read(iter.file, readptr, offset, - (ulint) n_bytes, - fil_space_is_page_compressed(space_id))) { + if (!os_file_read(iter.file, readptr, offset, (ulint) n_bytes)) { ib_logf(IB_LOG_LEVEL_ERROR, "os_file_read() failed"); @@ -6540,8 +6495,7 @@ fil_tablespace_iterate( /* Read the first page and determine the page and zip size. */ - if (!os_file_read(file, page, 0, UNIV_PAGE_SIZE, - dict_tf_get_page_compression(table->flags))) { + if (!os_file_read(file, page, 0, UNIV_PAGE_SIZE)) { err = DB_IO_ERROR; @@ -6601,7 +6555,7 @@ fil_tablespace_iterate( if (iter.crypt_data != NULL) { /* clear crypt data from page 0 and write it back */ - os_file_read(file, page, 0, UNIV_PAGE_SIZE, 0); + os_file_read(file, page, 0, UNIV_PAGE_SIZE); fil_space_clear_crypt_data(page, crypt_data_offset); lsn_t lsn = mach_read_from_8(page + FIL_PAGE_LSN); if (callback.get_zip_size() == 0) { @@ -6779,79 +6733,6 @@ fil_system_exit(void) mutex_exit(&fil_system->mutex); } -/*******************************************************************//** -Return space name */ -char* -fil_space_name( -/*===========*/ - fil_space_t* space) /*!< in: space */ -{ - return (space->name); -} - -/*******************************************************************//** -Return space flags */ -ulint -fil_space_flags( -/*===========*/ - fil_space_t* space) /*!< in: space */ -{ - return (space->flags); -} - -/*******************************************************************//** -Return page type name */ -const char* -fil_get_page_type_name( -/*===================*/ - ulint page_type) /*!< in: FIL_PAGE_TYPE */ -{ - switch(page_type) { - case FIL_PAGE_PAGE_COMPRESSED: - return (const char*)"PAGE_COMPRESSED"; - case FIL_PAGE_INDEX: - return (const char*)"INDEX"; - case FIL_PAGE_UNDO_LOG: - return (const char*)"UNDO LOG"; - case FIL_PAGE_INODE: - return (const char*)"INODE"; - case FIL_PAGE_IBUF_FREE_LIST: - return (const char*)"IBUF_FREE_LIST"; - case FIL_PAGE_TYPE_ALLOCATED: - return (const char*)"ALLOCATED"; - case FIL_PAGE_IBUF_BITMAP: - return (const char*)"IBUF_BITMAP"; - case FIL_PAGE_TYPE_SYS: - return (const char*)"SYS"; - case FIL_PAGE_TYPE_TRX_SYS: - return (const char*)"TRX_SYS"; - case FIL_PAGE_TYPE_FSP_HDR: - return (const char*)"FSP_HDR"; - case FIL_PAGE_TYPE_XDES: - return (const char*)"XDES"; - case FIL_PAGE_TYPE_BLOB: - return (const char*)"BLOB"; - case FIL_PAGE_TYPE_ZBLOB: - return (const char*)"ZBLOB"; - case FIL_PAGE_TYPE_ZBLOB2: - return (const char*)"ZBLOB2"; - case FIL_PAGE_TYPE_COMPRESSED: - return (const char*)"ORACLE PAGE COMPRESSED"; - default: - return (const char*)"PAGE TYPE CORRUPTED"; - } -} -/****************************************************************//** -Get block size from fil node -@return block size*/ -ulint -fil_node_get_block_size( -/*====================*/ - fil_node_t* node) /*!< in: Node where to get block - size */ -{ - return (node->file_block_size); -} /****************************************************************** Get id of first tablespace or ULINT_UNDEFINED if none */ diff --git a/storage/innobase/fil/fil0pagecompress.cc b/storage/innobase/fil/fil0pagecompress.cc index caed038b4ee..1c3db26372d 100644 --- a/storage/innobase/fil/fil0pagecompress.cc +++ b/storage/innobase/fil/fil0pagecompress.cc @@ -83,172 +83,6 @@ static ulint srv_data_read, srv_data_written; /* Used for debugging */ //#define UNIV_PAGECOMPRESS_DEBUG 1 -/****************************************************************//** -For page compressed pages decompress the page after actual read -operation. */ -static -void -fil_decompress_page_2( -/*==================*/ - byte* page_buf, /*!< out: destination buffer for - uncompressed data */ - byte* buf, /*!< in: source compressed data */ - ulong len, /*!< in: length of output buffer.*/ - ulint* write_size) /*!< in/out: Actual payload size of - the compressed data. */ -{ - ulint page_type = mach_read_from_2(buf + FIL_PAGE_TYPE); - - if (page_type != FIL_PAGE_TYPE_COMPRESSED) { - /* It is not a compressed page */ - return; - } - - byte* ptr = buf + FIL_PAGE_DATA; - ulint version = mach_read_from_1(buf + FIL_PAGE_VERSION); - int err = 0; - - ut_a(version == 1); - - /* Read the original page type, before we compressed the data. */ - page_type = mach_read_from_2(buf + FIL_PAGE_ORIGINAL_TYPE_V1); - - ulint original_len = mach_read_from_2(buf + FIL_PAGE_ORIGINAL_SIZE_V1); - - if (original_len < UNIV_PAGE_SIZE_MIN - (FIL_PAGE_DATA + 8) - || original_len > UNIV_PAGE_SIZE_MAX - FIL_PAGE_DATA - || len < original_len + FIL_PAGE_DATA) { - ib_logf(IB_LOG_LEVEL_ERROR, - "Corruption: We try to uncompress corrupted page. " - "Original len %lu len %lu.", - original_len, len); - - fflush(stderr); - ut_error; - - } - - ulint algorithm = mach_read_from_1(buf + FIL_PAGE_ALGORITHM_V1); - - switch(algorithm) { - case PAGE_ZLIB_ALGORITHM: { - - err = uncompress(page_buf, &len, ptr, original_len); - - /* If uncompress fails it means that page is corrupted */ - if (err != Z_OK) { - - ib_logf(IB_LOG_LEVEL_ERROR, - "Corruption: Page is marked as compressed " - "but uncompress failed with error %d " - " size %lu len %lu.", - err, original_len, len); - - fflush(stderr); - - ut_error; - } - - break; - } -#ifdef HAVE_LZ4 - case PAGE_LZ4_ALGORITHM: { - - err = LZ4_decompress_fast( - (const char*) ptr, (char*) (page_buf), original_len); - - if (err < 0) { - ib_logf(IB_LOG_LEVEL_ERROR, - "Corruption: Page is marked as compressed" - " but decompression read only %d bytes" - " size %lu len %lu.", - err, original_len, len); - fflush(stderr); - - ut_error; - } - break; - } -#endif /* HAVE_LZ4 */ - -#ifdef HAVE_LZMA - case PAGE_LZMA_ALGORITHM: { - - lzma_ret ret; - size_t src_pos = 0; - size_t dst_pos = 0; - uint64_t memlimit = UINT64_MAX; - - ret = lzma_stream_buffer_decode( - &memlimit, - 0, - NULL, - ptr, - &src_pos, - original_len, - (page_buf), - &dst_pos, - len); - - - if (ret != LZMA_OK || (dst_pos <= 0 || dst_pos > len)) { - ib_logf(IB_LOG_LEVEL_ERROR, - "Corruption: Page is marked as compressed" - " but decompression read only %ld bytes" - " size %lu len %lu.", - dst_pos, original_len, len); - fflush(stderr); - - ut_error; - } - - break; - } -#endif /* HAVE_LZMA */ - -#ifdef HAVE_LZO - case PAGE_LZO_ALGORITHM: { - ulint olen = 0; - - err = lzo1x_decompress((const unsigned char *)ptr, - original_len,(unsigned char *)(page_buf), &olen, NULL); - - if (err != LZO_E_OK || (olen == 0 || olen > UNIV_PAGE_SIZE)) { - ib_logf(IB_LOG_LEVEL_ERROR, - "Corruption: Page is marked as compressed" - " but decompression read only %ld bytes" - " size %lu len %lu.", - olen, original_len, len); - fflush(stderr); - - ut_error; - } - break; - } -#endif /* HAVE_LZO */ - - default: - ib_logf(IB_LOG_LEVEL_ERROR, - " Corruption: Page is marked as compressed " - " but compression algorithm %s" - " is not known." - ,fil_get_compression_alg_name(algorithm)); - - fflush(stderr); - ut_error; - break; - } - - /* Leave the header alone */ - memmove(buf+FIL_PAGE_DATA, page_buf, original_len); - - mach_write_to_2(buf + FIL_PAGE_TYPE, page_type); - - ut_ad(memcmp(buf + FIL_PAGE_LSN + 4, - buf + (original_len + FIL_PAGE_DATA) - - FIL_PAGE_END_LSN_OLD_CHKSUM + 4, 4) == 0); -} - /****************************************************************//** For page compressed pages compress the page before actual write operation. @@ -289,8 +123,7 @@ fil_compress_page( if (orig_page_type == 0 || orig_page_type == FIL_PAGE_TYPE_FSP_HDR || orig_page_type == FIL_PAGE_TYPE_XDES || - orig_page_type == FIL_PAGE_PAGE_COMPRESSED || - orig_page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { + orig_page_type == FIL_PAGE_PAGE_COMPRESSED) { *out_len = len; return (buf); } @@ -579,16 +412,6 @@ fil_decompress_page( in_buf = page_buf; } - if (ptype == FIL_PAGE_TYPE_COMPRESSED) { - - fil_decompress_page_2(in_buf, buf, len, write_size); - // Need to free temporal buffer if no buffer was given - if (page_buf == NULL) { - ut_free(in_buf); - } - return; - } - /* Before actual decompress, make sure that page type is correct */ if (mach_read_from_4(buf+FIL_PAGE_SPACE_OR_CHKSUM) != BUF_NO_CHECKSUM_MAGIC || diff --git a/storage/innobase/fsp/fsp0fsp.cc b/storage/innobase/fsp/fsp0fsp.cc index ee1f2fd9510..1cf30a56a98 100644 --- a/storage/innobase/fsp/fsp0fsp.cc +++ b/storage/innobase/fsp/fsp0fsp.cc @@ -31,6 +31,7 @@ Created 11/29/1995 Heikki Tuuri #include "buf0buf.h" #include "fil0fil.h" +#include "fil0crypt.h" #include "mtr0log.h" #include "ut0byte.h" #include "page0page.h" diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 3a5f961a9bb..311fdee4d3b 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -79,6 +79,7 @@ this program; if not, write to the Free Software Foundation, Inc., #include "fsp0fsp.h" #include "sync0sync.h" #include "fil0fil.h" +#include "fil0crypt.h" #include "trx0xa.h" #include "row0merge.h" #include "dict0boot.h" @@ -103,7 +104,6 @@ this program; if not, write to the Free Software Foundation, Inc., #include "fts0priv.h" #include "page0zip.h" #include "fil0pagecompress.h" -#include "fil0pageencryption.h" #define thd_get_trx_isolation(X) ((enum_tx_isolation)thd_tx_isolation(X)) @@ -11047,8 +11047,6 @@ innobase_table_flags( modified by another thread while the table is being created. */ const ulint default_compression_level = page_zip_level; - const ulint default_encryption_key = srv_default_page_encryption_key; - *flags = 0; *flags2 = 0; @@ -11250,10 +11248,7 @@ index_bad: options->page_compressed, options->page_compression_level == 0 ? default_compression_level : options->page_compression_level, - options->atomic_writes, - options->page_encryption, - options->page_encryption_key == 0 ? - default_encryption_key : options->page_encryption_key); + options->atomic_writes); if (create_info->options & HA_LEX_CREATE_TMP_TABLE) { *flags2 |= DICT_TF2_TEMPORARY; @@ -19156,7 +19151,7 @@ static MYSQL_SYSVAR_UINT(default_page_encryption_key, srv_default_page_encryptio "Encryption key used for page encryption.", NULL, NULL, - DEFAULT_ENCRYPTION_KEY, 1, 255, 0); + FIL_DEFAULT_ENCRYPTION_KEY, 1, 255, 0); static MYSQL_SYSVAR_BOOL(scrub_log, srv_scrub_log, PLUGIN_VAR_OPCMDARG | PLUGIN_VAR_READONLY, diff --git a/storage/innobase/handler/i_s.cc b/storage/innobase/handler/i_s.cc index 65fcb7051ef..705886736ae 100644 --- a/storage/innobase/handler/i_s.cc +++ b/storage/innobase/handler/i_s.cc @@ -61,6 +61,8 @@ Modified Dec 29, 2014 Jan Lindström (Added sys_semaphore_waits) #include "btr0btr.h" #include "page0zip.h" #include "sync0arr.h" +#include "fil0fil.h" +#include "fil0crypt.h" /** structure associates a name string with a file page type and/or buffer page state. */ diff --git a/storage/innobase/include/dict0dict.h b/storage/innobase/include/dict0dict.h index 326b9e7c986..20014e05784 100644 --- a/storage/innobase/include/dict0dict.h +++ b/storage/innobase/include/dict0dict.h @@ -929,10 +929,8 @@ dict_tf_set( pages */ ulint page_compression_level, /*!< in: table page compression level */ - ulint atomic_writes, /*!< in: table atomic + ulint atomic_writes) /*!< in: table atomic writes option value*/ - bool page_encrypted,/*!< in: table uses page encryption */ - ulint page_encryption_key) /*!< in: page encryption key */ __attribute__((nonnull)); /********************************************************************//** Convert a 32 bit integer table flags to the 32 bit integer that is diff --git a/storage/innobase/include/dict0dict.ic b/storage/innobase/include/dict0dict.ic index 7c51faf844e..dd42b478c1f 100644 --- a/storage/innobase/include/dict0dict.ic +++ b/storage/innobase/include/dict0dict.ic @@ -543,9 +543,6 @@ dict_tf_is_valid( ulint data_dir = DICT_TF_HAS_DATA_DIR(flags); ulint atomic_writes = DICT_TF_GET_ATOMIC_WRITES(flags); - ulint page_encryption = DICT_TF_GET_PAGE_ENCRYPTION(flags); - ulint page_encryption_key = DICT_TF_GET_PAGE_ENCRYPTION_KEY(flags); - /* Make sure there are no bits that we do not know about. */ if (unused != 0) { fprintf(stderr, @@ -555,12 +552,10 @@ dict_tf_is_valid( "InnoDB: compact %ld atomic_blobs %ld\n" "InnoDB: unused %ld data_dir %ld zip_ssize %ld\n" "InnoDB: page_compression %ld page_compression_level %ld\n" - "InnoDB: atomic_writes %ld\n" - "InnoDB: page_encryption %ld page_encryption_key %ld\n", + "InnoDB: atomic_writes %ld\n", unused, compact, atomic_blobs, unused, data_dir, zip_ssize, - page_compression, page_compression_level, atomic_writes, - page_encryption, page_encryption_key + page_compression, page_compression_level, atomic_writes ); return(false); @@ -857,9 +852,7 @@ dict_tf_set( pages */ ulint page_compression_level, /*!< in: table page compression level */ - ulint atomic_writes, /*!< in: table atomic writes setup */ - bool page_encrypted, /*!< in: table uses page encryption */ - ulint page_encryption_key /*!< in: page encryption key */) + ulint atomic_writes) /*!< in: table atomic writes setup */ { atomic_writes_t awrites = (atomic_writes_t)atomic_writes; @@ -900,11 +893,6 @@ dict_tf_set( if (use_data_dir) { *flags |= (1 << DICT_TF_POS_DATA_DIR); } - - if (page_encrypted) { - *flags |= (1 << DICT_TF_POS_PAGE_ENCRYPTION) - | (page_encryption_key << DICT_TF_POS_PAGE_ENCRYPTION_KEY); - } } /********************************************************************//** @@ -927,10 +915,6 @@ dict_tf_to_fsp_flags( ulint fsp_flags; ulint page_compression = DICT_TF_GET_PAGE_COMPRESSION(table_flags); ulint page_compression_level = DICT_TF_GET_PAGE_COMPRESSION_LEVEL(table_flags); - - ulint page_encryption = DICT_TF_GET_PAGE_ENCRYPTION(table_flags); - ulint page_encryption_key = DICT_TF_GET_PAGE_ENCRYPTION_KEY(table_flags); - ulint atomic_writes = DICT_TF_GET_ATOMIC_WRITES(table_flags); DBUG_EXECUTE_IF("dict_tf_to_fsp_flags_failure", @@ -958,14 +942,6 @@ dict_tf_to_fsp_flags( if page compression is used for this table. */ fsp_flags |= FSP_FLAGS_SET_PAGE_COMPRESSION_LEVEL(fsp_flags, page_compression_level); - /* In addition, tablespace flags also contain if the page - encryption is used for this table. */ - fsp_flags |= FSP_FLAGS_SET_PAGE_ENCRYPTION(fsp_flags, page_encryption); - - /* In addition, tablespace flags also contain page encryption key if the page - encryption is used for this table. */ - fsp_flags |= FSP_FLAGS_SET_PAGE_ENCRYPTION_KEY(fsp_flags, page_encryption_key); - /* In addition, tablespace flags also contain flag if atomic writes is used for this table */ fsp_flags |= FSP_FLAGS_SET_ATOMIC_WRITES(fsp_flags, atomic_writes); @@ -1007,8 +983,6 @@ dict_sys_tables_type_to_tf( | DICT_TF_MASK_PAGE_COMPRESSION | DICT_TF_MASK_PAGE_COMPRESSION_LEVEL | DICT_TF_MASK_ATOMIC_WRITES - | DICT_TF_MASK_PAGE_ENCRYPTION - | DICT_TF_MASK_PAGE_ENCRYPTION_KEY ); @@ -1045,9 +1019,7 @@ dict_tf_to_sys_tables_type( | DICT_TF_MASK_DATA_DIR | DICT_TF_MASK_PAGE_COMPRESSION | DICT_TF_MASK_PAGE_COMPRESSION_LEVEL - | DICT_TF_MASK_ATOMIC_WRITES - | DICT_TF_MASK_PAGE_ENCRYPTION - | DICT_TF_MASK_PAGE_ENCRYPTION_KEY); + | DICT_TF_MASK_ATOMIC_WRITES); return(type); } diff --git a/storage/innobase/include/dict0pagecompress.ic b/storage/innobase/include/dict0pagecompress.ic index a71b2b34b07..811976434a8 100644 --- a/storage/innobase/include/dict0pagecompress.ic +++ b/storage/innobase/include/dict0pagecompress.ic @@ -42,8 +42,6 @@ dict_tf_verify_flags( ulint page_compression = DICT_TF_GET_PAGE_COMPRESSION(table_flags); ulint page_compression_level = DICT_TF_GET_PAGE_COMPRESSION_LEVEL(table_flags); ulint atomic_writes = DICT_TF_GET_ATOMIC_WRITES(table_flags); - ulint page_encryption = DICT_TF_GET_PAGE_ENCRYPTION(table_flags); - ulint page_encryption_key = DICT_TF_GET_PAGE_ENCRYPTION_KEY(table_flags); ulint post_antelope = FSP_FLAGS_GET_POST_ANTELOPE(fsp_flags); ulint zip_ssize = FSP_FLAGS_GET_ZIP_SSIZE(fsp_flags); ulint fsp_atomic_blobs = FSP_FLAGS_HAS_ATOMIC_BLOBS(fsp_flags); @@ -52,9 +50,6 @@ dict_tf_verify_flags( ulint fsp_page_compression = FSP_FLAGS_GET_PAGE_COMPRESSION(fsp_flags); ulint fsp_page_compression_level = FSP_FLAGS_GET_PAGE_COMPRESSION_LEVEL(fsp_flags); ulint fsp_atomic_writes = FSP_FLAGS_GET_ATOMIC_WRITES(fsp_flags); - ulint fsp_page_encryption = FSP_FLAGS_GET_PAGE_ENCRYPTION(fsp_flags); - ulint fsp_page_encryption_key = FSP_FLAGS_GET_PAGE_ENCRYPTION_KEY(fsp_flags); - DBUG_EXECUTE_IF("dict_tf_verify_flags_failure", return(ULINT_UNDEFINED);); @@ -112,27 +107,6 @@ dict_tf_verify_flags( return (FALSE); } - if (page_encryption != fsp_page_encryption) { - fprintf(stderr, - "InnoDB: Error: table flags has page_encryption %ld" - " in the data dictionary\n" - "InnoDB: but the flags in file has page_encryption %ld\n", - page_encryption, fsp_page_encryption); - - return (FALSE); - } - - if (page_encryption_key != fsp_page_encryption_key) { - fprintf(stderr, - "InnoDB: Error: table flags has page_encryption_key %ld" - " in the data dictionary\n" - "InnoDB: but the flags in file has page_encryption_key %ld\n", - page_encryption_key, fsp_page_encryption_key); - - return (FALSE); - } - - return(TRUE); } diff --git a/storage/innobase/include/fil0crypt.h b/storage/innobase/include/fil0crypt.h new file mode 100644 index 00000000000..9d02034e4b2 --- /dev/null +++ b/storage/innobase/include/fil0crypt.h @@ -0,0 +1,394 @@ +/***************************************************************************** + +Copyright (c) 2015, MariaDB Corporation. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA + +*****************************************************************************/ + +/**************************************************//** +@file include/fil0crypt.h +The low-level file system encryption support functions + +Created 04/01/2015 Jan Lindström +*******************************************************/ + +#ifndef fil0crypt_h +#define fil0crypt_h + +#define FIL_MSG_CANNOT_DECRYPT "can not decrypt" +#define FIL_ENCRYPTION_WRONG_KEY 1 +#define FIL_ENCRYPTION_WRONG_PAGE_TYPE 2 +#define FIL_ENCRYPTION_ERROR 3 +#define FIL_ENCRYPTION_KEY_MISSING 4 +#define FIL_ENCRYPTION_OK 0 +#define FIL_ENCRYPTION_WILL_NOT_ENCRYPT 5 + +/* This key will be used if nothing else is given */ +#define FIL_DEFAULT_ENCRYPTION_KEY 1 + +/** + * CRYPT_SCHEME_UNENCRYPTED + * + * Used as intermediate state when convering a space from unencrypted + * to encrypted + */ +/** + * CRYPT_SCHEME_1 + * + * AES_CTR / AES_CBC: + * L = AES_ECB(KEY, IV) + * CRYPT(PAGE) = AES_CRT(KEY=L, IV=C, PAGE) + */ + +#define CRYPT_SCHEME_1 1 +#define CRYPT_SCHEME_1_IV_LEN 16 +#define CRYPT_SCHEME_UNENCRYPTED 0 + +/* Currently supported encryption methods */ +typedef enum { + CRYPT_SCHEME_1_UNENCRYPTED = 0, /*keys[0].key_version)) { + /* accessing table would surely fail, because no key or no key provider available */ + return FIL_ENCRYPTION_KEY_MISSING; + } + } + } else { + ulint key = mach_read_from_4(buf + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); + if (!has_encryption_key(key)) { + return FIL_ENCRYPTION_KEY_MISSING; + } + } + return 0; +} diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index e4f16b24392..2c6c5804f65 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -26,7 +26,6 @@ Created 10/25/1995 Heikki Tuuri #ifndef fil0fil_h #define fil0fil_h -#define MSG_CANNOT_DECRYPT "can not decrypt" #include "univ.i" #ifndef UNIV_INNOCHECKSUM @@ -136,24 +135,6 @@ extern fil_addr_t fil_addr_null; used to encrypt the page + 32-bit checksum or 64 bits of zero if no encryption */ -/** If page type is FIL_PAGE_COMPRESSED then the 8 bytes starting at -FIL_PAGE_FILE_FLUSH_LSN are broken down as follows: */ - -/** Control information version format (u8) */ -static const ulint FIL_PAGE_VERSION = FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION; - -/** Compression algorithm (u8) */ -static const ulint FIL_PAGE_ALGORITHM_V1 = FIL_PAGE_VERSION + 1; - -/** Original page type (u16) */ -static const ulint FIL_PAGE_ORIGINAL_TYPE_V1 = FIL_PAGE_ALGORITHM_V1 + 1; - -/** Original data size in bytes (u16)*/ -static const ulint FIL_PAGE_ORIGINAL_SIZE_V1 = FIL_PAGE_ORIGINAL_TYPE_V1 + 2; - -/** Size after compression (u16)*/ -static const ulint FIL_PAGE_COMPRESS_SIZE_V1 = FIL_PAGE_ORIGINAL_SIZE_V1 + 2; - #define FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID 34 /*!< starting from 4.1.x this contains the space id of the page */ #define FIL_PAGE_SPACE_ID FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID @@ -173,10 +154,7 @@ static const ulint FIL_PAGE_COMPRESS_SIZE_V1 = FIL_PAGE_ORIGINAL_SIZE_V1 + 2; /* @} */ /** File page types (values of FIL_PAGE_TYPE) @{ */ -#define FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED 35631 /* page compressed + - encrypted page */ #define FIL_PAGE_PAGE_COMPRESSED 34354 /*!< page compressed page */ -#define FIL_PAGE_PAGE_ENCRYPTED 34355 /*!< Page encrypted page */ #define FIL_PAGE_INDEX 17855 /*!< B-tree node */ #define FIL_PAGE_UNDO_LOG 2 /*!< Undo log page */ #define FIL_PAGE_INODE 3 /*!< Index node */ @@ -203,6 +181,9 @@ static const ulint FIL_PAGE_COMPRESS_SIZE_V1 = FIL_PAGE_ORIGINAL_SIZE_V1 + 2; #ifndef UNIV_INNOCHECKSUM +/* structure containing encryption specification */ +typedef struct fil_space_crypt_struct fil_space_crypt_t; + /** The number of fsyncs done to the log */ extern ulint fil_n_log_flushes; @@ -214,9 +195,6 @@ extern ulint fil_n_pending_tablespace_flushes; /** Number of files currently open */ extern ulint fil_n_file_opened; -/* structure containing encryption specification */ -typedef struct fil_space_crypt_struct fil_space_crypt_t; - struct fsp_open_info { ibool success; /*!< Has the tablespace been opened? */ const char* check_msg; /*!< fil_check_first_page() message */ @@ -997,13 +975,11 @@ fil_io( appropriately aligned */ void* message, /*!< in: message for aio handler if non-sync aio used, else ignored */ - ulint* write_size, /*!< in/out: Actual write size initialized + ulint* write_size) /*!< in/out: Actual write size initialized after fist successfull trim operation for this page and if initialized we do not trim again if actual page size does not decrease. */ - lsn_t lsn, /*!< in: lsn of the newest modification */ - bool encrypt_later) /*!< in: should we encrypt the page */ __attribute__((nonnull(8))); /**********************************************************************//** Waits for an aio operation to complete. This function is used to write the @@ -1256,6 +1232,7 @@ fil_user_tablespace_restore_page( /*******************************************************************//** Return space flags */ +UNIV_INLINE ulint fil_space_flags( /*===========*/ @@ -1281,12 +1258,6 @@ fil_space_t* fil_space_get_by_id( /*================*/ ulint id); /*!< in: space id */ -/*******************************************************************//** -Return space name */ -char* -fil_space_name( -/*===========*/ - fil_space_t* space); /*!< in: space */ /****************************************************************** Get id of first tablespace or ULINT_UNDEFINED if none */ @@ -1301,270 +1272,13 @@ ulint fil_get_next_space( ulint id); /*!< in: space id */ -/********************************************************************* -Init global resources needed for tablespace encryption/decryption */ -void -fil_space_crypt_init(); - -/********************************************************************* -Cleanup global resources needed for tablespace encryption/decryption */ -void -fil_space_crypt_cleanup(); - -/********************************************************************* -Create crypt data, i.e data that is used for a single tablespace */ -fil_space_crypt_t * -fil_space_create_crypt_data(); - -/********************************************************************* -Destroy crypt data */ -UNIV_INTERN -void -fil_space_destroy_crypt_data( -/*=========================*/ - fil_space_crypt_t **crypt_data); /*!< in/out: crypt data */ - -/********************************************************************* -Get crypt data for a space*/ -fil_space_crypt_t * -fil_space_get_crypt_data( -/*======================*/ - ulint space); /*!< in: tablespace id */ - -/********************************************************************* -Set crypt data for a space*/ -void -fil_space_set_crypt_data( -/*======================*/ - ulint space, /*!< in: tablespace id */ - fil_space_crypt_t* crypt_data); /*!< in: crypt data */ - -/********************************************************************* -Compare crypt data*/ -int -fil_space_crypt_compare( -/*======================*/ - const fil_space_crypt_t* crypt_data1, /*!< in: crypt data */ - const fil_space_crypt_t* crypt_data2); /*!< in: crypt data */ - -/********************************************************************* -Read crypt data from buffer page */ -fil_space_crypt_t * -fil_space_read_crypt_data( -/*======================*/ - ulint space, /*!< in: tablespace id */ - const byte* page, /*!< in: buffer page */ - ulint offset); /*!< in: offset where crypt data is stored */ - -/********************************************************************* -Write crypt data to buffer page */ -void -fil_space_write_crypt_data( -/*=======================*/ - ulint space, /*!< in: tablespace id */ - byte* page, /*!< in: buffer page */ - ulint offset, /*!< in: offset where to store data */ - ulint maxsize, /*!< in: max space available to store crypt data in */ - mtr_t * mtr); /*!< in: mini-transaction */ - -/********************************************************************* -Clear crypt data from page 0 (used for import tablespace) */ -void -fil_space_clear_crypt_data( -/*======================*/ - byte* page, /*!< in: buffer page */ - ulint offset); /*!< in: offset where crypt data is stored */ - -/********************************************************************* -Parse crypt data log record */ -byte* -fil_parse_write_crypt_data( -/*=======================*/ - byte* ptr, /*!< in: start of log record */ - byte* end_ptr, /*!< in: end of log record */ - buf_block_t*); /*!< in: buffer page to apply record to */ - -/********************************************************************* -Check if extra buffer shall be allocated for decrypting after read */ -UNIV_INTERN -bool -fil_space_check_encryption_read( -/*==============================*/ - ulint space); /*!< in: tablespace id */ - -/********************************************************************* -Check if page shall be encrypted before write */ -UNIV_INTERN -bool -fil_space_check_encryption_write( -/*==============================*/ - ulint space); /*!< in: tablespace id */ - -/********************************************************************* -Encrypt buffer page */ -void -fil_space_encrypt( -/*===============*/ - ulint space, /*!< in: tablespace id */ - ulint offset, /*!< in: page no */ - lsn_t lsn, /*!< in: page lsn */ - const byte* src_frame,/*!< in: page frame */ - ulint size, /*!< in: size of data to encrypt */ - byte* dst_frame, /*!< in: where to encrypt to */ - ulint page_encryption_key); /*!< in: page encryption key id if page - encrypted */ - -/********************************************************************* -Decrypt buffer page */ -void -fil_space_decrypt( -/*===============*/ - ulint space, /*!< in: tablespace id */ - const byte* src_frame,/*!< in: page frame */ - ulint page_size, /*!< in: size of data to encrypt */ - byte* dst_frame); /*!< in: where to decrypt to */ - - -/********************************************************************* -Decrypt buffer page -@return true if page was encrypted */ -bool -fil_space_decrypt( -/*===============*/ - fil_space_crypt_t* crypt_data, /*!< in: crypt data */ - const byte* src_frame,/*!< in: page frame */ - ulint page_size, /*!< in: page size */ - byte* dst_frame); /*!< in: where to decrypt to */ - -/********************************************************************* -fil_space_verify_crypt_checksum -NOTE: currently this function can only be run in single threaded mode -as it modifies srv_checksum_algorithm (temporarily) -@return true if page is encrypted AND OK, false otherwise */ -bool -fil_space_verify_crypt_checksum( -/*===============*/ - const byte* src_frame,/*!< in: page frame */ - ulint zip_size); /*!< in: size of data to encrypt */ - -/********************************************************************* -Init threads for key rotation */ -void -fil_crypt_threads_init(); - -/********************************************************************* -Set thread count (e.g start or stops threads) used for key rotation */ -void -fil_crypt_set_thread_cnt( -/*=====================*/ - uint new_cnt); /*!< in: requested #threads */ - -/********************************************************************* -End threads for key rotation */ -void -fil_crypt_threads_end(); - -/********************************************************************* -Cleanup resources for threads for key rotation */ -void -fil_crypt_threads_cleanup(); - -/********************************************************************* -Set rotate key age */ -void -fil_crypt_set_rotate_key_age( -/*=====================*/ - uint rotate_age); /*!< in: requested rotate age */ - -/********************************************************************* -Set rotation threads iops */ -void -fil_crypt_set_rotation_iops( -/*=====================*/ - uint iops); /*!< in: requested iops */ - -/********************************************************************* -Mark a space as closing */ -UNIV_INTERN -void -fil_space_crypt_mark_space_closing( -/*===============*/ - ulint space); /*!< in: tablespace id */ - -/********************************************************************* -Wait for crypt threads to stop accessing space */ -UNIV_INTERN -void -fil_space_crypt_close_tablespace( -/*===============*/ - ulint space); /*!< in: tablespace id */ - -/** Struct for retreiving info about encryption */ -struct fil_space_crypt_status_t { - ulint space; /*!< tablespace id */ - ulint scheme; /*!< encryption scheme */ - uint min_key_version; /*!< min key version */ - uint current_key_version;/*!< current key version */ - uint keyserver_requests;/*!< no of key requests to key server */ - bool rotating; /*!< is key rotation ongoing */ - bool flushing; /*!< is flush at end of rotation ongoing */ - ulint rotate_next_page_number; /*!< next page if key rotating */ - ulint rotate_max_page_number; /*!< max page if key rotating */ -}; - -/********************************************************************* -Get crypt status for a space -@return 0 if crypt data found */ -int -fil_space_crypt_get_status( -/*==================*/ - ulint id, /*!< in: space id */ - struct fil_space_crypt_status_t * status); /*!< out: status */ - -/** Struct for retreiving statistics about encryption key rotation */ -struct fil_crypt_stat_t { - ulint pages_read_from_cache; - ulint pages_read_from_disk; - ulint pages_modified; - ulint pages_flushed; - ulint estimated_iops; -}; - -/********************************************************************* -Get crypt rotation statistics */ -void -fil_crypt_total_stat( -/*==================*/ - fil_crypt_stat_t* stat); /*!< out: crypt stat */ - -/** Struct for retreiving info about scrubbing */ -struct fil_space_scrub_status_t { - ulint space; /*!< tablespace id */ - bool compressed; /*!< is space compressed */ - time_t last_scrub_completed; /*!< when was last scrub completed */ - bool scrubbing; /*!< is scrubbing ongoing */ - time_t current_scrub_started; /*!< when started current scrubbing */ - ulint current_scrub_active_threads; /*!< current scrub active threads */ - ulint current_scrub_page_number; /*!< current scrub page no */ - ulint current_scrub_max_page_number; /*!< current scrub max page no */ -}; - -/********************************************************************* -Get scrub status for a space -@return 0 if no scrub info found */ -int -fil_space_get_scrub_status( -/*==================*/ - ulint id, /*!< in: space id */ - struct fil_space_scrub_status_t * status); /*!< out: status */ #endif -/*******************************************************************//** -Return page type name */ -const char* -fil_get_page_type_name( -/*===================*/ - ulint page_type); /*!< in: FIL_PAGE_TYPE */ +#ifndef UNIV_INNOCHECKSUM +#ifndef UNIV_NONINL +#include "fil0fil.ic" +#endif +#endif #endif /* fil0fil_h */ diff --git a/storage/innobase/include/fil0fil.ic b/storage/innobase/include/fil0fil.ic new file mode 100644 index 00000000000..b1e65e6dddb --- /dev/null +++ b/storage/innobase/include/fil0fil.ic @@ -0,0 +1,108 @@ +/***************************************************************************** + +Copyright (c) 2015, MariaDB Corporation. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA + +*****************************************************************************/ + +/**************************************************//** +@file include/fil0fil.ic +The low-level file system support functions + +Created 31/03/2015 Jan Lindström +*******************************************************/ + +#ifndef fil0fil_ic +#define fil0fil_ic + +/*******************************************************************//** +Return space name */ +UNIV_INLINE +char* +fil_space_name( +/*===========*/ + fil_space_t* space) /*!< in: space */ +{ + return (space->name); +} + +/*******************************************************************//** +Return space flags */ +UNIV_INLINE +ulint +fil_space_flags( +/*===========*/ + fil_space_t* space) /*!< in: space */ +{ + return (space->flags); +} + +/*******************************************************************//** +Return page type name */ +UNIV_INLINE +const char* +fil_get_page_type_name( +/*===================*/ + ulint page_type) /*!< in: FIL_PAGE_TYPE */ +{ + switch(page_type) { + case FIL_PAGE_PAGE_COMPRESSED: + return (const char*)"PAGE_COMPRESSED"; + case FIL_PAGE_INDEX: + return (const char*)"INDEX"; + case FIL_PAGE_UNDO_LOG: + return (const char*)"UNDO LOG"; + case FIL_PAGE_INODE: + return (const char*)"INODE"; + case FIL_PAGE_IBUF_FREE_LIST: + return (const char*)"IBUF_FREE_LIST"; + case FIL_PAGE_TYPE_ALLOCATED: + return (const char*)"ALLOCATED"; + case FIL_PAGE_IBUF_BITMAP: + return (const char*)"IBUF_BITMAP"; + case FIL_PAGE_TYPE_SYS: + return (const char*)"SYS"; + case FIL_PAGE_TYPE_TRX_SYS: + return (const char*)"TRX_SYS"; + case FIL_PAGE_TYPE_FSP_HDR: + return (const char*)"FSP_HDR"; + case FIL_PAGE_TYPE_XDES: + return (const char*)"XDES"; + case FIL_PAGE_TYPE_BLOB: + return (const char*)"BLOB"; + case FIL_PAGE_TYPE_ZBLOB: + return (const char*)"ZBLOB"; + case FIL_PAGE_TYPE_ZBLOB2: + return (const char*)"ZBLOB2"; + case FIL_PAGE_TYPE_COMPRESSED: + return (const char*)"ORACLE PAGE COMPRESSED"; + default: + return (const char*)"PAGE TYPE CORRUPTED"; + } +} + +/****************************************************************//** +Get block size from fil node +@return block size*/ +UNIV_INLINE +ulint +fil_node_get_block_size( +/*====================*/ + fil_node_t* node) /*!< in: Node where to get block + size */ +{ + return (node->file_block_size); +} + +#endif /* fil0fil_ic */ diff --git a/storage/innobase/include/fil0pageencryption.h b/storage/innobase/include/fil0pageencryption.h deleted file mode 100644 index 405b91c8c06..00000000000 --- a/storage/innobase/include/fil0pageencryption.h +++ /dev/null @@ -1,79 +0,0 @@ -/***************************************************************************** - -Copyright (C) 2014 eperi GmbH. All Rights Reserved. - -This program is free software; you can redistribute it and/or modify it under -the terms of the GNU General Public License as published by the Free Software -Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, but WITHOUT -ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - -You should have received a copy of the GNU General Public License along with -this program; if not, write to the Free Software Foundation, Inc., -51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - -*****************************************************************************/ - -#ifndef fil0pageencryption_h -#define fil0pageencryption_h - -#define PAGE_ENCRYPTION_WRONG_KEY 1 -#define PAGE_ENCRYPTION_WRONG_PAGE_TYPE 2 -#define PAGE_ENCRYPTION_ERROR 3 -#define PAGE_ENCRYPTION_KEY_MISSING 4 -#define PAGE_ENCRYPTION_OK 0 -#define PAGE_ENCRYPTION_WILL_NOT_ENCRYPT 5 - -/* This key will be used if nothing else is given */ -#define DEFAULT_ENCRYPTION_KEY 1 - -#include "fsp0fsp.h" -#include "fsp0pageencryption.h" - -/******************************************************************//** -@file include/fil0pageencryption.h -Helper functions for encryption/decryption page data on to table space. - -Created 08/25/2014 -***********************************************************************/ - -/*******************************************************************//** -Find out whether the page is page encrypted -Returns the page encryption flag of the space, or false if the space -is not encrypted. The tablespace must be cached in the memory cache. -@return true if page encrypted, false if not or space not found */ -ibool -fil_space_is_page_encrypted( -/*========================*/ - ulint id); /*!< in: space id */ - -/*******************************************************************//** -Find out whether the page is page encrypted -@return true if page is page encrypted, false if not */ -UNIV_INLINE -ibool -fil_page_is_encrypted( -/*==================*/ - const byte *buf); /*!< in: page */ - -/*******************************************************************//** -Find out whether the page is page compressed and then encrypted -@return true if page is page compressed+encrypted, false if not */ -UNIV_INLINE -ibool -fil_page_is_compressed_encrypted( -/*=============================*/ - const byte *buf); /*!< in: page */ - -/*******************************************************************//** -Find out whether the page can be decrypted -@return true if page can be decrypted, false if not. */ -UNIV_INLINE -ulint -fil_page_encryption_status( -/*=======================*/ - const byte *buf); /*!< in: page */ - -#endif // fil0pageencryption_h diff --git a/storage/innobase/include/fsp0fsp.h b/storage/innobase/include/fsp0fsp.h index ee1ab4c7f55..2bac42eb081 100644 --- a/storage/innobase/include/fsp0fsp.h +++ b/storage/innobase/include/fsp0fsp.h @@ -59,10 +59,6 @@ is found in a remote location, not the default data directory. */ #define FSP_FLAGS_WIDTH_PAGE_COMPRESSION 1 #define FSP_FLAGS_WIDTH_PAGE_COMPRESSION_LEVEL 4 -/** Number of flag bits used to indicate the page compression and compression level */ -#define FSP_FLAGS_WIDTH_PAGE_ENCRYPTION 1 -#define FSP_FLAGS_WIDTH_PAGE_ENCRYPTION_KEY 8 - /** Number of flag bits used to indicate atomic writes for this tablespace */ #define FSP_FLAGS_WIDTH_ATOMIC_WRITES 2 @@ -74,9 +70,7 @@ is found in a remote location, not the default data directory. */ + FSP_FLAGS_WIDTH_DATA_DIR \ + FSP_FLAGS_WIDTH_PAGE_COMPRESSION \ + FSP_FLAGS_WIDTH_PAGE_COMPRESSION_LEVEL \ - + FSP_FLAGS_WIDTH_ATOMIC_WRITES \ - + FSP_FLAGS_WIDTH_PAGE_ENCRYPTION \ - + FSP_FLAGS_WIDTH_PAGE_ENCRYPTION_KEY) + + FSP_FLAGS_WIDTH_ATOMIC_WRITES ) /** A mask of all the known/used bits in tablespace flags */ #define FSP_FLAGS_MASK (~(~0 << FSP_FLAGS_WIDTH)) @@ -100,15 +94,9 @@ dictionary */ /** Zero relative shift position of the ATOMIC_WRITES field */ #define FSP_FLAGS_POS_ATOMIC_WRITES (FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL \ + FSP_FLAGS_WIDTH_PAGE_COMPRESSION_LEVEL) -/** Zero relative shift position of the PAGE_ENCRYPTION field */ -#define FSP_FLAGS_POS_PAGE_ENCRYPTION (FSP_FLAGS_POS_ATOMIC_WRITES \ - + FSP_FLAGS_WIDTH_ATOMIC_WRITES) -/** Zero relative shift position of the PAGE_ENCRYPTION_KEY field */ -#define FSP_FLAGS_POS_PAGE_ENCRYPTION_KEY (FSP_FLAGS_POS_PAGE_ENCRYPTION \ - + FSP_FLAGS_WIDTH_PAGE_ENCRYPTION) /** Zero relative shift position of the PAGE_SSIZE field */ -#define FSP_FLAGS_POS_PAGE_SSIZE (FSP_FLAGS_POS_PAGE_ENCRYPTION_KEY \ - + FSP_FLAGS_WIDTH_PAGE_ENCRYPTION_KEY) +#define FSP_FLAGS_POS_PAGE_SSIZE (FSP_FLAGS_POS_ATOMIC_WRITES \ + + FSP_FLAGS_WIDTH_ATOMIC_WRITES) /** Zero relative shift position of the start of the UNUSED bits */ #define FSP_FLAGS_POS_DATA_DIR (FSP_FLAGS_POS_PAGE_SSIZE \ + FSP_FLAGS_WIDTH_PAGE_SSIZE) @@ -144,14 +132,6 @@ dictionary */ #define FSP_FLAGS_MASK_PAGE_COMPRESSION_LEVEL \ ((~(~0 << FSP_FLAGS_WIDTH_PAGE_COMPRESSION_LEVEL)) \ << FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL) -/** Bit mask of the PAGE_ENCRYPTION field */ -#define FSP_FLAGS_MASK_PAGE_ENCRYPTION \ - ((~(~0 << FSP_FLAGS_WIDTH_PAGE_ENCRYPTION)) \ - << FSP_FLAGS_POS_PAGE_ENCRYPTION) -/** Bit mask of the PAGE_ENCRYPTION_KEY field */ -#define FSP_FLAGS_MASK_PAGE_ENCRYPTION_KEY \ - ((~(~0 << FSP_FLAGS_WIDTH_PAGE_ENCRYPTION_KEY)) \ - << FSP_FLAGS_POS_PAGE_ENCRYPTION_KEY) /** Bit mask of the ATOMIC_WRITES field */ #define FSP_FLAGS_MASK_ATOMIC_WRITES \ ((~(~0 << FSP_FLAGS_WIDTH_ATOMIC_WRITES)) \ @@ -192,14 +172,6 @@ dictionary */ #define FSP_FLAGS_GET_ATOMIC_WRITES(flags) \ ((flags & FSP_FLAGS_MASK_ATOMIC_WRITES) \ >> FSP_FLAGS_POS_ATOMIC_WRITES) -/** Return the value of the PAGE_ENCRYPTION field */ -#define FSP_FLAGS_GET_PAGE_ENCRYPTION(flags) \ - ((flags & FSP_FLAGS_MASK_PAGE_ENCRYPTION) \ - >> FSP_FLAGS_POS_PAGE_ENCRYPTION) -/** Return the value of the PAGE_ENCRYPTION_KEY field */ -#define FSP_FLAGS_GET_PAGE_ENCRYPTION_KEY(flags) \ - ((flags & FSP_FLAGS_MASK_PAGE_ENCRYPTION_KEY) \ - >> FSP_FLAGS_POS_PAGE_ENCRYPTION_KEY) /** Set a PAGE_SSIZE into the correct bits in a given tablespace flags. */ @@ -216,13 +188,6 @@ tablespace flags. */ #define FSP_FLAGS_SET_PAGE_COMPRESSION_LEVEL(flags, level) \ (flags | (level << FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL)) -/** Set a PAGE_ENCRYPTION into the correct bits in a given tablespace flags. */ -#define FSP_FLAGS_SET_PAGE_ENCRYPTION(flags, encryption) \ - (flags | (encryption << FSP_FLAGS_POS_PAGE_ENCRYPTION)) -/** Set a PAGE_ENCRYPTION_KEY into the correct bits in a given tablespace flags. */ -#define FSP_FLAGS_SET_PAGE_ENCRYPTION_KEY(flags, encryption_key) \ - (flags | (encryption_key << FSP_FLAGS_POS_PAGE_ENCRYPTION_KEY)) - /** Set a ATOMIC_WRITES into the correct bits in a given tablespace flags. */ #define FSP_FLAGS_SET_ATOMIC_WRITES(flags, atomics) \ diff --git a/storage/innobase/include/fsp0pageencryption.h b/storage/innobase/include/fsp0pageencryption.h deleted file mode 100644 index 52365c8e93c..00000000000 --- a/storage/innobase/include/fsp0pageencryption.h +++ /dev/null @@ -1,66 +0,0 @@ -/***************************************************************************** - - Copyright (C) 2014 eperi GmbH. All Rights Reserved. - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; version 2 of the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ - -/******************************************************************/ - -/******************************************************************//** -@file include/fsp0pageencryption.h -Helper functions for extracting/storing page encryption information to file space. - -Created 08/28/2014 -***********************************************************************/ - -#ifndef FSP0PAGEENCRYPTION_H_ -#define FSP0PAGEENCRYPTION_H_ - - -#define FIL_PAGE_ENCRYPTION_AES_128 16 /*!< Encryption algorithm AES-128. */ -#define FIL_PAGE_ENCRYPTION_AES_196 24 /*!< Encryption algorithm AES-196. */ -#define FIL_PAGE_ENCRYPTION_AES_256 32 /*!< Encryption algorithm AES-256. */ - -#define FIL_PAGE_ENCRYPTED_SIZE 2 /*!< Number of bytes used to store - actual payload data size on encrypted pages. */ - -/********************************************************************//** -Determine if the tablespace is page encrypted from dict_table_t::flags. -@return TRUE if page encrypted, FALSE if not page encrypted */ -UNIV_INLINE -ibool -fsp_flags_is_page_encrypted( -/*=========================*/ - ulint flags); /*!< in: tablespace flags */ - - -/********************************************************************//** -Extract the page encryption key from tablespace flags. -A tablespace has only one physical page encryption key -whether that page is encrypted or not. -@return page encryption key of the file-per-table tablespace, -or zero if the table is not encrypted. */ -UNIV_INLINE -ulint -fsp_flags_get_page_encryption_key( -/*=================================*/ - ulint flags); /*!< in: tablespace flags */ - - -#ifndef UNIV_NONINL -#include "fsp0pageencryption.ic" -#endif - - -#endif /* FSP0PAGEENCRYPTION_H_ */ diff --git a/storage/innobase/include/fsp0pageencryption.ic b/storage/innobase/include/fsp0pageencryption.ic deleted file mode 100644 index b5c3f5ab666..00000000000 --- a/storage/innobase/include/fsp0pageencryption.ic +++ /dev/null @@ -1,167 +0,0 @@ -/***************************************************************************** - - Copyright (C) 2014 eperi GmbH. All Rights Reserved. - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; version 2 of the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ - -/******************************************************************//** -@file include/fsp0pageencryption.ic -Implementation for helper functions for encrypting/decrypting pages -and atomic writes information to file space. - -Created 08/28/2014 -***********************************************************************/ - -#include "fsp0fsp.h" -#include "fil0pageencryption.h" - - -/********************************************************************//** -Determine if the tablespace is page encrypted from dict_table_t::flags. -@return TRUE if page encrypted, FALSE if not page encrypted */ -UNIV_INLINE -ibool -fsp_flags_is_page_encrypted( -/*=========================*/ - ulint flags) /*!< in: tablespace flags */ -{ - return(FSP_FLAGS_GET_PAGE_ENCRYPTION(flags)); -} - -/********************************************************************//** -Extract the page encryption key from tablespace flags. -A tablespace has only one physical page encryption key -whether that page is encrypted or not. -@return page encryption key of the file-per-table tablespace, -or zero if the table is not encrypted. */ -UNIV_INLINE -ulint -fsp_flags_get_page_encryption_key( -/*=================================*/ - ulint flags) /*!< in: tablespace flags */ -{ - return(FSP_FLAGS_GET_PAGE_ENCRYPTION_KEY(flags)); -} - - -/*******************************************************************//** -Returns the page encryption flag of the space, or false if the space -is not encrypted. The tablespace must be cached in the memory cache. -@return true if page encrypted, false if not or space not found */ -UNIV_INLINE -ibool -fil_space_is_page_encrypted( -/*=========================*/ - ulint id) /*!< in: space id */ -{ - ulint flags; - - flags = fil_space_get_flags(id); - - if (flags && flags != ULINT_UNDEFINED) { - - return(fsp_flags_is_page_encrypted(flags)); - } - - return(flags); -} - -/*******************************************************************//** -Returns the page encryption key of the space, or 0 if the space -is not encrypted. The tablespace must be cached in the memory cache. -@return page compression level, ULINT_UNDEFINED if space not found */ -UNIV_INLINE -ulint -fil_space_get_page_encryption_key( -/*=================================*/ - ulint id) /*!< in: space id */ -{ - ulint flags; - - flags = fil_space_get_flags(id); - - if (flags && flags != ULINT_UNDEFINED) { - - return(fsp_flags_get_page_encryption_key(flags)); - } - - return(flags); -} - -/*******************************************************************//** -Find out whether the page is page is encrypted -@return true if page is page encrypted, false if not */ -UNIV_INLINE -ibool -fil_page_is_encrypted( -/*==================*/ - const byte *buf) /*!< in: page */ -{ - return(mach_read_from_2(buf+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_ENCRYPTED); -} - -/*******************************************************************//** -Find out whether the page is page is first compressed and then encrypted -@return true if page is page compressed+encrypted, false if not */ -UNIV_INLINE -ibool -fil_page_is_compressed_encrypted( -/*=============================*/ - const byte *buf) /*!< in: page */ -{ - return(mach_read_from_2(buf+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED); -} - -/*******************************************************************//** -Find out whether the page can be decrypted. -This is the case, if the page is already decrypted and is not the first page of the table space. -If the page is already decrypted it is not of the FIL_PAGE_PAGE_ENCRYPTED type. -if it is the first page of the table space, it is assumed that a page can be decrypted if the -key found in the flags (part of the 1st page) can be read from the key provider. -The case, if the key changed, is currently not caught. -The function for decrypting the page should already be executed before this. -@return PAGE_ENCRYPTION_KEY_MISSING if key provider is available, but key is not available - PAGE_ENCRYPTION_ERROR if other error occurred - 0 if decryption should be possible -*/ -UNIV_INLINE -ulint -fil_page_encryption_status( -/*=====================*/ - const byte *buf) /*!< in: page */ -{ - ulint page_type = mach_read_from_2(buf+FIL_PAGE_TYPE); - - if (page_type == FIL_PAGE_TYPE_FSP_HDR) { - ulint flags = mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + buf); - if (fsp_flags_is_page_encrypted(flags)) { - if (!has_encryption_key(fsp_flags_get_page_encryption_key(flags))) { - /* accessing table would surely fail, because no key or no key provider available */ - if (!has_encryption_key(fsp_flags_get_page_encryption_key(flags))) { - return PAGE_ENCRYPTION_KEY_MISSING; - } - return PAGE_ENCRYPTION_ERROR; - } - } - } - - if(page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { - ulint key = mach_read_from_4(buf + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); - if (!has_encryption_key(key)) { - return PAGE_ENCRYPTION_KEY_MISSING; - } - return PAGE_ENCRYPTION_ERROR; - } - return 0; -} diff --git a/storage/innobase/include/os0file.h b/storage/innobase/include/os0file.h index 99490fa044e..fe6b755dc59 100644 --- a/storage/innobase/include/os0file.h +++ b/storage/innobase/include/os0file.h @@ -312,25 +312,21 @@ The wrapper functions have the prefix of "innodb_". */ pfs_os_file_close_func(file, __FILE__, __LINE__) # define os_aio(type, mode, name, file, buf, offset, \ - n, message1, message2, write_size, \ - page_compression, page_compression_level, \ - page_encryption, page_encryption_key, lsn, encrypt) \ + n, message1, message2, write_size) \ pfs_os_aio_func(type, mode, name, file, buf, offset, \ n, message1, message2, write_size, \ - page_compression, page_compression_level, \ - page_encryption, page_encryption_key, \ - lsn, encrypt, __FILE__, __LINE__) + __FILE__, __LINE__) -# define os_file_read(file, buf, offset, n, compressed) \ - pfs_os_file_read_func(file, buf, offset, n, compressed, __FILE__, __LINE__) +# define os_file_read(file, buf, offset, n) \ + pfs_os_file_read_func(file, buf, offset, n, __FILE__, __LINE__) -# define os_file_read_no_error_handling(file, buf, offset, n, compressed) \ +# define os_file_read_no_error_handling(file, buf, offset, n) \ pfs_os_file_read_no_error_handling_func(file, buf, offset, n, \ - compressed, __FILE__, __LINE__) + __FILE__, __LINE__) -# define os_file_write(name, file, buf, offset, n) \ - pfs_os_file_write_func(name, file, buf, offset, \ +# define os_file_write(name, file, buf, offset, n) \ + pfs_os_file_write_func(name, file, buf, offset, \ n, __FILE__, __LINE__) # define os_file_flush(file) \ @@ -362,18 +358,15 @@ to original un-instrumented file I/O APIs */ # define os_file_close(file) os_file_close_func(file) # define os_aio(type, mode, name, file, buf, offset, n, message1, \ - message2, write_size, page_compression, page_compression_level, \ - page_encryption, page_encryption_key, lsn, encrypt) \ + message2, write_size) \ os_aio_func(type, mode, name, file, buf, offset, n, \ - message1, message2, write_size, \ - page_compression, page_compression_level, \ - page_encryption, page_encryption_key, lsn, encrypt) + message1, message2, write_size) -# define os_file_read(file, buf, offset, n, compressed) \ - os_file_read_func(file, buf, offset, n, compressed) +# define os_file_read(file, buf, offset, n) \ + os_file_read_func(file, buf, offset, n) -# define os_file_read_no_error_handling(file, buf, offset, n, compressed) \ - os_file_read_no_error_handling_func(file, buf, offset, n, compressed) +# define os_file_read_no_error_handling(file, buf, offset, n) \ + os_file_read_no_error_handling_func(file, buf, offset, n) # define os_file_write(name, file, buf, offset, n) \ os_file_write_func(name, file, buf, offset, n) @@ -724,8 +717,6 @@ pfs_os_file_read_func( void* buf, /*!< in: buffer where to read */ os_offset_t offset, /*!< in: file offset where to read */ ulint n, /*!< in: number of bytes to read */ - ibool compressed, /*!< in: is this file space - compressed ? */ const char* src_file,/*!< in: file name where func invoked */ ulint src_line);/*!< in: line where the func invoked */ @@ -744,8 +735,6 @@ pfs_os_file_read_no_error_handling_func( void* buf, /*!< in: buffer where to read */ os_offset_t offset, /*!< in: file offset where to read */ ulint n, /*!< in: number of bytes to read */ - ibool compressed, /*!< in: is this file space - compressed ? */ const char* src_file,/*!< in: file name where func invoked */ ulint src_line);/*!< in: line where the func invoked */ @@ -781,16 +770,6 @@ pfs_os_aio_func( operation for this page and if initialized we do not trim again if actual page size does not decrease. */ - ibool page_compression, /*!< in: is page compression used - on this file space */ - ulint page_compression_level, /*!< page compression - level to be used */ - ibool page_encryption, /*!< in: is page encryption used - on this file space */ - ulint page_encryption_key, /*!< in: page encryption - key to be used */ - lsn_t lsn, /*!< in: lsn of the newest modification */ - bool encrypt_later, /*!< in: should we encrypt ? */ const char* src_file,/*!< in: file name where func invoked */ ulint src_line);/*!< in: line where the func invoked */ /*******************************************************************//** @@ -947,9 +926,7 @@ os_file_read_func( os_file_t file, /*!< in: handle to a file */ void* buf, /*!< in: buffer where to read */ os_offset_t offset, /*!< in: file offset where to read */ - ulint n, /*!< in: number of bytes to read */ - ibool compressed); /*!< in: is this file space - compressed ? */ + ulint n); /*!< in: number of bytes to read */ /*******************************************************************//** Rewind file to its start, read at most size - 1 bytes from it to str, and NUL-terminate str. All errors are silently ignored. This function is @@ -974,9 +951,7 @@ os_file_read_no_error_handling_func( os_file_t file, /*!< in: handle to a file */ void* buf, /*!< in: buffer where to read */ os_offset_t offset, /*!< in: file offset where to read */ - ulint n, /*!< in: number of bytes to read */ - ibool compressed); /*!< in: is this file space - compressed ? */ + ulint n); /*!< in: number of bytes to read */ /*******************************************************************//** NOTE! Use the corresponding macro os_file_write(), not directly this @@ -1160,21 +1135,11 @@ os_aio_func( (can be used to identify a completed aio operation); ignored if mode is OS_AIO_SYNC */ - ulint* write_size,/*!< in/out: Actual write size initialized + ulint* write_size);/*!< in/out: Actual write size initialized after fist successfull trim operation for this page and if initialized we do not trim again if actual page size does not decrease. */ - ibool page_compression, /*!< in: is page compression used - on this file space */ - ulint page_compression_level, /*!< page compression - level to be used */ - ibool page_encryption, /*!< in: is page encryption used - on this file space */ - ulint page_encryption_key, /*!< in: page encryption key - to be used */ - lsn_t lsn, /*!< in: lsn of the newest modification */ - bool encrypt_later); /*!< in: should we encrypt ? */ /************************************************************************//** Wakes up all async i/o threads so that they know to exit themselves in shutdown. */ diff --git a/storage/innobase/include/os0file.ic b/storage/innobase/include/os0file.ic index 9839a841188..db525bcea19 100644 --- a/storage/innobase/include/os0file.ic +++ b/storage/innobase/include/os0file.ic @@ -220,17 +220,6 @@ pfs_os_aio_func( operation for this page and if initialized we do not trim again if actual page size does not decrease. */ - ibool page_compression, /*!< in: is page compression used - on this file space */ - ulint page_compression_level, /*!< in: page compression - level to be used */ - ibool page_encryption, /*!< in: is page encryption used - on this file space */ - ulint page_encryption_key, /*!< in: page encryption - key to be used */ - lsn_t lsn, /*!< in: lsn of the newest - modification */ - bool encrypt_later, /*!< in: encrypt later ? */ const char* src_file,/*!< in: file name where func invoked */ ulint src_line)/*!< in: line where the func invoked */ { @@ -246,9 +235,7 @@ pfs_os_aio_func( src_file, src_line); result = os_aio_func(type, mode, name, file, buf, offset, - n, message1, message2, write_size, - page_compression, page_compression_level, - page_encryption, page_encryption_key, lsn, encrypt_later); + n, message1, message2, write_size); register_pfs_file_io_end(locker, n); @@ -269,8 +256,6 @@ pfs_os_file_read_func( void* buf, /*!< in: buffer where to read */ os_offset_t offset, /*!< in: file offset where to read */ ulint n, /*!< in: number of bytes to read */ - ibool compressed, /*!< in: is this file space - compressed ? */ const char* src_file,/*!< in: file name where func invoked */ ulint src_line)/*!< in: line where the func invoked */ { @@ -281,7 +266,7 @@ pfs_os_file_read_func( register_pfs_file_io_begin(&state, locker, file, n, PSI_FILE_READ, src_file, src_line); - result = os_file_read_func(file, buf, offset, n, compressed); + result = os_file_read_func(file, buf, offset, n); register_pfs_file_io_end(locker, n); @@ -304,8 +289,6 @@ pfs_os_file_read_no_error_handling_func( void* buf, /*!< in: buffer where to read */ os_offset_t offset, /*!< in: file offset where to read */ ulint n, /*!< in: number of bytes to read */ - ibool compressed, /*!< in: is this file space - compressed ? */ const char* src_file,/*!< in: file name where func invoked */ ulint src_line)/*!< in: line where the func invoked */ { @@ -316,7 +299,7 @@ pfs_os_file_read_no_error_handling_func( register_pfs_file_io_begin(&state, locker, file, n, PSI_FILE_READ, src_file, src_line); - result = os_file_read_no_error_handling_func(file, buf, offset, n, compressed); + result = os_file_read_no_error_handling_func(file, buf, offset, n); register_pfs_file_io_end(locker, n); diff --git a/storage/innobase/log/log0log.cc b/storage/innobase/log/log0log.cc index 12c05e16b6b..1c08c9fbfb4 100644 --- a/storage/innobase/log/log0log.cc +++ b/storage/innobase/log/log0log.cc @@ -1276,7 +1276,7 @@ log_group_file_header_flush( (ulint) (dest_offset / UNIV_PAGE_SIZE), (ulint) (dest_offset % UNIV_PAGE_SIZE), OS_FILE_LOG_BLOCK_SIZE, - buf, group, 0, 0, false); + buf, group, 0); srv_stats.os_log_pending_writes.dec(); } @@ -1443,7 +1443,7 @@ loop: fil_io(OS_FILE_WRITE | OS_FILE_LOG, true, group->space_id, 0, (ulint) (next_offset / UNIV_PAGE_SIZE), (ulint) (next_offset % UNIV_PAGE_SIZE), write_len, buf, - group, 0, 0, false); + group, 0); srv_stats.os_log_pending_writes.dec(); @@ -2011,7 +2011,7 @@ log_group_checkpoint( write_offset / UNIV_PAGE_SIZE, write_offset % UNIV_PAGE_SIZE, OS_FILE_LOG_BLOCK_SIZE, - buf, ((byte*) group + 1), 0, 0, false); + buf, ((byte*) group + 1), 0); ut_ad(((ulint) group & 0x1UL) == 0); } @@ -2093,7 +2093,7 @@ log_group_read_checkpoint_info( fil_io(OS_FILE_READ | OS_FILE_LOG, true, group->space_id, 0, field / UNIV_PAGE_SIZE, field % UNIV_PAGE_SIZE, - OS_FILE_LOG_BLOCK_SIZE, log_sys->checkpoint_buf, NULL, 0, 0, false); + OS_FILE_LOG_BLOCK_SIZE, log_sys->checkpoint_buf, NULL, 0); } /******************************************************//** @@ -2417,7 +2417,7 @@ loop: fil_io(OS_FILE_READ | OS_FILE_LOG, sync, group->space_id, 0, (ulint) (source_offset / UNIV_PAGE_SIZE), (ulint) (source_offset % UNIV_PAGE_SIZE), - len, buf, NULL, 0, 0, false); + len, buf, NULL, 0); if (recv_sys->recv_log_crypt_ver != UNENCRYPTED_KEY_VER && !log_group_decrypt_after_read(group, buf, len)) diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index d0f2cd360f9..3b8c4878e32 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -49,6 +49,7 @@ Created 9/20/1997 Heikki Tuuri #include "trx0undo.h" #include "trx0rec.h" #include "fil0fil.h" +#include "fil0crypt.h" #ifndef UNIV_HOTBACKUP # include "buf0rea.h" # include "srv0srv.h" @@ -3101,7 +3102,7 @@ recv_recovery_from_checkpoint_start_func( fil_io(OS_FILE_READ | OS_FILE_LOG, true, max_cp_group->space_id, 0, 0, 0, LOG_FILE_HDR_SIZE, - log_hdr_buf, max_cp_group, 0, 0, false); + log_hdr_buf, max_cp_group, 0); if (0 == ut_memcmp(log_hdr_buf + LOG_FILE_WAS_CREATED_BY_HOT_BACKUP, (byte*)"ibbackup", (sizeof "ibbackup") - 1)) { @@ -3132,7 +3133,7 @@ recv_recovery_from_checkpoint_start_func( fil_io(OS_FILE_WRITE | OS_FILE_LOG, true, max_cp_group->space_id, 0, 0, 0, OS_FILE_LOG_BLOCK_SIZE, - log_hdr_buf, max_cp_group, 0, 0, false); + log_hdr_buf, max_cp_group, 0); } #ifdef UNIV_LOG_ARCHIVE diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index d49f5f9900d..1287ee76819 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -43,9 +43,9 @@ Created 10/21/1995 Heikki Tuuri #include "srv0srv.h" #include "srv0start.h" #include "fil0fil.h" +#include "fil0crypt.h" #include "fsp0fsp.h" #include "fil0pagecompress.h" -#include "fil0pageencryption.h" #include "buf0buf.h" #include "srv0mon.h" #include "srv0srv.h" @@ -224,49 +224,14 @@ struct os_aio_slot_t{ completed */ ulint bitmap; - byte* page_compression_page; /*!< Memory allocated for - page compressed page and - freed after the write - has been completed */ - - byte* page_encryption_page; /*!< Memory allocated for - page encrypted page and - freed after the write - has been completed */ - - ibool page_compression; - ulint page_compression_level; - - ibool page_encryption; - ulint page_encryption_key; - ulint* write_size; /*!< Actual write size initialized after fist successfull trim operation for this page and if initialized we do not trim again if actual page size does not decrease. */ - byte* page_buf; /*!< Actual page buffer for - page compressed pages, do not - free this */ - - byte* page_buf2; /*!< Actual page buffer for - page encrypted pages, do not - free this */ - byte* tmp_encryption_buf; /*!< a temporal buffer used by page encryption */ - - ibool page_compression_success; - /*!< TRUE if page compression was successfull, false if not */ - ibool page_encryption_success; - /*!< TRUE if page encryption was successfull, false if not */ - - lsn_t lsn; /* lsn of the newest modification */ - ulint file_block_size;/*!< file block size */ - bool encrypt_later; /*!< should the page be encrypted - before write */ - #ifdef WIN_ASYNC_IO HANDLE handle; /*!< handle object we need in the OVERLAPPED struct */ @@ -403,39 +368,6 @@ os_file_trim( /*=========*/ os_aio_slot_t* slot); /*!< in: slot structure */ -/**********************************************************************//** -Allocate memory for temporal buffer used for page compression. This -buffer is freed later. */ -UNIV_INTERN -void -os_slot_alloc_page_buf( -/*===================*/ - os_aio_slot_t* slot); /*!< in: slot structure */ - -#ifdef HAVE_LZO -/**********************************************************************//** -Allocate memory for temporal memory used for page compression when -LZO compression method is used */ -UNIV_INTERN -void -os_slot_alloc_lzo_mem( -/*===================*/ - os_aio_slot_t* slot); /*!< in: slot structure */ -#endif - -/**********************************************************************//** -Allocate memory for temporal buffer used for page encryption. This -buffer is freed later. */ -UNIV_INTERN -void -os_slot_alloc_page_buf2( - os_aio_slot_t* slot); /*!< in: slot structure */ -/**********************************************************************//** -Allocate memory for temporal buffer used for page encryption. */ -UNIV_INTERN -void -os_slot_alloc_tmp_encryption_buf( - os_aio_slot_t* slot); /*!< in: slot structure */ /****************************************************************//** Does error handling when a file operation fails. @return TRUE if we should retry the operation */ @@ -2891,9 +2823,7 @@ os_file_read_func( os_file_t file, /*!< in: handle to a file */ void* buf, /*!< in: buffer where to read */ os_offset_t offset, /*!< in: file offset where to read */ - ulint n, /*!< in: number of bytes to read */ - ibool compressed) /*!< in: is this file space - compressed ? */ + ulint n) /*!< in: number of bytes to read */ { #ifdef __WIN__ BOOL ret; @@ -3024,9 +2954,7 @@ os_file_read_no_error_handling_func( os_file_t file, /*!< in: handle to a file */ void* buf, /*!< in: buffer where to read */ os_offset_t offset, /*!< in: file offset where to read */ - ulint n, /*!< in: number of bytes to read */ - ibool compressed) /*!< in: is this file space - compressed ? */ + ulint n) /*!< in: number of bytes to read */ { #ifdef __WIN__ BOOL ret; @@ -4184,9 +4112,8 @@ os_aio_array_free( /*==============*/ os_aio_array_t*& array) /*!< in, own: array to free */ { - ulint i; #ifdef WIN_ASYNC_IO - + ulint i; for (i = 0; i < array->n_slots; i++) { os_aio_slot_t* slot = os_aio_array_get_nth_slot(array, i); CloseHandle(slot->handle); @@ -4207,31 +4134,6 @@ os_aio_array_free( } #endif /* LINUX_NATIVE_AIO */ - for (i = 0; i < array->n_slots; i++) { - os_aio_slot_t* slot = os_aio_array_get_nth_slot(array, i); - - if (slot->page_compression_page) { - ut_free(slot->page_compression_page); - slot->page_compression_page = NULL; - } - - if (slot->lzo_mem) { - ut_free(slot->lzo_mem); - slot->lzo_mem = NULL; - } - - if (slot->page_encryption_page) { - ut_free(slot->page_encryption_page); - slot->page_encryption_page = NULL; - } - - if (slot->tmp_encryption_buf) { - ut_free(slot->tmp_encryption_buf); - slot->tmp_encryption_buf = NULL; - } - } - - ut_free(array->slots); ut_free(array); @@ -4566,22 +4468,11 @@ os_aio_array_reserve_slot( to write */ os_offset_t offset, /*!< in: file offset */ ulint len, /*!< in: length of the block to read or write */ - ulint* write_size,/*!< in/out: Actual write size initialized + ulint* write_size)/*!< in/out: Actual write size initialized after fist successfull trim operation for this page and if initialized we do not trim again if actual page size does not decrease. */ - ibool page_compression, /*!< in: is page compression used - on this file space */ - ulint page_compression_level, /*!< page compression - level to be used */ - ibool page_encryption, /*!< in: is page encryption used - on this file space */ - ulint page_encryption_key, /*!< in: page encryption key - to be used */ - lsn_t lsn, /*!< in: lsn of the newest modification */ - bool encrypt_later) /*!< in: should we encrypt before - writing the page */ { os_aio_slot_t* slot = NULL; #ifdef WIN_ASYNC_IO @@ -4669,95 +4560,13 @@ found: slot->len = len; slot->type = type; slot->offset = offset; - slot->lsn = lsn; slot->io_already_done = FALSE; - slot->page_compression_success = FALSE; - slot->page_encryption_success = FALSE; slot->write_size = write_size; - slot->page_compression_level = page_compression_level; - slot->page_compression = page_compression; - slot->page_encryption_key = page_encryption_key; - slot->page_encryption = page_encryption; - slot->encrypt_later = encrypt_later; if (message1) { slot->file_block_size = fil_node_get_block_size(message1); } - /* If the space is page compressed and this is write operation - then we compress the page */ - if (message1 && type == OS_FILE_WRITE && page_compression ) { - ulint real_len = len; - byte* tmp = NULL; - - /* Release the array mutex while compressing */ - os_mutex_exit(array->mutex); - - // We allocate memory for page compressed buffer if and only - // if it is not yet allocated. - os_slot_alloc_page_buf(slot); - -#ifdef HAVE_LZO - if (innodb_compression_algorithm == 3) { - os_slot_alloc_lzo_mem(slot); - } -#endif - - /* Call page compression */ - tmp = fil_compress_page( - fil_node_get_space_id(slot->message1), - (byte *)buf, - slot->page_buf, - len, - page_compression_level, - fil_node_get_block_size(slot->message1), - &real_len, - slot->lzo_mem - ); - - /* If compression succeeded, set up the length and buffer */ - if (tmp != buf) { - len = real_len; - buf = slot->page_buf; - slot->len = real_len; - slot->page_compression_success = TRUE; - } else { - slot->page_compression_success = FALSE; - } - - /* Take array mutex back, not sure if this is really needed - below */ - os_mutex_enter(array->mutex); - - } - - /* If the space is page encryption and this is write operation - then we encrypt the page */ - if (message1 && type == OS_FILE_WRITE && (page_encryption == 1 || encrypt_later)) { - ut_a(page_encryption == 1 || srv_encrypt_tables == 1); - /* Release the array mutex while encrypting */ - os_mutex_exit(array->mutex); - - // We allocate memory for page encrypted buffer if and only - // if it is not yet allocated. - os_slot_alloc_page_buf2(slot); - - fil_space_encrypt( - fil_node_get_space_id(slot->message1), - slot->offset, - slot->lsn, - (byte *)buf, - slot->len, - slot->page_buf2, - slot->page_encryption_key); - - slot->page_encryption_success = TRUE; - buf = slot->page_buf2; - - /* Take array mutex back */ - os_mutex_enter(array->mutex); - } - slot->buf = static_cast(buf); #ifdef WIN_ASYNC_IO @@ -5037,22 +4846,11 @@ os_aio_func( (can be used to identify a completed aio operation); ignored if mode is OS_AIO_SYNC */ - ulint* write_size,/*!< in/out: Actual write size initialized + ulint* write_size)/*!< in/out: Actual write size initialized after fist successfull trim operation for this page and if initialized we do not trim again if actual page size does not decrease. */ - ibool page_compression, /*!< in: is page compression used - on this file space */ - ulint page_compression_level, /*!< page compression - level to be used */ - ibool page_encryption, /*!< in: is page encryption used - on this file space */ - ulint page_encryption_key, /*!< in: page encryption key - to be used */ - lsn_t lsn, /*!< in: lsn of the newest modification */ - bool encrypt_later) /*!< in: should we encrypt page - before write */ { os_aio_array_t* array; os_aio_slot_t* slot; @@ -5104,8 +4902,7 @@ os_aio_func( and os_file_write_func() */ if (type == OS_FILE_READ) { - ret = os_file_read_func(file, buf, offset, n, - page_compression); + ret = os_file_read_func(file, buf, offset, n); } else { ut_ad(!srv_read_only_mode); @@ -5163,9 +4960,7 @@ try_again: } slot = os_aio_array_reserve_slot(type, array, message1, message2, file, - name, buf, offset, n, write_size, - page_compression, page_compression_level, - page_encryption, page_encryption_key, lsn, encrypt_later); + name, buf, offset, n, write_size); if (type == OS_FILE_READ) { if (srv_use_native_aio) { @@ -5192,18 +4987,9 @@ try_again: if (srv_use_native_aio) { os_n_file_writes++; #ifdef WIN_ASYNC_IO - if (page_encryption && slot->page_encryption_success) { - buffer = slot->page_buf2; - n = slot->len; - } else { - if (page_compression && slot->page_compression_success) { - buffer = slot->page_buf; - n = slot->len; - } else { - buffer = buf; - } - } + n = slot->len; + buffer = buf; ret = WriteFile(file, buffer, (DWORD) n, &len, &(slot->control)); @@ -5409,27 +5195,9 @@ os_aio_windows_handle( switch (slot->type) { case OS_FILE_WRITE: - if (slot->message1 - && slot->page_encryption - && slot->page_encryption_success) { - ret_val = os_file_write(slot->name, - slot->file, - slot->page_buf2, - slot->offset, - slot->len); - } else { - if (slot->message1 - && slot->page_compression - && slot->page_compression_success) { - ret = WriteFile(slot->file, slot->page_buf, - (DWORD) slot->len, &len, - &(slot->control)); - } else { - ret = WriteFile(slot->file, slot->buf, - (DWORD) slot->len, &len, - &(slot->control)); - } - } + ret = WriteFile(slot->file, slot->buf, + (DWORD) slot->len, &len, + &(slot->control)); break; case OS_FILE_READ: ret = ReadFile(slot->file, slot->buf, @@ -5460,47 +5228,9 @@ os_aio_windows_handle( ret_val = ret && len == slot->len; } - if (slot->type == OS_FILE_READ) { - if (fil_page_is_compressed_encrypted(slot->buf) || - fil_page_is_encrypted(slot->buf)) { - ut_ad(slot->message1 != NULL); - os_slot_alloc_page_buf2(slot); - os_slot_alloc_tmp_encryption_buf(slot); - - // Decrypt the data - fil_space_decrypt( - fil_node_get_space_id(slot->message1), - slot->buf, - slot->len, - slot->page_buf2); - // Copy decrypted buffer back to buf - memcpy(slot->buf, slot->page_buf2, slot->len); - } - if (fil_page_is_compressed(slot->buf)) { - /* We allocate memory for page compressed buffer if - and only if it is not yet allocated. */ - os_slot_alloc_page_buf(slot); -#ifdef HAVE_LZO - if (fil_page_is_lzo_compressed(slot->buf)) { - os_slot_alloc_lzo_mem(slot); - } -#endif - fil_decompress_page( - slot->page_buf, - slot->buf, - slot->len, - slot->write_size); - } - } else { - /* OS_FILE_WRITE */ - if (slot->page_compression_success && - (fil_page_is_compressed(slot->page_buf) || - fil_page_is_compressed_encrypted(slot->buf))) { - if (srv_use_trim && os_fallocate_failed == FALSE) { - // Deallocate unused blocks from file system - os_file_trim(slot); - } - } + if (slot->type == OS_FILE_WRITE && srv_use_trim && os_fallocate_failed == FALSE) { + // Deallocate unused blocks from file system + os_file_trim(slot); } os_aio_array_free_slot(array, slot); @@ -5592,50 +5322,9 @@ retry: /* We have not overstepped to next segment. */ ut_a(slot->pos < end_pos); - if (slot->type == OS_FILE_READ) { - /* If the page is page encrypted we encrypt */ - if (fil_page_is_compressed_encrypted(slot->buf) || - fil_page_is_encrypted(slot->buf)) { - os_slot_alloc_page_buf2(slot); - os_slot_alloc_tmp_encryption_buf(slot); - ut_ad(slot->message1 != NULL); - - // Decrypt the data - fil_space_decrypt( - fil_node_get_space_id(slot->message1), - slot->buf, - slot->len, - slot->page_buf2); - // Copy decrypted buffer back to buf - memcpy(slot->buf, slot->page_buf2, slot->len); - } - - /* If the table is page compressed and this - is read, we decompress before we announce - the read is complete. For writes, we free - the compressed page. */ - if (fil_page_is_compressed(slot->buf)) { - // We allocate memory for page compressed buffer if and only - // if it is not yet allocated. - os_slot_alloc_page_buf(slot); -#ifdef HAVE_LZO - if (fil_page_is_lzo_compressed(slot->buf)) { - os_slot_alloc_lzo_mem(slot); - } -#endif - - fil_decompress_page(slot->page_buf, slot->buf, slot->len, slot->write_size); - } - } else { - /* OS_FILE_WRITE */ - if (slot->page_compression_success && - (fil_page_is_compressed(slot->page_buf) || - fil_page_is_compressed_encrypted(slot->buf))) { - if (srv_use_trim && os_fallocate_failed == FALSE) { - // Deallocate unused blocks from file system - os_file_trim(slot); - } - } + if (slot->type == OS_FILE_WRITE && srv_use_trim && os_fallocate_failed == FALSE) { + // Deallocate unused blocks from file system + os_file_trim(slot); } /* Mark this request as completed. The error handling @@ -6080,8 +5769,7 @@ consecutive_loop: } else { ret = os_file_read( aio_slot->file, combined_buf, - aio_slot->offset, total_len, - aio_slot->page_compression); + aio_slot->offset, total_len); } srv_set_io_thread_op_info(global_segment, "file i/o done"); @@ -6653,92 +6341,6 @@ os_file_trim( } #endif /* !UNIV_HOTBACKUP */ -/**********************************************************************//** -Allocate memory for temporal buffer used for page encryption. This -buffer is freed later. */ -UNIV_INTERN -void -os_slot_alloc_page_buf2( -/*===================*/ - os_aio_slot_t* slot) /*!< in: slot structure */ -{ - ut_a(slot != NULL); - - if(slot->page_buf2 == NULL) { - byte* cbuf2; - byte* cbuf; - - cbuf2 = static_cast(ut_malloc(UNIV_PAGE_SIZE*2)); - cbuf = static_cast(ut_align(cbuf2, UNIV_PAGE_SIZE)); - slot->page_encryption_page = static_cast(cbuf2); - slot->page_buf2 = static_cast(cbuf); - memset(slot->page_encryption_page, 0, UNIV_PAGE_SIZE*2); - } -} - -/**********************************************************************//** -Allocate memory for temporal buffer used for page compression. This -buffer is freed later. */ -UNIV_INTERN -void -os_slot_alloc_page_buf( -/*===================*/ - os_aio_slot_t* slot) /*!< in: slot structure */ -{ - ut_a(slot != NULL); - if (slot->page_buf == NULL) { - byte* cbuf2; - byte* cbuf; - ulint asize = UNIV_PAGE_SIZE; -#ifdef HAVE_SNAPPY - asize += snappy_max_compressed_length(asize) - UNIV_PAGE_SIZE; -#endif - - /* We allocate extra to avoid memory overwrite on compression */ - cbuf2 = static_cast(ut_malloc(asize*2)); - cbuf = static_cast(ut_align(cbuf2, UNIV_PAGE_SIZE)); - slot->page_compression_page = static_cast(cbuf2); - slot->page_buf = static_cast(cbuf); - ut_a(slot->page_buf != NULL); - memset(slot->page_compression_page, 0, asize*2); - } -} - -#ifdef HAVE_LZO -/**********************************************************************//** -Allocate memory for temporal memory used for page compression when -LZO compression method is used */ -UNIV_INTERN -void -os_slot_alloc_lzo_mem( -/*===================*/ - os_aio_slot_t* slot) /*!< in: slot structure */ -{ - ut_a(slot != NULL); - if(slot->lzo_mem == NULL) { - slot->lzo_mem = static_cast(ut_malloc(LZO1X_1_15_MEM_COMPRESS)); - ut_a(slot->lzo_mem != NULL); - memset(slot->lzo_mem, 0, LZO1X_1_15_MEM_COMPRESS); - } -} -#endif - -/**********************************************************************//** -Allocate memory for temporal buffer used for page encryption. */ -UNIV_INTERN -void -os_slot_alloc_tmp_encryption_buf( -/*=============================*/ - os_aio_slot_t* slot) /*!< in: slot structure */ -{ - ut_a(slot != NULL); - if (slot->tmp_encryption_buf == NULL) { - slot->tmp_encryption_buf = static_cast(ut_malloc(64)); - memset(slot->tmp_encryption_buf, 0, 64); - } -} - - /***********************************************************************//** Try to get number of bytes per sector from file system. @return file block size */ diff --git a/storage/innobase/row/row0log.cc b/storage/innobase/row/row0log.cc index caed087b439..c63dbffcba9 100644 --- a/storage/innobase/row/row0log.cc +++ b/storage/innobase/row/row0log.cc @@ -2554,7 +2554,7 @@ all_done: success = os_file_read_no_error_handling( OS_FILE_FROM_FD(index->online_log->fd), index->online_log->head.block, ofs, - srv_sort_buf_size, FALSE); + srv_sort_buf_size); if (!success) { fprintf(stderr, "InnoDB: unable to read temporary file" @@ -3385,7 +3385,7 @@ all_done: success = os_file_read_no_error_handling( OS_FILE_FROM_FD(index->online_log->fd), index->online_log->head.block, ofs, - srv_sort_buf_size, FALSE); + srv_sort_buf_size); if (!success) { fprintf(stderr, "InnoDB: unable to read temporary file" diff --git a/storage/innobase/row/row0merge.cc b/storage/innobase/row/row0merge.cc index 97746aa088e..d2734ecd6b5 100644 --- a/storage/innobase/row/row0merge.cc +++ b/storage/innobase/row/row0merge.cc @@ -786,7 +786,7 @@ row_merge_read( #endif /* UNIV_DEBUG */ success = os_file_read_no_error_handling(OS_FILE_FROM_FD(fd), buf, - ofs, srv_sort_buf_size, FALSE); + ofs, srv_sort_buf_size); #ifdef POSIX_FADV_DONTNEED /* Each block is read exactly once. Free up the file cache. */ diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index 3dd65c540c3..d9fef775d40 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -57,6 +57,7 @@ Created 9/17/2000 Heikki Tuuri #include "btr0sea.h" #include "btr0defragment.h" #include "fil0fil.h" +#include "fil0crypt.h" #include "ibuf0ibuf.h" #include "fts0fts.h" #include "fts0types.h" diff --git a/storage/innobase/srv/srv0srv.cc b/storage/innobase/srv/srv0srv.cc index ee88c1f9e33..78954725001 100644 --- a/storage/innobase/srv/srv0srv.cc +++ b/storage/innobase/srv/srv0srv.cc @@ -74,9 +74,9 @@ Created 10/8/1995 Heikki Tuuri #include "mysql/plugin.h" #include "mysql/service_thd_wait.h" #include "fil0fil.h" +#include "fil0crypt.h" #include "fil0pagecompress.h" #include "btr0scrub.h" -#include "fil0pageencryption.h" #ifdef WITH_WSREP extern int wsrep_debug; @@ -524,7 +524,7 @@ second. */ static time_t srv_last_log_flush_time; /** Default encryption key used for page encryption */ -UNIV_INTERN uint srv_default_page_encryption_key = DEFAULT_ENCRYPTION_KEY; +UNIV_INTERN uint srv_default_page_encryption_key = FIL_DEFAULT_ENCRYPTION_KEY; /** Enable semaphore request instrumentation */ UNIV_INTERN my_bool srv_instrument_semaphores = FALSE; diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index 3822a9abf2d..3cc66778de9 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -3,7 +3,7 @@ Copyright (c) 1996, 2014, Oracle and/or its affiliates. All rights reserved. Copyright (c) 2008, Google Inc. Copyright (c) 2009, Percona Inc. -Copyright (c) 2013, 2014, SkySQL Ab. All Rights Reserved. +Copyright (c) 2013, 2015, MariaDB Corporation Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -53,6 +53,7 @@ Created 2/16/1996 Heikki Tuuri #include "os0file.h" #include "os0thread.h" #include "fil0fil.h" +#include "fil0crypt.h" #include "fsp0fsp.h" #include "rem0rec.h" #include "mtr0mtr.h" diff --git a/storage/xtradb/buf/buf0buf.cc b/storage/xtradb/buf/buf0buf.cc index 0ef97fd3e21..56b0245da93 100644 --- a/storage/xtradb/buf/buf0buf.cc +++ b/storage/xtradb/buf/buf0buf.cc @@ -40,6 +40,7 @@ Created 11/5/1995 Heikki Tuuri #include "mem0mem.h" #include "btr0btr.h" #include "fil0fil.h" +#include "fil0crypt.h" #ifndef UNIV_HOTBACKUP #include "buf0buddy.h" #include "lock0lock.h" @@ -57,8 +58,6 @@ Created 11/5/1995 Heikki Tuuri #include "trx0trx.h" #include "srv0start.h" #include "ut0byte.h" - -#include "fil0pageencryption.h" #include "fil0pagecompress.h" @@ -574,7 +573,7 @@ buf_page_is_corrupted( ulint zip_size) /*!< in: size of compressed page; 0 for uncompressed pages */ { - ulint page_encrypted = fil_page_is_compressed_encrypted(read_buf) || fil_page_is_encrypted(read_buf); + ulint page_encrypted = fil_page_is_encrypted(read_buf); ulint checksum_field1; ulint checksum_field2; ibool crc32_inited = FALSE; @@ -5938,7 +5937,7 @@ buf_page_decrypt_after_read( unsigned key_version = mach_read_from_4(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); - bool page_compressed_encrypted = fil_page_is_compressed_encrypted(dst_frame); + bool page_compressed = fil_page_is_compressed(dst_frame); if (key_version == 0) { /* the page we read is unencrypted */ @@ -5974,7 +5973,7 @@ buf_page_decrypt_after_read( /* decompress from dst_frame to comp_buf and then copy to buffer pool */ - if (page_compressed_encrypted) { + if (page_compressed) { if (bpage->comp_buf_free == NULL) { bpage->comp_buf_free = (byte *)malloc(UNIV_PAGE_SIZE*2); // TODO: is 4k aligment enough ? diff --git a/storage/xtradb/buf/buf0dblwr.cc b/storage/xtradb/buf/buf0dblwr.cc index 871f723549b..ec733dd5a73 100644 --- a/storage/xtradb/buf/buf0dblwr.cc +++ b/storage/xtradb/buf/buf0dblwr.cc @@ -36,6 +36,8 @@ Created 2011/12/19 #include "srv0srv.h" #include "page0zip.h" #include "trx0sys.h" +#include "fil0fil.h" +#include "fil0crypt.h" #ifndef UNIV_HOTBACKUP @@ -385,7 +387,7 @@ buf_dblwr_init_or_load_pages( /* Read the trx sys header to check if we are using the doublewrite buffer */ off_t trx_sys_page = TRX_SYS_PAGE_NO * UNIV_PAGE_SIZE; - os_file_read(file, read_buf, trx_sys_page, UNIV_PAGE_SIZE, FALSE); + os_file_read(file, read_buf, trx_sys_page, UNIV_PAGE_SIZE); doublewrite = read_buf + TRX_SYS_DOUBLEWRITE; @@ -430,9 +432,9 @@ buf_dblwr_init_or_load_pages( block_bytes = TRX_SYS_DOUBLEWRITE_BLOCK_SIZE * UNIV_PAGE_SIZE; - os_file_read(file, buf, block1 * UNIV_PAGE_SIZE, block_bytes, FALSE); + os_file_read(file, buf, block1 * UNIV_PAGE_SIZE, block_bytes); os_file_read(file, buf + block_bytes, block2 * UNIV_PAGE_SIZE, - block_bytes, FALSE); + block_bytes); /* Check if any of these pages is half-written in data files, in the intended position */ @@ -530,9 +532,7 @@ buf_dblwr_process() zip_size ? zip_size : UNIV_PAGE_SIZE, read_buf, NULL, - 0, - 0, - false); + 0); if (fil_space_verify_crypt_checksum(read_buf, zip_size)) { /* page is encrypted and checksum is OK */ @@ -593,9 +593,7 @@ buf_dblwr_process() zip_size ? zip_size : UNIV_PAGE_SIZE, page, NULL, - 0, - 0, - false); + 0); ib_logf(IB_LOG_LEVEL_INFO, "Recovered the page from" @@ -619,9 +617,7 @@ buf_dblwr_process() zip_size ? zip_size : UNIV_PAGE_SIZE, page, NULL, - 0, - 0, - false); + 0); } } } @@ -643,9 +639,9 @@ buf_dblwr_process() memset(buf, 0, bytes); fil_io(OS_FILE_WRITE, true, TRX_SYS_SPACE, 0, - buf_dblwr->block1, 0, bytes, buf, NULL, NULL, 0, false); + buf_dblwr->block1, 0, bytes, buf, NULL, NULL); fil_io(OS_FILE_WRITE, true, TRX_SYS_SPACE, 0, - buf_dblwr->block2, 0, bytes, buf, NULL, NULL, 0, false); + buf_dblwr->block2, 0, bytes, buf, NULL, NULL); ut_free(unaligned_buf); } @@ -859,9 +855,7 @@ buf_dblwr_write_block_to_datafile( buf_page_get_zip_size(bpage), frame, (void*) bpage, - 0, - bpage->newest_modification, - bpage->encrypt_later); + 0); return; } @@ -880,9 +874,7 @@ buf_dblwr_write_block_to_datafile( UNIV_PAGE_SIZE, frame, (void*) block, - (ulint *)&bpage->write_size, - bpage->newest_modification, - bpage->encrypt_later); + (ulint *)&bpage->write_size); } /********************************************************************//** @@ -984,9 +976,7 @@ try_again: (void*) write_buf, NULL, - 0, - 0, - false); + 0); if (buf_dblwr->first_free <= TRX_SYS_DOUBLEWRITE_BLOCK_SIZE) { /* No unwritten pages in the second block. */ @@ -1009,9 +999,7 @@ try_again: len, (void*) write_buf, NULL, - 0, - 0, - false); + 0); flush: /* increment the doublewrite flushed pages counter */ @@ -1250,9 +1238,7 @@ retry: UNIV_PAGE_SIZE, (void*) (buf_dblwr->write_buf + UNIV_PAGE_SIZE * i), NULL, - 0, - bpage->newest_modification, - bpage->encrypt_later); + 0); } else { /* It is a regular page. Write it directly to the doublewrite buffer */ @@ -1264,9 +1250,7 @@ retry: 0, UNIV_PAGE_SIZE, frame, NULL, - 0, - bpage->newest_modification, - bpage->encrypt_later); + 0); } /* Now flush the doublewrite buffer data to disk */ diff --git a/storage/xtradb/buf/buf0flu.cc b/storage/xtradb/buf/buf0flu.cc index 4331db08cf5..d023e2ff227 100644 --- a/storage/xtradb/buf/buf0flu.cc +++ b/storage/xtradb/buf/buf0flu.cc @@ -963,9 +963,7 @@ buf_flush_write_block_low( zip_size ? zip_size : UNIV_PAGE_SIZE, frame, bpage, - &bpage->write_size, - bpage->newest_modification, - bpage->encrypt_later); + &bpage->write_size); } else { /* InnoDB uses doublewrite buffer and doublewrite buffer is initialized. User can define do we use atomic writes @@ -984,9 +982,7 @@ buf_flush_write_block_low( zip_size ? zip_size : UNIV_PAGE_SIZE, frame, bpage, - &bpage->write_size, - bpage->newest_modification, - bpage->encrypt_later); + &bpage->write_size); } else if (flush_type == BUF_FLUSH_SINGLE_PAGE) { buf_dblwr_write_single_page(bpage, sync); } else { diff --git a/storage/xtradb/buf/buf0rea.cc b/storage/xtradb/buf/buf0rea.cc index 88741f987ca..164aa3c62c1 100644 --- a/storage/xtradb/buf/buf0rea.cc +++ b/storage/xtradb/buf/buf0rea.cc @@ -232,14 +232,14 @@ not_to_recover: *err = _fil_io(OS_FILE_READ | wake_later | ignore_nonexistent_pages, sync, space, zip_size, offset, 0, zip_size, - frame, bpage, 0, trx, 0, false); + frame, bpage, 0, trx); } else { ut_a(buf_page_get_state(bpage) == BUF_BLOCK_FILE_PAGE); *err = _fil_io(OS_FILE_READ | wake_later | ignore_nonexistent_pages, sync, space, 0, offset, 0, UNIV_PAGE_SIZE, - frame, bpage, &bpage->write_size, trx, 0, false); + frame, bpage, &bpage->write_size, trx); } if (sync) { diff --git a/storage/xtradb/fil/fil0crypt.cc b/storage/xtradb/fil/fil0crypt.cc index 9f8315a8016..aab0b1f5568 100644 --- a/storage/xtradb/fil/fil0crypt.cc +++ b/storage/xtradb/fil/fil0crypt.cc @@ -24,6 +24,7 @@ Modified Jan Lindström jan.lindstrom@mariadb.com *******************************************************/ #include "fil0fil.h" +#include "fil0crypt.h" #include "srv0srv.h" #include "srv0start.h" #include "mach0data.h" @@ -35,14 +36,13 @@ Modified Jan Lindström jan.lindstrom@mariadb.com #include "btr0scrub.h" #include "fsp0fsp.h" #include "fil0pagecompress.h" -#include "fil0pageencryption.h" #include "ha_prototypes.h" // IB_LOG_ #include #include #include - +#include "fil0crypt.h" /** Mutex for keys */ UNIV_INTERN ib_mutex_t fil_crypt_key_mutex; @@ -119,67 +119,40 @@ static const unsigned char CRYPT_MAGIC[MAGIC_SZ] = { static const unsigned char EMPTY_PATTERN[MAGIC_SZ] = { 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 }; -/** - * CRYPT_SCHEME_UNENCRYPTED - * - * Used as intermediate state when convering a space from unencrypted - * to encrypted - */ -#define CRYPT_SCHEME_UNENCRYPTED 0 - -/** - * CRYPT_SCHEME_1 - * - * L = AES_ECB(KEY, IV) - * CRYPT(PAGE) = AES_CRT(KEY=L, IV=C, PAGE) - */ -#define CRYPT_SCHEME_1 1 -#define CRYPT_SCHEME_1_IV_LEN 16 -// cached L given key_version -struct key_struct +/****************************************************************** +Map used AES method to crypt scheme +@return used AES crypt scheme */ +UNIV_INTERN +uint +fil_crypt_get_aes_method( +/*=====================*/ + uint aes_method) { - uint key_version; - byte key[CRYPT_SCHEME_1_IV_LEN]; -}; + switch (aes_method) { + case MY_AES_ALGORITHM_NONE: + return (uint) CRYPT_SCHEME_1_UNENCRYPTED; + break; + case MY_AES_ALGORITHM_CTR: + return (uint) CRYPT_SCHEME_1_CTR; + break; + case MY_AES_ALGORITHM_CBC: + return (uint) CRYPT_SCHEME_1_CBC; + break; + default: + ib_logf(IB_LOG_LEVEL_FATAL, + "Current AES method %d not supported.\n", aes_method); + ut_error; + } -struct fil_space_rotate_state_t -{ - time_t start_time; // time when rotation started - ulint active_threads; // active threads in space - ulint next_offset; // next "free" offset - ulint max_offset; // max offset needing to be rotated - uint min_key_version_found; // min key version found but not rotated - lsn_t end_lsn; // max lsn created when rotating this space - bool starting; // initial write of IV - bool flushing; // space is being flushed at end of rotate - struct { - bool is_active; // is scrubbing active in this space - time_t last_scrub_completed; // when was last scrub completed - } scrubbing; -}; - -struct fil_space_crypt_struct -{ - ulint type; // CRYPT_SCHEME - uint keyserver_requests; // no of key requests to key server - uint key_count; // No of initalized key-structs - key_struct keys[3]; // cached L = AES_ECB(KEY, IV) - uint min_key_version; // min key version for this space - ulint page0_offset; // byte offset on page 0 for crypt data - - ib_mutex_t mutex; // mutex protecting following variables - bool closing; // is tablespace being closed - fil_space_rotate_state_t rotate_state; - - uint iv_length; // length of IV - byte iv[1]; // IV-data -}; + return (uint) CRYPT_SCHEME_1_UNENCRYPTED; +} /********************************************************************* Init space crypt */ UNIV_INTERN void fil_space_crypt_init() +/*==================*/ { mutex_create(fil_crypt_key_mutex_key, &fil_crypt_key_mutex, SYNC_NO_ORDER_CHECK); @@ -196,6 +169,7 @@ Cleanup space crypt */ UNIV_INTERN void fil_space_crypt_cleanup() +/*=====================*/ { os_event_free(fil_crypt_throttle_sleep_event); } @@ -204,31 +178,35 @@ fil_space_crypt_cleanup() Get key bytes for a space/key-version */ static void -fil_crypt_get_key(byte *dst, uint* key_length, - fil_space_crypt_t* crypt_data, uint version, bool page_encrypted) +fil_crypt_get_key( +/*==============*/ + byte* dst, /*mutex); - if (!page_encrypted) { - // Check if we already have key - for (uint i = 0; i < crypt_data->key_count; i++) { - if (crypt_data->keys[i].key_version == version) { - memcpy(dst, crypt_data->keys[i].key, - sizeof(crypt_data->keys[i].key)); - mutex_exit(&crypt_data->mutex); - return; - } + // Check if we already have key + for (uint i = 0; i < crypt_data->key_count; i++) { + if (crypt_data->keys[i].key_version == version) { + memcpy(dst, crypt_data->keys[i].key, + crypt_data->keys[i].key_length); + *key_length = crypt_data->keys[i].key_length; + mutex_exit(&crypt_data->mutex); + return; } - // Not found! - crypt_data->keyserver_requests++; + } - // Rotate keys to make room for a new - for (uint i = 1; i < array_elements(crypt_data->keys); i++) { - crypt_data->keys[i] = crypt_data->keys[i - 1]; - } - } + // Not found! + crypt_data->keyserver_requests++; + + // Rotate keys to make room for a new + for (uint i = 1; i < array_elements(crypt_data->keys); i++) { + crypt_data->keys[i] = crypt_data->keys[i - 1]; + } if (has_encryption_key(version)) { int rc; @@ -247,51 +225,45 @@ fil_crypt_get_key(byte *dst, uint* key_length, ut_error; } + /* Now compute L by encrypting IV using this key. Note + that we use random IV from crypt data. */ + const unsigned char* src = crypt_data->iv; + const int srclen = crypt_data->iv_length; + unsigned char* buf = crypt_data->keys[0].key; + uint32 buflen = CRYPT_SCHEME_1_IV_LEN; - // do ctr key initialization - if (current_aes_dynamic_method == MY_AES_ALGORITHM_CTR) - { - /* Now compute L by encrypting IV using this key. Note - that we use random IV from crypt data. */ - const unsigned char* src = crypt_data->iv; - const int srclen = crypt_data->iv_length; - unsigned char* buf = page_encrypted ? keybuf : crypt_data->keys[0].key; - uint32 buflen = page_encrypted ? *key_length : sizeof(crypt_data->keys[0].key); + /* We use AES_ECB to encryp IV */ + my_aes_encrypt_dynamic_type func = get_aes_encrypt_func(MY_AES_ALGORITHM_ECB); - // call ecb explicit - my_aes_encrypt_dynamic_type func = get_aes_encrypt_func(MY_AES_ALGORITHM_ECB); - int rc = (*func)(src, srclen, - buf, &buflen, - (unsigned char*)keybuf, *key_length, - NULL, 0, - 1); + int rc = (*func)(src, /* Data to be encrypted = IV */ + srclen, /* data length */ + buf, /* Output buffer */ + &buflen, /* Output buffer */ + keybuf, /* Key */ + *key_length, /* Key length */ + NULL, /* AES_ECB does not use IV */ + 0, /* IV-length */ + 1); /* NoPadding */ - if (rc != AES_OK) { - ib_logf(IB_LOG_LEVEL_FATAL, - "Unable to encrypt key-block " - " src: %p srclen: %d buf: %p buflen: %d." - " return-code: %d. Can't continue!\n", - src, srclen, buf, buflen, rc); - ut_error; - } - - if (!page_encrypted) { - crypt_data->keys[0].key_version = version; - crypt_data->key_count++; - - if (crypt_data->key_count > array_elements(crypt_data->keys)) { - crypt_data->key_count = array_elements(crypt_data->keys); - } - } - - // set the key size to the aes block size because this encrypted data is the key - *key_length = MY_AES_BLOCK_SIZE; - memcpy(dst, buf, buflen); - } else { - // otherwise keybuf contains the right key - memcpy(dst, keybuf, *key_length); + if (rc != AES_OK) { + ib_logf(IB_LOG_LEVEL_FATAL, + "Unable to encrypt key-block " + " src: %p srclen: %d buf: %p buflen: %d." + " return-code: %d. Can't continue!\n", + src, srclen, buf, buflen, rc); + ut_error; } + crypt_data->keys[0].key_version = version; + crypt_data->key_count++; + *key_length = buflen; + crypt_data->keys[0].key_length = buflen; + + if (crypt_data->key_count > array_elements(crypt_data->keys)) { + crypt_data->key_count = array_elements(crypt_data->keys); + } + + memcpy(dst, buf, buflen); mutex_exit(&crypt_data->mutex); } @@ -299,8 +271,12 @@ fil_crypt_get_key(byte *dst, uint* key_length, Get key bytes for a space/latest(key-version) */ static inline void -fil_crypt_get_latest_key(byte *dst, uint* key_length, - fil_space_crypt_t* crypt_data, uint *version) +fil_crypt_get_latest_key( +/*=====================*/ + byte* dst, /*!< out: Key */ + uint* key_length, /*!< out: Key length */ + fil_space_crypt_t* crypt_data, /*!< in: crypt data */ + uint* version) /*!< in: Key version */ { if (srv_encrypt_tables) { // used for key rotation - get the next key id from the key provider @@ -312,29 +288,33 @@ fil_crypt_get_latest_key(byte *dst, uint* key_length, } } - return fil_crypt_get_key(dst, key_length, crypt_data, *version, srv_encrypt_tables == FALSE); + return fil_crypt_get_key(dst, key_length, crypt_data, *version); } /****************************************************************** -Create a fil_space_crypt_t object */ +Create a fil_space_crypt_t object +@return crypt object */ UNIV_INTERN fil_space_crypt_t* fil_space_create_crypt_data() +/*=========================*/ { const uint iv_length = CRYPT_SCHEME_1_IV_LEN; const uint sz = sizeof(fil_space_crypt_t) + iv_length; fil_space_crypt_t* crypt_data = static_cast(malloc(sz)); + uint aes_method = current_aes_dynamic_method; + memset(crypt_data, 0, sz); if (srv_encrypt_tables == FALSE) { - crypt_data->type = CRYPT_SCHEME_UNENCRYPTED; crypt_data->min_key_version = 0; } else { - crypt_data->type = CRYPT_SCHEME_1; crypt_data->min_key_version = get_latest_encryption_key_version(); } + /* Set up the current AES method */ + crypt_data->type = (fil_crypt_method_t)fil_crypt_get_aes_method(aes_method); mutex_create(fil_crypt_data_mutex_key, &crypt_data->mutex, SYNC_NO_ORDER_CHECK); crypt_data->iv_length = iv_length; @@ -346,13 +326,18 @@ fil_space_create_crypt_data() Compare two crypt objects */ UNIV_INTERN int -fil_space_crypt_compare(const fil_space_crypt_t* crypt_data1, - const fil_space_crypt_t* crypt_data2) +fil_space_crypt_compare( +/*====================*/ + const fil_space_crypt_t* crypt_data1,/*!< in: Crypt data */ + const fil_space_crypt_t* crypt_data2)/*!< in: Crypt data */ { - ut_a(crypt_data1->type == CRYPT_SCHEME_UNENCRYPTED || - crypt_data1->type == CRYPT_SCHEME_1); - ut_a(crypt_data2->type == CRYPT_SCHEME_UNENCRYPTED || - crypt_data2->type == CRYPT_SCHEME_1); + ut_a(crypt_data1->type == CRYPT_SCHEME_1_UNENCRYPTED || + crypt_data1->type == CRYPT_SCHEME_1_CTR || + crypt_data1->type == CRYPT_SCHEME_1_CBC); + + ut_a(crypt_data2->type == CRYPT_SCHEME_1_UNENCRYPTED || + crypt_data2->type == CRYPT_SCHEME_1_CTR || + crypt_data1->type == CRYPT_SCHEME_1_CBC); ut_a(crypt_data1->iv_length == CRYPT_SCHEME_1_IV_LEN); ut_a(crypt_data2->iv_length == CRYPT_SCHEME_1_IV_LEN); @@ -365,10 +350,15 @@ fil_space_crypt_compare(const fil_space_crypt_t* crypt_data1, } /****************************************************************** -Read crypt data from a page (0) */ +Read crypt data from a page (0) +@return crypt data from page 0. */ UNIV_INTERN fil_space_crypt_t* -fil_space_read_crypt_data(ulint space, const byte* page, ulint offset) +fil_space_read_crypt_data( +/*======================*/ + ulint space, /*!< in: file space id*/ + const byte* page, /*!< in: page 0 */ + ulint offset) /*!< in: offset */ { if (memcmp(page + offset, EMPTY_PATTERN, MAGIC_SZ) == 0) { /* crypt is not stored */ @@ -393,8 +383,10 @@ fil_space_read_crypt_data(ulint space, const byte* page, ulint offset) ulint type = mach_read_from_1(page + offset + MAGIC_SZ + 0); - if (! (type == CRYPT_SCHEME_UNENCRYPTED || - type == CRYPT_SCHEME_1)) { + if (! (type == CRYPT_SCHEME_1_UNENCRYPTED || + type == CRYPT_SCHEME_1_CTR || + type == CRYPT_SCHEME_1_CBC)) { + ib_logf(IB_LOG_LEVEL_ERROR, "Found non sensible crypt scheme: %lu for space %lu " " offset: %lu bytes: " @@ -449,7 +441,9 @@ fil_space_read_crypt_data(ulint space, const byte* page, ulint offset) Free a crypt data object */ UNIV_INTERN void -fil_space_destroy_crypt_data(fil_space_crypt_t **crypt_data) +fil_space_destroy_crypt_data( +/*=========================*/ + fil_space_crypt_t **crypt_data) /*!< out: crypt data */ { if (crypt_data != NULL && (*crypt_data) != NULL) { /* lock (and unlock) mutex to make sure no one has it locked @@ -466,10 +460,14 @@ fil_space_destroy_crypt_data(fil_space_crypt_t **crypt_data) Write crypt data to a page (0) */ static void -fil_space_write_crypt_data_low(fil_space_crypt_t *crypt_data, - ulint type, - byte* page, ulint offset, - ulint maxsize, mtr_t* mtr) +fil_space_write_crypt_data_low( +/*===========================*/ + fil_space_crypt_t* crypt_data, /* 0 && offset < UNIV_PAGE_SIZE); ulint space_id = mach_read_from_4( @@ -492,10 +490,8 @@ fil_space_write_crypt_data_low(fil_space_crypt_t *crypt_data, mlog_write_ulint(page + offset + MAGIC_SZ + 2 + len, min_key_version, MLOG_4BYTES, mtr); - DBUG_EXECUTE_IF("ib_file_crypt_redo_crash_1", - ut_error;); - byte* log_ptr = mlog_open(mtr, 11 + 12 + len); + if (log_ptr != NULL) { log_ptr = mlog_write_initial_log_record_fast( page, @@ -515,25 +511,26 @@ fil_space_write_crypt_data_low(fil_space_crypt_t *crypt_data, mlog_catenate_string(mtr, crypt_data->iv, len); } - - DBUG_EXECUTE_IF("ib_file_crypt_redo_crash_2", - ut_error;); } /****************************************************************** Write crypt data to a page (0) */ UNIV_INTERN void -fil_space_write_crypt_data(ulint space, byte* page, ulint offset, - ulint maxsize, mtr_t* mtr) +fil_space_write_crypt_data( +/*=======================*/ + ulint space, /*mutex); - memcpy(iv, crypt_data->iv, crypt_data->iv_length); - mutex_exit(&crypt_data->mutex); - } + /* For AES CTR create counter and AES CBS IV */ + mach_write_to_4(iv + 0, space); + ulint space_offset = mach_read_from_4(src_frame + FIL_PAGE_OFFSET); + mach_write_to_4(iv + 4, space_offset); + mach_write_to_8(iv + 8, lsn); ibool page_compressed = (mach_read_from_2(src_frame+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED); - ibool page_encrypted = fil_space_is_page_encrypted(space); - ulint compression_alg = mach_read_from_8(src_frame+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); + ulint compression_algo = page_compressed ? mach_read_from_8(src_frame+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION) : 0; - // copy page header + /* FIL page header is not encrypted */ memcpy(dst_frame, src_frame, FIL_PAGE_DATA); - if (page_encrypted && !page_compressed) { - // key id - mach_write_to_2(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, - key_version); - // original page type - mach_write_to_2(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 2, - orig_page_type); - // new page type - mach_write_to_2(dst_frame+FIL_PAGE_TYPE, FIL_PAGE_PAGE_ENCRYPTED); - } else { - // store key version - mach_write_to_4(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, - key_version); - } + /* Store key version */ + mach_write_to_4(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, key_version); - // encrypt page data + /* Calculate the start offset in a page */ ulint unencrypted_bytes = FIL_PAGE_DATA + FIL_PAGE_DATA_END; ulint srclen = page_size - unencrypted_bytes; const byte* src = src_frame + FIL_PAGE_DATA; byte* dst = dst_frame + FIL_PAGE_DATA; - uint32 dstlen; + uint32 dstlen=0; + /* For page compressed tables we encrypt only the actual compressed + payload. Note that first two bytes of page data is actual payload + size and that should not be encrypted. */ if (page_compressed) { - srclen = page_size - FIL_PAGE_DATA; + ulint payload = mach_read_from_2(src_frame + FIL_PAGE_DATA); + mach_write_to_2(dst_frame + FIL_PAGE_DATA, payload); + srclen = payload; + src+=2; + dst+=2; } - int rc = (* my_aes_encrypt_dynamic)(src, srclen, - dst, &dstlen, - (unsigned char*)key, key_length, - (unsigned char*)iv, sizeof(iv), - 1); + /* Get encryption method */ + my_aes_encrypt_dynamic_type func = get_aes_encrypt_func((enum_my_aes_encryption_algorithm)fil_crypt_map_aes_method(aes_method)); + + /* Encrypt the data */ + int rc = (*func)(src, /* Original page */ + srclen, /* Page length */ + dst, /* Output buffer */ + &dstlen, /* Output length */ + key, /* Encryption key */ + key_length, /* Key length */ + iv, /* IV */ + sizeof(iv), /* IV length */ + 1); /* Use noPadding */ if (! ((rc == AES_OK) && ((ulint) dstlen == srclen))) { ib_logf(IB_LOG_LEVEL_FATAL, @@ -735,18 +767,164 @@ fil_space_encrypt(ulint space, ulint offset, lsn_t lsn, ut_error; } + /* Set up the checksum */ + mach_write_to_4(dst_frame+FIL_PAGE_SPACE_OR_CHKSUM, BUF_NO_CHECKSUM_MAGIC); + + /* For compressed tables we do not store the FIL header because + the whole page is not stored to the disk. In compressed tables only + the FIL header + compressed (and now encrypted) payload alligned + to sector boundary is written. */ if (!page_compressed) { - // copy page trailer + /* FIL page trailer is also not encrypted */ memcpy(dst_frame + page_size - FIL_PAGE_DATA_END, src_frame + page_size - FIL_PAGE_DATA_END, FIL_PAGE_DATA_END); + } - /* handle post encryption checksum */ + /* Store AES encryption method */ + mach_write_to_2(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4, + aes_method); + /* Store compression algorithm (for page compresed tables) or 0 */ + mach_write_to_2(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 6, + compression_algo); +} + +/********************************************************************* +Check if extra buffer shall be allocated for decrypting after read +@return true if fil space has encryption data. */ +UNIV_INTERN +bool +fil_space_check_encryption_read( +/*=============================*/ + ulint space) /*!< in: tablespace id */ +{ + fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space); + + if (crypt_data == NULL) { + return false; + } + + if (crypt_data->type == CRYPT_SCHEME_UNENCRYPTED) { + return false; + } + + return true; +} + +/****************************************************************** +Decrypt a page +@return true if page was encrypted */ +UNIV_INTERN +bool +fil_space_decrypt( +/*==============*/ + fil_space_crypt_t* crypt_data, /*!< in: crypt data */ + const byte* src_frame, /*!< in: input buffer */ + ulint page_size, /*!< in: page size */ + byte* dst_frame) /*!< out: output buffer */ +{ + ulint page_type = mach_read_from_2(src_frame+FIL_PAGE_TYPE); + uint key_version = mach_read_from_4(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); + bool page_compressed = (page_type == FIL_PAGE_PAGE_COMPRESSED); + ulint compression_algo = mach_read_from_2(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 6); + uint aes_method = 0; + + /* Page is not encrypted if key_version is 0 */ + if (key_version == 0) { + //TODO: is this really needed ? + memcpy(dst_frame, src_frame, page_size); + return false; /* page not decrypted */ + } + + /* read space & offset & lsn */ + ulint space = mach_read_from_4( + src_frame + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); + ulint offset = mach_read_from_4( + src_frame + FIL_PAGE_OFFSET); + ib_uint64_t lsn = mach_read_from_8(src_frame + FIL_PAGE_LSN); + + /* Copy FIL page header, it is not encrypted */ + memcpy(dst_frame, src_frame, FIL_PAGE_DATA); + + /* Get key */ + byte key[MY_AES_MAX_KEY_LENGTH]; + uint key_length; + unsigned char iv[MY_AES_BLOCK_SIZE]; + fil_crypt_get_key(key, &key_length, crypt_data, key_version); + aes_method = mach_read_from_2(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4); + + /* Verify used AES method */ + if ( aes_method != CRYPT_SCHEME_1_CTR && + aes_method != CRYPT_SCHEME_1_CBC) { + ib_logf(IB_LOG_LEVEL_FATAL, + "Unable to decrypt data-block " + " aes_method: %d unknown!\n", + aes_method); + ut_error; + } + + /* Create counter used as IV */ + mach_write_to_4(iv + 0, space); + mach_write_to_4(iv + 4, offset); + mach_write_to_8(iv + 8, lsn); + + /* Calculate the offset where decryption starts */ + const byte* src = src_frame + FIL_PAGE_DATA; + byte* dst = dst_frame + FIL_PAGE_DATA; + uint32 dstlen=0; + ulint srclen = page_size - (FIL_PAGE_DATA + FIL_PAGE_DATA_END); + + /* For page compressed tables we decrypt only the actual compressed + payload. Note that first two bytes of page data is actual payload + size and that should not be decrypted. */ + if (page_compressed) { + ulint compressed_len = mach_read_from_2(src_frame + FIL_PAGE_DATA); + src+=2; + dst+=2; + mach_write_to_2(dst_frame + FIL_PAGE_DATA, compressed_len); + srclen = compressed_len; + } + + /* Get AES method */ + my_aes_encrypt_dynamic_type func = get_aes_decrypt_func((enum_my_aes_encryption_algorithm)fil_crypt_map_aes_method(aes_method)); + + /* Decrypt the data */ + int rc = (*func)(src, /* Data to be encrypted */ + srclen, /* data length */ + dst, /* Output buffer */ + &dstlen, /* Output buffer */ + key, /* Key */ + key_length, /* Key length */ + iv, /* IV */ + sizeof(iv), /* IV-length */ + 1); /* NoPadding */ + + if (! ((rc == AES_OK) && ((ulint) dstlen == srclen))) { + ib_logf(IB_LOG_LEVEL_FATAL, + "Unable to decrypt data-block " + " src: %p srclen: %ld buf: %p buflen: %d." + " return-code: %d. Can't continue!\n", + src, (long)srclen, + dst, dstlen, rc); + ut_error; + } + + /* For compressed tables we do not store the FIL header because + the whole page is not stored to the disk. In compressed tables only + the FIL header + compressed (and now encrypted) payload alligned + to sector boundary is written. */ + if (!page_compressed) { + /* Copy FIL trailer */ + memcpy(dst_frame + page_size - FIL_PAGE_DATA_END, + src_frame + page_size - FIL_PAGE_DATA_END, + FIL_PAGE_DATA_END); + + /* handle post decryption checksum */ ib_uint32_t checksum = 0; srv_checksum_algorithm_t algorithm = static_cast(srv_checksum_algorithm); - if (zip_size == 0) { + if (page_size == UNIV_PAGE_SIZE) { switch (algorithm) { case SRV_CHECKSUM_ALGORITHM_CRC32: case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32: @@ -765,173 +943,23 @@ fil_space_encrypt(ulint space, ulint offset, lsn_t lsn, * if new enum is added and not handled here */ } } else { - checksum = page_zip_calc_checksum(dst_frame, zip_size, + checksum = page_zip_calc_checksum(dst_frame, page_size, algorithm); } - // store the post-encryption checksum after the key-version - mach_write_to_4(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4, - checksum); - } else { - /* Page compressed and encrypted tables have different - FIL_HEADER */ - ulint page_len = log10((double)page_size)/log10((double)2); - /* Set up the correct page type */ - mach_write_to_2(dst_frame+FIL_PAGE_TYPE, FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED); - /* Set up the compression algorithm */ - mach_write_to_2(dst_frame+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION+4, orig_page_type); - /* Set up the compressed size */ - mach_write_to_1(dst_frame+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION+6, page_len); - /* Set up the compression method */ - mach_write_to_1(dst_frame+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION+7, compression_alg); + mach_write_to_4(dst_frame + FIL_PAGE_SPACE_OR_CHKSUM, checksum); + + fprintf(stderr, "KUKKU %ld:%lu\n", mach_read_from_4(dst_frame + FIL_PAGE_SPACE_OR_CHKSUM), + mach_read_from_4(dst_frame + FIL_PAGE_END_LSN_OLD_CHKSUM)); } -} - -/********************************************************************* -Check if extra buffer shall be allocated for decrypting after read */ -UNIV_INTERN -bool -fil_space_check_encryption_read( -/*==============================*/ - ulint space) /*!< in: tablespace id */ -{ - fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space); - - if (crypt_data == NULL) { - return false; - } - - if (crypt_data->type == CRYPT_SCHEME_UNENCRYPTED) { - return false; - } - - return true; -} - -/****************************************************************** -Decrypt a page */ -UNIV_INTERN -bool -fil_space_decrypt(fil_space_crypt_t* crypt_data, - const byte* src_frame, ulint page_size, byte* dst_frame) -{ - ulint page_type = mach_read_from_2(src_frame+FIL_PAGE_TYPE); - // key version - uint key_version; - bool page_encrypted = (page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED - || page_type == FIL_PAGE_PAGE_ENCRYPTED); - - bool page_compressed = (page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED - || page_type == FIL_PAGE_PAGE_COMPRESSED); - - ulint orig_page_type=0; - - if (page_type == FIL_PAGE_PAGE_ENCRYPTED) { - key_version = mach_read_from_2( - src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); - orig_page_type = mach_read_from_2( - src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 2); - } else { - key_version = mach_read_from_4( - src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); - } - - if (key_version == 0 && !page_encrypted) { - //TODO: is this really needed ? - memcpy(dst_frame, src_frame, page_size); - return false; /* page not decrypted */ - } - - // read space & offset & lsn - ulint space = mach_read_from_4( - src_frame + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); - ulint offset = mach_read_from_4( - src_frame + FIL_PAGE_OFFSET); - ib_uint64_t lsn = mach_read_from_8(src_frame + FIL_PAGE_LSN); - - // copy page header - memcpy(dst_frame, src_frame, FIL_PAGE_DATA); - - if (page_type == FIL_PAGE_PAGE_ENCRYPTED) { - // orig page type - mach_write_to_2(dst_frame+FIL_PAGE_TYPE, orig_page_type); - } - - // get key - byte key[MY_AES_MAX_KEY_LENGTH]; - uint key_length; - fil_crypt_get_key(key, &key_length, crypt_data, key_version, page_encrypted); - - // get the iv - unsigned char iv[MY_AES_BLOCK_SIZE]; - - if (current_aes_dynamic_method == MY_AES_ALGORITHM_CTR) { - // create counter block - - mach_write_to_4(iv + 0, space); - mach_write_to_4(iv + 4, offset); - mach_write_to_8(iv + 8, lsn); - } else { - // Get random IV from crypt_data - mutex_enter(&crypt_data->mutex); - memcpy(iv, crypt_data->iv, crypt_data->iv_length); - mutex_exit(&crypt_data->mutex); - } - - const byte* src = src_frame + FIL_PAGE_DATA; - byte* dst = dst_frame + FIL_PAGE_DATA; - uint32 dstlen; - ulint srclen = page_size - (FIL_PAGE_DATA + FIL_PAGE_DATA_END); - ulint compressed_len; - ulint compression_method; + /* Clear key-version & aes_method from dst */ + memset(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, 0, 8); + /* For page compressed tables store compression algorithm back */ if (page_compressed) { - orig_page_type = mach_read_from_2(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION+4); - compressed_len = mach_read_from_1(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION+6); - compression_method = mach_read_from_1(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION+7); - } - - if (page_encrypted && !page_compressed) { - orig_page_type = mach_read_from_2(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION+2); - } - - if (page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { - srclen = pow((double)2, (double)((int)compressed_len)) - FIL_PAGE_DATA; - } - - int rc = (* my_aes_decrypt_dynamic)(src, srclen, - dst, &dstlen, - (unsigned char*)key, key_length, - (unsigned char*)iv, sizeof(iv), - 1); - - if (! ((rc == AES_OK) && ((ulint) dstlen == srclen))) { - ib_logf(IB_LOG_LEVEL_FATAL, - "Unable to decrypt data-block " - " src: %p srclen: %ld buf: %p buflen: %d." - " return-code: %d. Can't continue!\n", - src, (long)srclen, - dst, dstlen, rc); - ut_error; - } - - if (page_type != FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { - // copy page trailer - memcpy(dst_frame + page_size - FIL_PAGE_DATA_END, - src_frame + page_size - FIL_PAGE_DATA_END, - FIL_PAGE_DATA_END); - - // clear key-version & crypt-checksum from dst - memset(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, 0, 8); - } else { - /* For page compressed tables we set up the FIL_HEADER again */ - /* setting original page type */ - mach_write_to_2(dst_frame + FIL_PAGE_TYPE, orig_page_type); - /* page_compression uses BUF_NO_CHECKSUM_MAGIC as checksum */ - mach_write_to_4(dst_frame + FIL_PAGE_SPACE_OR_CHKSUM, BUF_NO_CHECKSUM_MAGIC); - /* Set up the flush lsn to be compression algorithm */ - mach_write_to_8(dst_frame+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, compression_method); + mach_write_to_8(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, + compression_algo); } return true; /* page was decrypted */ @@ -941,8 +969,12 @@ fil_space_decrypt(fil_space_crypt_t* crypt_data, Decrypt a page */ UNIV_INTERN void -fil_space_decrypt(ulint space, - const byte* src_frame, ulint page_size, byte* dst_frame) +fil_space_decrypt( +/*==============*/ + ulint space, /*!< in: Fil space id */ + const byte* src_frame, /*!< in: input buffer */ + ulint page_size, /*!< in: page size */ + byte* dst_frame) /*!< out: output buffer */ { fil_space_decrypt(fil_space_get_crypt_data(space), src_frame, page_size, dst_frame); @@ -953,8 +985,13 @@ Verify checksum for a page (iff it's encrypted) NOTE: currently this function can only be run in single threaded mode as it modifies srv_checksum_algorithm (temporarily) @return true if page is encrypted AND OK, false otherwise */ +UNIV_INTERN bool -fil_space_verify_crypt_checksum(const byte* src_frame, ulint zip_size) +fil_space_verify_crypt_checksum( +/*============================*/ + const byte* src_frame, /*!< in: page the verify */ + ulint zip_size) /*!< in: compressed size if + row_format compressed */ { // key version uint key_version = mach_read_from_4( @@ -1041,7 +1078,8 @@ struct key_state_t { Copy global key state */ static void fil_crypt_get_key_state( - key_state_t *new_state) +/*====================*/ + key_state_t *new_state) /*!< out: key state */ { if (srv_encrypt_tables == TRUE) { new_state->key_version = get_latest_encryption_key_version(); @@ -1054,9 +1092,13 @@ fil_crypt_get_key_state( } /*********************************************************************** -Check if a key needs rotation given a key_state */ +Check if a key needs rotation given a key_state +@return true if key needs rotation, false if not */ static bool -fil_crypt_needs_rotation(uint key_version, const key_state_t *key_state) +fil_crypt_needs_rotation( +/*=====================*/ + uint key_version, /*!< in: Key version */ + const key_state_t* key_state) /*!< in: Key state */ { // TODO(jonaso): Add support for rotating encrypted => unencrypted @@ -1081,9 +1123,13 @@ fil_crypt_needs_rotation(uint key_version, const key_state_t *key_state) } /*********************************************************************** -Check if a space is closing (i.e just before drop) */ -UNIV_INTERN bool -fil_crypt_is_closing(ulint space) +Check if a space is closing (i.e just before drop) +@return true if space is closing, false if not. */ +UNIV_INTERN +bool +fil_crypt_is_closing( +/*=================*/ + ulint space) /*!< in: FIL space id */ { bool closing; fil_space_crypt_t *crypt_data = fil_space_get_crypt_data(space); @@ -1097,16 +1143,22 @@ fil_crypt_is_closing(ulint space) Start encrypting a space @return true if a pending op (fil_inc_pending_ops/fil_decr_pending_ops) is held */ -static bool -fil_crypt_start_encrypting_space(ulint space, bool *recheck) { +static +bool +fil_crypt_start_encrypting_space( +/*=============================*/ + ulint space, /*!< in: FIL space id */ + bool* recheck)/*!< out: true if recheck needed */ +{ /* we have a pending op when entering function */ bool pending_op = true; + uint aes_method = current_aes_dynamic_method; mutex_enter(&fil_crypt_threads_mutex); fil_space_crypt_t *crypt_data = fil_space_get_crypt_data(space); - ibool page_encrypted = fil_space_is_page_encrypted(space); + ibool page_encrypted = (crypt_data != NULL); /*If spage is not encrypted and encryption is not enabled, then do not continue encrypting the space. */ @@ -1184,7 +1236,7 @@ fil_crypt_start_encrypting_space(ulint space, bool *recheck) { /* 4 - write crypt data to page 0 */ fil_space_write_crypt_data_low(crypt_data, - CRYPT_SCHEME_1, + fil_crypt_get_aes_method(aes_method), frame, crypt_data->page0_offset, maxsize, &mtr); @@ -1232,7 +1284,7 @@ fil_crypt_start_encrypting_space(ulint space, bool *recheck) { /* 5 - publish crypt data */ mutex_enter(&fil_crypt_threads_mutex); mutex_enter(&crypt_data->mutex); - crypt_data->type = CRYPT_SCHEME_1; + crypt_data->type = fil_crypt_get_aes_method(aes_method); ut_a(crypt_data->rotate_state.active_threads == 1); crypt_data->rotate_state.active_threads = 0; crypt_data->rotate_state.starting = false; @@ -1257,10 +1309,14 @@ fil_crypt_start_encrypting_space(ulint space, bool *recheck) { } /*********************************************************************** -Check if space needs rotation given a key_state */ -static bool -fil_crypt_space_needs_rotation(uint space, const key_state_t *key_state, - bool *recheck) +Check if space needs rotation given a key_state +@return true if space needs key rotation */ +static +bool +fil_crypt_space_needs_rotation( + uint space, /*!< in: FIL space id */ + const key_state_t* key_state, /*!< in: Key state */ + bool* recheck) /*!< out: needs recheck ? */ { if (fil_space_get_type(space) != FIL_TABLESPACE) { return false; @@ -1374,7 +1430,9 @@ struct rotate_thread_t { /*********************************************************************** Update global statistics with thread statistics */ static void -fil_crypt_update_total_stat(rotate_thread_t *state) +fil_crypt_update_total_stat( +/*========================*/ + rotate_thread_t *state) /*!< in: Key rotation status */ { mutex_enter(&crypt_stat_mutex); crypt_stat.pages_read_from_cache += @@ -1397,9 +1455,13 @@ fil_crypt_update_total_stat(rotate_thread_t *state) /*********************************************************************** Allocate iops to thread from global setting, -used before starting to rotate a space */ -static bool -fil_crypt_alloc_iops(rotate_thread_t *state) +used before starting to rotate a space. +@return true if allocation succeeded, false if failed */ +static +bool +fil_crypt_alloc_iops( +/*=================*/ + rotate_thread_t *state) /*!< in: Key rotation status */ { ut_ad(state->allocated_iops == 0); @@ -1429,8 +1491,11 @@ fil_crypt_alloc_iops(rotate_thread_t *state) /*********************************************************************** Reallocate iops to thread, used when inside a space */ -static void -fil_crypt_realloc_iops(rotate_thread_t *state) +static +void +fil_crypt_realloc_iops( +/*========================*/ + rotate_thread_t *state) /*!< in: Key rotation status */ { ut_a(state->allocated_iops > 0); @@ -1519,8 +1584,11 @@ fil_crypt_realloc_iops(rotate_thread_t *state) /*********************************************************************** Return allocated iops to global */ -static void -fil_crypt_return_iops(rotate_thread_t *state) +static +void +fil_crypt_return_iops( +/*========================*/ + rotate_thread_t *state) /*!< in: Key rotation status */ { if (state->allocated_iops > 0) { uint iops = state->allocated_iops; @@ -1544,11 +1612,14 @@ fil_crypt_return_iops(rotate_thread_t *state) /*********************************************************************** Search for a space needing rotation */ +UNIV_INTERN bool fil_crypt_find_space_to_rotate( - const key_state_t *key_state, - rotate_thread_t *state, - bool *recheck) +/*===========================*/ + const key_state_t* key_state, /*!< in: Key state */ + rotate_thread_t* state, /*!< in: Key rotation state */ + bool* recheck) /*!< out: true if recheck + needed */ { /* we need iops to start rotating */ while (!state->should_shutdown() && !fil_crypt_alloc_iops(state)) { @@ -1591,8 +1662,9 @@ Start rotating a space */ static void fil_crypt_start_rotate_space( - const key_state_t *key_state, - rotate_thread_t *state) +/*=========================*/ + const key_state_t* key_state, /*!< in: Key state */ + rotate_thread_t* state) /*!< in: Key rotation state */ { ulint space = state->space; fil_space_crypt_t *crypt_data = fil_space_get_crypt_data(space); @@ -1629,12 +1701,14 @@ fil_crypt_start_rotate_space( } /*********************************************************************** -Search for batch of pages needing rotation */ +Search for batch of pages needing rotation +@return true if page needing key rotation found, false if not found */ static bool fil_crypt_find_page_to_rotate( - const key_state_t *key_state, - rotate_thread_t *state) +/*==========================*/ + const key_state_t* key_state, /*!< in: Key state */ + rotate_thread_t* state) /*!< in: Key rotation state */ { ulint batch = srv_alloc_time * state->allocated_iops; ulint space = state->space; @@ -1665,9 +1739,15 @@ fil_crypt_find_page_to_rotate( } /*********************************************************************** -Check if a page is uninitialized (doesn't need to be rotated) */ -static bool -fil_crypt_is_page_uninitialized(const byte* frame, uint zip_size) +Check if a page is uninitialized (doesn't need to be rotated) +@return true if page is uninitialized, false if not.*/ +static +bool +fil_crypt_is_page_uninitialized( +/*============================*/ + const byte *frame, /*!< in: Page */ + uint zip_size) /*!< in: compressed size if + row_format compressed */ { if (zip_size) { ulint stored_checksum = mach_read_from_4( @@ -1696,15 +1776,20 @@ fil_crypt_is_page_uninitialized(const byte* frame, uint zip_size) sleeptime_ms, __FILE__, __LINE__) /*********************************************************************** -Get a page and compute sleep time */ +Get a page and compute sleep time +@return page */ static buf_block_t* -fil_crypt_get_page_throttle_func(rotate_thread_t *state, - ulint space, uint zip_size, ulint offset, - mtr_t *mtr, - ulint *sleeptime_ms, - const char *file, - ulint line) +fil_crypt_get_page_throttle_func( + rotate_thread_t* state, /*!< in/out: Key rotation state */ + ulint space, /*!< in: FIL space id */ + uint zip_size, /*!< in: compressed size if + row_format compressed */ + ulint offset, /*!< in: page offsett */ + mtr_t* mtr, /*!< in/out: minitransaction */ + ulint* sleeptime_ms, /*!< out: sleep time */ + const char* file, /*!< in: file name */ + ulint line) /*!< in: file line */ { buf_block_t* block = buf_page_try_get_func(space, offset, RW_X_LATCH, true, @@ -1753,17 +1838,22 @@ Get block and allocation status note: innodb locks fil_space_latch and then block when allocating page but locks block and then fil_space_latch when freeing page. +@return block */ static buf_block_t* btr_scrub_get_block_and_allocation_status( - rotate_thread_t *state, - ulint space, - ulint zip_size, - ulint offset, - mtr_t *mtr, +/*======================================*/ + rotate_thread_t* state, /*!< in/out: Key rotation state */ + ulint space, /*!< in: FIL space id */ + uint zip_size, /*!< in: compressed size if + row_format compressed */ + ulint offset, /*!< in: page offsett */ + mtr_t* mtr, /*!< in/out: minitransaction + */ btr_scrub_page_allocation_status_t *allocation_status, - ulint *sleeptime_ms) + /*!< in/out: allocation status */ + ulint* sleeptime_ms) /*!< out: sleep time */ { mtr_t local_mtr; buf_block_t *block = NULL; @@ -1807,8 +1897,9 @@ Rotate one page */ static void fil_crypt_rotate_page( - const key_state_t *key_state, - rotate_thread_t *state) +/*===================*/ + const key_state_t* key_state, /*!< in: Key state */ + rotate_thread_t* state) /*!< in: Key rotation state */ { ulint space = state->space; ulint offset = state->offset; @@ -1960,8 +2051,9 @@ Rotate a batch of pages */ static void fil_crypt_rotate_pages( - const key_state_t *key_state, - rotate_thread_t *state) +/*===================*/ + const key_state_t* key_state, /*!< in: Key state */ + rotate_thread_t* state) /*!< in: Key rotation state */ { ulint space = state->space; ulint end = state->offset + state->batch; @@ -1989,7 +2081,10 @@ fil_crypt_rotate_pages( Flush rotated pages and then update page 0 */ static void -fil_crypt_flush_space(rotate_thread_t *state, ulint space) +fil_crypt_flush_space( +/*==================*/ + rotate_thread_t* state, /*!< in: Key rotation state */ + ulint space) /*!< in: FIL space id */ { fil_space_crypt_t *crypt_data = fil_space_get_crypt_data(space); @@ -2033,9 +2128,10 @@ fil_crypt_flush_space(rotate_thread_t *state, ulint space) RW_X_LATCH, NULL, BUF_GET, __FILE__, __LINE__, &mtr); byte* frame = buf_block_get_frame(block); - ulint maxsize = 0; + ulint maxsize; crypt_data->page0_offset = fsp_header_get_crypt_offset(zip_size, &maxsize); + fil_space_write_crypt_data(space, frame, crypt_data->page0_offset, ULINT_MAX, &mtr); @@ -2048,8 +2144,9 @@ Complete rotating a space */ static void fil_crypt_complete_rotate_space( - const key_state_t *key_state, - rotate_thread_t *state) +/*============================*/ + const key_state_t* key_state, /*!< in: Key state */ + rotate_thread_t* state) /*!< in: Key rotation state */ { ulint space = state->space; fil_space_crypt_t *crypt_data = fil_space_get_crypt_data(space); @@ -2123,7 +2220,7 @@ A thread which monitors global key state and rotates tablespaces accordingly extern "C" UNIV_INTERN os_thread_ret_t DECLARE_THREAD(fil_crypt_thread)( -/*===============================*/ +/*=============================*/ void* arg __attribute__((unused))) /*!< in: a dummy parameter required * by os_thread_create */ { @@ -2240,13 +2337,19 @@ DECLARE_THREAD(fil_crypt_thread)( Adjust thread count for key rotation */ UNIV_INTERN void -fil_crypt_set_thread_cnt(uint new_cnt) +fil_crypt_set_thread_cnt( +/*=====================*/ + uint new_cnt) /*!< in: New key rotation thread count */ { if (new_cnt > srv_n_fil_crypt_threads) { uint add = new_cnt - srv_n_fil_crypt_threads; srv_n_fil_crypt_threads = new_cnt; for (uint i = 0; i < add; i++) { - os_thread_create(fil_crypt_thread, NULL, NULL); + os_thread_id_t rotation_thread_id; + os_thread_create(fil_crypt_thread, NULL, &rotation_thread_id); + ib_logf(IB_LOG_LEVEL_INFO, + "Creating #%d thread id %lu total threads %du\n", + i, os_thread_pf(rotation_thread_id), new_cnt); } } else if (new_cnt < srv_n_fil_crypt_threads) { srv_n_fil_crypt_threads = new_cnt; @@ -2263,7 +2366,9 @@ fil_crypt_set_thread_cnt(uint new_cnt) Adjust max key age */ UNIV_INTERN void -fil_crypt_set_rotate_key_age(uint val) +fil_crypt_set_rotate_key_age( +/*=========================*/ + uint val) /*!< in: New max key age */ { srv_fil_crypt_rotate_key_age = val; os_event_set(fil_crypt_threads_event); @@ -2273,7 +2378,9 @@ fil_crypt_set_rotate_key_age(uint val) Adjust rotation iops */ UNIV_INTERN void -fil_crypt_set_rotation_iops(uint val) +fil_crypt_set_rotation_iops( +/*========================*/ + uint val) /*!< in: New iops setting */ { srv_n_fil_crypt_iops = val; os_event_set(fil_crypt_threads_event); @@ -2284,6 +2391,7 @@ Init threads for key rotation */ UNIV_INTERN void fil_crypt_threads_init() +/*====================*/ { fil_crypt_event = os_event_create(); fil_crypt_threads_event = os_event_create(); @@ -2300,6 +2408,7 @@ End threads for key rotation */ UNIV_INTERN void fil_crypt_threads_end() +/*===================*/ { /* stop threads */ fil_crypt_set_thread_cnt(0); @@ -2309,7 +2418,9 @@ fil_crypt_threads_end() Clean up key rotation threads resources */ UNIV_INTERN void -fil_crypt_threads_cleanup() { +fil_crypt_threads_cleanup() +/*=======================*/ +{ os_event_free(fil_crypt_event); os_event_free(fil_crypt_threads_event); } @@ -2319,7 +2430,8 @@ Mark a space as closing */ UNIV_INTERN void fil_space_crypt_mark_space_closing( - ulint space) +/*===============================*/ + ulint space) /*!< in: Space id */ { mutex_enter(&fil_crypt_threads_mutex); fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space); @@ -2340,7 +2452,8 @@ Wait for crypt threads to stop accessing space */ UNIV_INTERN void fil_space_crypt_close_tablespace( - ulint space) +/*=============================*/ + ulint space) /*!< in: Space id */ { mutex_enter(&fil_crypt_threads_mutex); fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space); @@ -2352,11 +2465,14 @@ fil_space_crypt_close_tablespace( uint start = time(0); uint last = start; + mutex_enter(&crypt_data->mutex); mutex_exit(&fil_crypt_threads_mutex); crypt_data->closing = true; + uint cnt = crypt_data->rotate_state.active_threads; bool flushing = crypt_data->rotate_state.flushing; + while (cnt > 0 || flushing) { mutex_exit(&crypt_data->mutex); /* release dict mutex so that scrub threads can release their @@ -2386,11 +2502,12 @@ fil_space_crypt_close_tablespace( /********************************************************************* Get crypt status for a space (used by information_schema) return 0 if crypt data present */ +UNIV_INTERN int fil_space_crypt_get_status( -/*==================*/ - ulint id, /*!< in: space id */ - struct fil_space_crypt_status_t* status) /*!< out: status */ +/*=======================*/ + ulint id, /*!< in: space id */ + struct fil_space_crypt_status_t* status) /*!< out: status */ { fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(id); @@ -2400,6 +2517,7 @@ fil_space_crypt_get_status( mutex_enter(&crypt_data->mutex); status->keyserver_requests = crypt_data->keyserver_requests; status->min_key_version = crypt_data->min_key_version; + if (crypt_data->rotate_state.active_threads > 0 || crypt_data->rotate_state.flushing) { status->rotating = true; @@ -2427,8 +2545,11 @@ fil_space_crypt_get_status( /********************************************************************* Return crypt statistics */ +UNIV_INTERN void -fil_crypt_total_stat(fil_crypt_stat_t *stat) +fil_crypt_total_stat( +/*=================*/ + fil_crypt_stat_t *stat) /*!< out: Crypt statistics */ { mutex_enter(&crypt_stat_mutex); *stat = crypt_stat; @@ -2438,11 +2559,12 @@ fil_crypt_total_stat(fil_crypt_stat_t *stat) /********************************************************************* Get scrub status for a space (used by information_schema) return 0 if data found */ +UNIV_INTERN int fil_space_get_scrub_status( -/*==================*/ - ulint id, /*!< in: space id */ - struct fil_space_scrub_status_t* status) /*!< out: status */ +/*=======================*/ + ulint id, /*!< in: space id */ + struct fil_space_scrub_status_t* status) /*!< out: status */ { fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(id); memset(status, 0, sizeof(*status)); diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index 14878f59f7d..6b776edd2f9 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -27,8 +27,7 @@ Created 10/25/1995 Heikki Tuuri #include "fil0fil.h" #include "fil0pagecompress.h" #include "fsp0pagecompress.h" -#include "fil0pageencryption.h" -#include "fsp0pageencryption.h" +#include "fil0crypt.h" #include #include @@ -287,7 +286,7 @@ fil_read( actual page size does not decrease. */ { return(fil_io(OS_FILE_READ, sync, space_id, zip_size, block_offset, - byte_offset, len, buf, message, write_size, 0, false)); + byte_offset, len, buf, message, write_size)); } /********************************************************************//** @@ -314,18 +313,16 @@ fil_write( this must be appropriately aligned */ void* message, /*!< in: message for aio handler if non-sync aio used, else ignored */ - ulint* write_size, /*!< in/out: Actual write size initialized + ulint* write_size) /*!< in/out: Actual write size initialized after fist successfull trim operation for this page and if initialized we do not trim again if actual page size does not decrease. */ - lsn_t lsn, /*!< in: lsn of the newest modification */ - bool encrypt_later) /*!< in: encrypt later ? */ { ut_ad(!srv_read_only_mode); return(fil_io(OS_FILE_WRITE, sync, space_id, zip_size, block_offset, - byte_offset, len, buf, message, write_size, lsn, encrypt_later)); + byte_offset, len, buf, message, write_size)); } /*******************************************************************//** @@ -651,10 +648,9 @@ fil_node_open_file( set */ page = static_cast(ut_align(buf2, UNIV_PAGE_SIZE)); - success = os_file_read(node->handle, page, 0, UNIV_PAGE_SIZE, - space->flags); + success = os_file_read(node->handle, page, 0, UNIV_PAGE_SIZE); - if (fil_page_encryption_status(page)) { + if (fil_page_is_encrypted(page)) { /* if page is (still) encrypted, write an error and return. * Otherwise the server would crash if decrypting is not possible. * This may be the case, if the key file could not be @@ -1191,21 +1187,6 @@ fil_space_create( ut_a(fil_system); - if (fsp_flags_is_page_encrypted(flags)) { - if (!has_encryption_key(fsp_flags_get_page_encryption_key(flags))) { - /* by returning here it should be avoided that - * the server crashes, if someone tries to access an - * encrypted table and the encryption key is not available. - * The the table is treaded as non-existent. - */ - ib_logf(IB_LOG_LEVEL_WARN, - "Tablespace '%s' can not be opened, because " - " encryption key can not be found (space id: %lu, key %lu)\n" - , name, (ulong) id, fsp_flags_get_page_encryption_key(flags)); - return (FALSE); - } - } - /* Look for a matching tablespace and if found free it. */ do { mutex_enter(&fil_system->mutex); @@ -1879,7 +1860,7 @@ fil_write_lsn_and_arch_no_to_file( lsn); err = fil_write(TRUE, space, 0, sum_of_sizes, 0, - UNIV_PAGE_SIZE, buf, NULL, 0, 0, false); + UNIV_PAGE_SIZE, buf, NULL, 0); } mem_free(buf1); @@ -1970,13 +1951,10 @@ fil_check_first_page( or the encryption key is not available, the check for reading the first page should intentionally fail with "can not decrypt" message. */ - page_is_encrypted = fil_page_encryption_status(page); - if (page_is_encrypted == PAGE_ENCRYPTION_KEY_MISSING && page_is_encrypted) { - page_is_encrypted = 1; - } else { - page_is_encrypted = 0; + page_is_encrypted = fil_page_encryption_status(page, space_id); + if (!page_is_encrypted) { if (UNIV_PAGE_SIZE != fsp_flags_get_page_size(flags)) { - fprintf(stderr, + fprintf(stderr, "InnoDB: Error: Current page size %lu != " " page size on page %lu\n", UNIV_PAGE_SIZE, fsp_flags_get_page_size(flags)); @@ -2006,7 +1984,7 @@ fil_check_first_page( /* this error message is interpreted by the calling method, which is * executed if the server starts in recovery mode. */ - return(MSG_CANNOT_DECRYPT); + return(FIL_MSG_CANNOT_DECRYPT); } } @@ -2053,10 +2031,7 @@ fil_read_first_page( page = static_cast(ut_align(buf, UNIV_PAGE_SIZE)); - os_file_read(data_file, page, 0, UNIV_PAGE_SIZE, - orig_space_id != ULINT_UNDEFINED ? - fil_space_is_page_compressed(orig_space_id) : - FALSE); + os_file_read(data_file, page, 0, UNIV_PAGE_SIZE); /* The FSP_HEADER on page 0 is only valid for the first file in a tablespace. So if this is not the first datafile, leave @@ -4106,8 +4081,7 @@ fil_user_tablespace_find_space_id( for (ulint j = 0; j < page_count; ++j) { - st = os_file_read(fsp->file, page, (j* page_size), page_size, - fsp_flags_is_page_compressed(fsp->flags)); + st = os_file_read(fsp->file, page, (j* page_size), page_size); if (!st) { ib_logf(IB_LOG_LEVEL_INFO, @@ -4250,7 +4224,7 @@ check_first_page: "%s in tablespace %s (table %s)", check_msg, fsp->filepath, tablename); fsp->success = FALSE; - if (strncmp(check_msg, MSG_CANNOT_DECRYPT, strlen(check_msg))==0) { + if (strncmp(check_msg, FIL_MSG_CANNOT_DECRYPT, strlen(check_msg))==0) { /* by returning here, it should be avoided, that the server crashes, * if started in recovery mode and can not decrypt tables, if * the key file can not be read. @@ -5258,7 +5232,7 @@ retry: success = os_aio(OS_FILE_WRITE, OS_AIO_SYNC, node->name, node->handle, buf, offset, page_size * n_pages, - node, NULL, space_id, NULL, 0, 0, 0, 0, 0, 0, false); + node, NULL, space_id, NULL, 0); #endif /* UNIV_HOTBACKUP */ DBUG_EXECUTE_IF("ib_os_aio_func_io_failure_28", @@ -5643,9 +5617,7 @@ _fil_io( operation for this page and if initialized we do not trim again if actual page size does not decrease. */ - trx_t* trx, - lsn_t lsn, /*!< in: lsn of the newest modification */ - bool encrypt_later) /*!< in: encrypt later ? */ + trx_t* trx) { ulint mode; fil_space_t* space; @@ -5655,10 +5627,6 @@ _fil_io( ulint wake_later; os_offset_t offset; ibool ignore_nonexistent_pages; - ibool page_compressed = FALSE; - ulint page_compression_level = 0; - ibool page_encrypted; - ulint page_encryption_key; is_log = type & OS_FILE_LOG; type = type & ~OS_FILE_LOG; @@ -5726,13 +5694,6 @@ _fil_io( space = fil_space_get_by_id(space_id); - page_compressed = fsp_flags_is_page_compressed(space->flags); - page_compression_level = fsp_flags_get_page_compression_level(space->flags); - - page_encrypted = fsp_flags_is_page_encrypted(space->flags); - page_encryption_key = fsp_flags_get_page_encryption_key(space->flags); - - /* If we are deleting a tablespace we don't allow any read operations on that. However, we do allow write operations. */ if (space == 0 || (type == OS_FILE_READ && space->stop_new_ops)) { @@ -5889,13 +5850,7 @@ _fil_io( message, space_id, trx, - page_compressed, - page_compression_level, - write_size, - page_encrypted, - page_encryption_key, - lsn, - encrypt_later); + write_size); #else /* In mysqlbackup do normal i/o, not aio */ @@ -6454,9 +6409,7 @@ fil_iterate( readptr = iter.crypt_io_buffer; } - if (!os_file_read(iter.file, readptr, offset, - (ulint) n_bytes, - fil_space_is_page_compressed(space_id))) { + if (!os_file_read(iter.file, readptr, offset, (ulint) n_bytes)) { ib_logf(IB_LOG_LEVEL_ERROR, "os_file_read() failed"); @@ -6607,8 +6560,7 @@ fil_tablespace_iterate( /* Read the first page and determine the page and zip size. */ - if (!os_file_read(file, page, 0, UNIV_PAGE_SIZE, - dict_tf_get_page_compression(table->flags))) { + if (!os_file_read(file, page, 0, UNIV_PAGE_SIZE)) { err = DB_IO_ERROR; @@ -6668,7 +6620,7 @@ fil_tablespace_iterate( if (iter.crypt_data != NULL) { /* clear crypt data from page 0 and write it back */ - os_file_read(file, page, 0, UNIV_PAGE_SIZE, 0); + os_file_read(file, page, 0, UNIV_PAGE_SIZE); fil_space_clear_crypt_data(page, crypt_data_offset); lsn_t lsn = mach_read_from_8(page + FIL_PAGE_LSN); if (callback.get_zip_size() == 0) { @@ -6915,79 +6867,6 @@ fil_system_exit(void) mutex_exit(&fil_system->mutex); } -/*******************************************************************//** -Return space name */ -char* -fil_space_name( -/*===========*/ - fil_space_t* space) /*!< in: space */ -{ - return (space->name); -} - -/*******************************************************************//** -Return space flags */ -ulint -fil_space_flags( -/*===========*/ - fil_space_t* space) /*!< in: space */ -{ - return (space->flags); -} - -/*******************************************************************//** -Return page type name */ -const char* -fil_get_page_type_name( -/*===================*/ - ulint page_type) /*!< in: FIL_PAGE_TYPE */ -{ - switch(page_type) { - case FIL_PAGE_PAGE_COMPRESSED: - return (const char*)"PAGE_COMPRESSED"; - case FIL_PAGE_INDEX: - return (const char*)"INDEX"; - case FIL_PAGE_UNDO_LOG: - return (const char*)"UNDO LOG"; - case FIL_PAGE_INODE: - return (const char*)"INODE"; - case FIL_PAGE_IBUF_FREE_LIST: - return (const char*)"IBUF_FREE_LIST"; - case FIL_PAGE_TYPE_ALLOCATED: - return (const char*)"ALLOCATED"; - case FIL_PAGE_IBUF_BITMAP: - return (const char*)"IBUF_BITMAP"; - case FIL_PAGE_TYPE_SYS: - return (const char*)"SYS"; - case FIL_PAGE_TYPE_TRX_SYS: - return (const char*)"TRX_SYS"; - case FIL_PAGE_TYPE_FSP_HDR: - return (const char*)"FSP_HDR"; - case FIL_PAGE_TYPE_XDES: - return (const char*)"XDES"; - case FIL_PAGE_TYPE_BLOB: - return (const char*)"BLOB"; - case FIL_PAGE_TYPE_ZBLOB: - return (const char*)"ZBLOB"; - case FIL_PAGE_TYPE_ZBLOB2: - return (const char*)"ZBLOB2"; - case FIL_PAGE_TYPE_COMPRESSED: - return (const char*)"ORACLE PAGE COMPRESSED"; - default: - return (const char*)"PAGE TYPE CORRUPTED"; - } -} -/****************************************************************//** -Get block size from fil node -@return block size*/ -ulint -fil_node_get_block_size( -/*====================*/ - fil_node_t* node) /*!< in: Node where to get block - size */ -{ - return (node->file_block_size); -} /****************************************************************** Get id of first tablespace or ULINT_UNDEFINED if none */ diff --git a/storage/xtradb/fil/fil0pagecompress.cc b/storage/xtradb/fil/fil0pagecompress.cc index 3c2edd832c4..d394ca5215c 100644 --- a/storage/xtradb/fil/fil0pagecompress.cc +++ b/storage/xtradb/fil/fil0pagecompress.cc @@ -83,173 +83,6 @@ static ulint srv_data_read, srv_data_written; /* Used for debugging */ //#define UNIV_PAGECOMPRESS_DEBUG 1 -/****************************************************************//** -For page compressed pages decompress the page after actual read -operation. */ -static -void -fil_decompress_page_2( -/*==================*/ - byte* page_buf, /*!< out: destination buffer for - uncompressed data */ - byte* buf, /*!< in: source compressed data */ - ulong len, /*!< in: length of output buffer.*/ - ulint* write_size) /*!< in/out: Actual payload size of - the compressed data. */ -{ - ulint page_type = mach_read_from_2(buf + FIL_PAGE_TYPE); - - if (page_type != FIL_PAGE_TYPE_COMPRESSED) { - /* It is not a compressed page */ - return; - } - - byte* ptr = buf + FIL_PAGE_DATA; - ulint version = mach_read_from_1(buf + FIL_PAGE_VERSION); - int err = 0; - - ut_a(version == 1); - - /* Read the original page type, before we compressed the data. */ - page_type = mach_read_from_2(buf + FIL_PAGE_ORIGINAL_TYPE_V1); - - ulint original_len = mach_read_from_2(buf + FIL_PAGE_ORIGINAL_SIZE_V1); - - if (original_len < UNIV_PAGE_SIZE_MIN - (FIL_PAGE_DATA + 8) - || original_len > UNIV_PAGE_SIZE_MAX - FIL_PAGE_DATA - || len < original_len + FIL_PAGE_DATA) { - ib_logf(IB_LOG_LEVEL_ERROR, - "Corruption: We try to uncompress corrupted page. " - "Original len %lu len %lu.", - original_len, len); - - fflush(stderr); - ut_error; - - } - - ulint algorithm = mach_read_from_1(buf + FIL_PAGE_ALGORITHM_V1); - - switch(algorithm) { - case PAGE_ZLIB_ALGORITHM: { - - err = uncompress(page_buf, &len, ptr, original_len); - - /* If uncompress fails it means that page is corrupted */ - if (err != Z_OK) { - - ib_logf(IB_LOG_LEVEL_ERROR, - "Corruption: Page is marked as compressed " - "but uncompress failed with error %d " - " size %lu len %lu.", - err, original_len, len); - - fflush(stderr); - - ut_error; - } - - break; - } -#ifdef HAVE_LZ4 - case PAGE_LZ4_ALGORITHM: { - - err = LZ4_decompress_fast( - (const char*) ptr, (char*) (page_buf), original_len); - - if (err < 0) { - ib_logf(IB_LOG_LEVEL_ERROR, - "Corruption: Page is marked as compressed" - " but decompression read only %d bytes" - " size %lu len %lu.", - err, original_len, len); - fflush(stderr); - - ut_error; - } - break; - } -#endif /* HAVE_LZ4 */ - -#ifdef HAVE_LZMA - case PAGE_LZMA_ALGORITHM: { - - lzma_ret ret; - size_t src_pos = 0; - size_t dst_pos = 0; - uint64_t memlimit = UINT64_MAX; - - ret = lzma_stream_buffer_decode( - &memlimit, - 0, - NULL, - ptr, - &src_pos, - original_len, - (page_buf), - &dst_pos, - len); - - - if (ret != LZMA_OK || (dst_pos <= 0 || dst_pos > len)) { - ib_logf(IB_LOG_LEVEL_ERROR, - "Corruption: Page is marked as compressed" - " but decompression read only %ld bytes" - " size %lu len %lu.", - dst_pos, original_len, len); - fflush(stderr); - - ut_error; - } - - break; - } -#endif /* HAVE_LZMA */ - -#ifdef HAVE_LZO - case PAGE_LZO_ALGORITHM: { - ulint olen = 0; - - err = lzo1x_decompress((const unsigned char *)ptr, - original_len,(unsigned char *)(page_buf), &olen, NULL); - - if (err != LZO_E_OK || (olen == 0 || olen > UNIV_PAGE_SIZE)) { - ib_logf(IB_LOG_LEVEL_ERROR, - "Corruption: Page is marked as compressed" - " but decompression read only %ld bytes" - " size %lu len %lu.", - olen, original_len, len); - - fflush(stderr); - - ut_error; - } - break; - } -#endif /* HAVE_LZO */ - - default: - ib_logf(IB_LOG_LEVEL_ERROR, - " Corruption: Page is marked as compressed " - " but compression algorithm %s" - " is not known." - ,fil_get_compression_alg_name(algorithm)); - - fflush(stderr); - ut_error; - break; - } - - /* Leave the header alone */ - memmove(buf+FIL_PAGE_DATA, page_buf, original_len); - - mach_write_to_2(buf + FIL_PAGE_TYPE, page_type); - - ut_ad(memcmp(buf + FIL_PAGE_LSN + 4, - buf + (original_len + FIL_PAGE_DATA) - - FIL_PAGE_END_LSN_OLD_CHKSUM + 4, 4) == 0); -} - /****************************************************************//** For page compressed pages compress the page before actual write operation. @@ -270,7 +103,7 @@ fil_compress_page( byte* lzo_mem) /*!< in: temporal memory used by LZO */ { int err = Z_OK; - int level = 0; + int level = compression_level; ulint header_len = FIL_PAGE_DATA + FIL_PAGE_COMPRESSED_SIZE; ulint write_size=0; /* Cache to avoid change during function execution */ @@ -290,15 +123,11 @@ fil_compress_page( if (orig_page_type == 0 || orig_page_type == FIL_PAGE_TYPE_FSP_HDR || orig_page_type == FIL_PAGE_TYPE_XDES || - orig_page_type == FIL_PAGE_PAGE_COMPRESSED || - orig_page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { + orig_page_type == FIL_PAGE_PAGE_COMPRESSED) { *out_len = len; return (buf); } - level = compression_level; - ut_ad(fil_space_is_page_compressed(space_id)); - fil_system_enter(); fil_space_t* space = fil_space_get_by_id(space_id); fil_system_exit(); @@ -577,16 +406,6 @@ fil_decompress_page( in_buf = page_buf; } - if (ptype == FIL_PAGE_TYPE_COMPRESSED) { - - fil_decompress_page_2(in_buf, buf, len, write_size); - // Need to free temporal buffer if no buffer was given - if (page_buf == NULL) { - ut_free(in_buf); - } - return; - } - /* Before actual decompress, make sure that page type is correct */ if (mach_read_from_4(buf+FIL_PAGE_SPACE_OR_CHKSUM) != BUF_NO_CHECKSUM_MAGIC || diff --git a/storage/xtradb/fsp/fsp0fsp.cc b/storage/xtradb/fsp/fsp0fsp.cc index 0fffe60253e..34acab04241 100644 --- a/storage/xtradb/fsp/fsp0fsp.cc +++ b/storage/xtradb/fsp/fsp0fsp.cc @@ -31,6 +31,7 @@ Created 11/29/1995 Heikki Tuuri #include "buf0buf.h" #include "fil0fil.h" +#include "fil0crypt.h" #include "mtr0log.h" #include "ut0byte.h" #include "page0page.h" diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index 5cc232a3b35..cb6b7d1a263 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -81,6 +81,7 @@ this program; if not, write to the Free Software Foundation, Inc., #include "fsp0fsp.h" #include "sync0sync.h" #include "fil0fil.h" +#include "fil0crypt.h" #include "trx0xa.h" #include "row0merge.h" #include "dict0boot.h" @@ -106,7 +107,6 @@ this program; if not, write to the Free Software Foundation, Inc., #include "fts0priv.h" #include "page0zip.h" #include "fil0pagecompress.h" -#include "fil0pageencryption.h" #define thd_get_trx_isolation(X) ((enum_tx_isolation)thd_tx_isolation(X)) @@ -11557,8 +11557,6 @@ innobase_table_flags( modified by another thread while the table is being created. */ const ulint default_compression_level = page_zip_level; - const ulint default_encryption_key = srv_default_page_encryption_key; - *flags = 0; *flags2 = 0; @@ -11757,10 +11755,7 @@ index_bad: options->page_compressed, options->page_compression_level == 0 ? default_compression_level : options->page_compression_level, - options->atomic_writes, - options->page_encryption, - options->page_encryption_key == 0 ? - default_encryption_key : options->page_encryption_key); + options->atomic_writes); if (create_info->options & HA_LEX_CREATE_TMP_TABLE) { *flags2 |= DICT_TF2_TEMPORARY; @@ -20339,7 +20334,7 @@ static MYSQL_SYSVAR_UINT(default_page_encryption_key, srv_default_page_encryptio "Encryption key used for page encryption.", NULL, NULL, - DEFAULT_ENCRYPTION_KEY, 1, 255, 0); + FIL_DEFAULT_ENCRYPTION_KEY, 1, 255, 0); static MYSQL_SYSVAR_BOOL(scrub_log, srv_scrub_log, PLUGIN_VAR_OPCMDARG | PLUGIN_VAR_READONLY, diff --git a/storage/xtradb/handler/i_s.cc b/storage/xtradb/handler/i_s.cc index 20bc4331bc8..332af275c55 100644 --- a/storage/xtradb/handler/i_s.cc +++ b/storage/xtradb/handler/i_s.cc @@ -72,6 +72,8 @@ Modified Dec 29, 2014 Jan Lindström (Added sys_semaphore_waits) #include "btr0btr.h" #include "page0zip.h" #include "sync0arr.h" +#include "fil0fil.h" +#include "fil0crypt.h" /** structure associates a name string with a file page type and/or buffer page state. */ diff --git a/storage/xtradb/include/dict0dict.h b/storage/xtradb/include/dict0dict.h index ef8c9878297..0d3aac32df9 100644 --- a/storage/xtradb/include/dict0dict.h +++ b/storage/xtradb/include/dict0dict.h @@ -929,10 +929,8 @@ dict_tf_set( pages */ ulint page_compression_level, /*!< in: table page compression level */ - ulint atomic_writes, /*!< in: table atomic + ulint atomic_writes) /*!< in: table atomic writes option value*/ - bool page_encrypted,/*!< in: table uses page encryption */ - ulint page_encryption_key) /*!< in: page encryption key */ __attribute__((nonnull)); /********************************************************************//** Convert a 32 bit integer table flags to the 32 bit integer that is diff --git a/storage/xtradb/include/dict0dict.ic b/storage/xtradb/include/dict0dict.ic index c480b3c6216..e5f4ec21bd2 100644 --- a/storage/xtradb/include/dict0dict.ic +++ b/storage/xtradb/include/dict0dict.ic @@ -543,9 +543,6 @@ dict_tf_is_valid( ulint data_dir = DICT_TF_HAS_DATA_DIR(flags); ulint atomic_writes = DICT_TF_GET_ATOMIC_WRITES(flags); - ulint page_encryption = DICT_TF_GET_PAGE_ENCRYPTION(flags); - ulint page_encryption_key = DICT_TF_GET_PAGE_ENCRYPTION_KEY(flags); - /* Make sure there are no bits that we do not know about. */ if (unused != 0) { @@ -556,12 +553,10 @@ dict_tf_is_valid( "InnoDB: compact %ld atomic_blobs %ld\n" "InnoDB: unused %ld data_dir %ld zip_ssize %ld\n" "InnoDB: page_compression %ld page_compression_level %ld\n" - "InnoDB: atomic_writes %ld\n" - "InnoDB: page_encryption %ld page_encryption_key %ld\n", + "InnoDB: atomic_writes %ld\n", unused, compact, atomic_blobs, unused, data_dir, zip_ssize, - page_compression, page_compression_level, atomic_writes, - page_encryption, page_encryption_key + page_compression, page_compression_level, atomic_writes ); return(false); @@ -861,9 +856,7 @@ dict_tf_set( pages */ ulint page_compression_level, /*!< in: table page compression level */ - ulint atomic_writes, /*!< in: table atomic writes setup */ - bool page_encrypted, /*!< in: table uses page encryption */ - ulint page_encryption_key /*!< in: page encryption key */) + ulint atomic_writes) /*!< in: table atomic writes setup */ { atomic_writes_t awrites = (atomic_writes_t)atomic_writes; @@ -904,11 +897,6 @@ dict_tf_set( *flags |= (atomic_writes << DICT_TF_POS_ATOMIC_WRITES); ut_a(dict_tf_get_atomic_writes(*flags) == awrites); - - if (page_encrypted) { - *flags |= (1 << DICT_TF_POS_PAGE_ENCRYPTION) - | (page_encryption_key << DICT_TF_POS_PAGE_ENCRYPTION_KEY); - } } /********************************************************************//** @@ -931,11 +919,6 @@ dict_tf_to_fsp_flags( ulint fsp_flags; ulint page_compression = DICT_TF_GET_PAGE_COMPRESSION(table_flags); ulint page_compression_level = DICT_TF_GET_PAGE_COMPRESSION_LEVEL(table_flags); - - ulint page_encryption = DICT_TF_GET_PAGE_ENCRYPTION(table_flags); - /* Keys are limited to 255 values */ - ulint page_encryption_key = DICT_TF_GET_PAGE_ENCRYPTION_KEY(table_flags); - ulint atomic_writes = DICT_TF_GET_ATOMIC_WRITES(table_flags); DBUG_EXECUTE_IF("dict_tf_to_fsp_flags_failure", @@ -963,14 +946,6 @@ dict_tf_to_fsp_flags( if page compression is used for this table. */ fsp_flags |= FSP_FLAGS_SET_PAGE_COMPRESSION_LEVEL(fsp_flags, page_compression_level); - /* In addition, tablespace flags also contain if the page - encryption is used for this table. */ - fsp_flags |= FSP_FLAGS_SET_PAGE_ENCRYPTION(fsp_flags, page_encryption); - - /* In addition, tablespace flags also contain page encryption key if the page - encryption is used for this table. */ - fsp_flags |= FSP_FLAGS_SET_PAGE_ENCRYPTION_KEY(fsp_flags, page_encryption_key); - /* In addition, tablespace flags also contain flag if atomic writes is used for this table */ fsp_flags |= FSP_FLAGS_SET_ATOMIC_WRITES(fsp_flags, atomic_writes); @@ -1012,8 +987,6 @@ dict_sys_tables_type_to_tf( | DICT_TF_MASK_PAGE_COMPRESSION | DICT_TF_MASK_PAGE_COMPRESSION_LEVEL | DICT_TF_MASK_ATOMIC_WRITES - | DICT_TF_MASK_PAGE_ENCRYPTION - | DICT_TF_MASK_PAGE_ENCRYPTION_KEY ); @@ -1050,9 +1023,7 @@ dict_tf_to_sys_tables_type( | DICT_TF_MASK_DATA_DIR | DICT_TF_MASK_PAGE_COMPRESSION | DICT_TF_MASK_PAGE_COMPRESSION_LEVEL - | DICT_TF_MASK_ATOMIC_WRITES - | DICT_TF_MASK_PAGE_ENCRYPTION - | DICT_TF_MASK_PAGE_ENCRYPTION_KEY); + | DICT_TF_MASK_ATOMIC_WRITES); return(type); } diff --git a/storage/xtradb/include/dict0pagecompress.ic b/storage/xtradb/include/dict0pagecompress.ic index a71b2b34b07..811976434a8 100644 --- a/storage/xtradb/include/dict0pagecompress.ic +++ b/storage/xtradb/include/dict0pagecompress.ic @@ -42,8 +42,6 @@ dict_tf_verify_flags( ulint page_compression = DICT_TF_GET_PAGE_COMPRESSION(table_flags); ulint page_compression_level = DICT_TF_GET_PAGE_COMPRESSION_LEVEL(table_flags); ulint atomic_writes = DICT_TF_GET_ATOMIC_WRITES(table_flags); - ulint page_encryption = DICT_TF_GET_PAGE_ENCRYPTION(table_flags); - ulint page_encryption_key = DICT_TF_GET_PAGE_ENCRYPTION_KEY(table_flags); ulint post_antelope = FSP_FLAGS_GET_POST_ANTELOPE(fsp_flags); ulint zip_ssize = FSP_FLAGS_GET_ZIP_SSIZE(fsp_flags); ulint fsp_atomic_blobs = FSP_FLAGS_HAS_ATOMIC_BLOBS(fsp_flags); @@ -52,9 +50,6 @@ dict_tf_verify_flags( ulint fsp_page_compression = FSP_FLAGS_GET_PAGE_COMPRESSION(fsp_flags); ulint fsp_page_compression_level = FSP_FLAGS_GET_PAGE_COMPRESSION_LEVEL(fsp_flags); ulint fsp_atomic_writes = FSP_FLAGS_GET_ATOMIC_WRITES(fsp_flags); - ulint fsp_page_encryption = FSP_FLAGS_GET_PAGE_ENCRYPTION(fsp_flags); - ulint fsp_page_encryption_key = FSP_FLAGS_GET_PAGE_ENCRYPTION_KEY(fsp_flags); - DBUG_EXECUTE_IF("dict_tf_verify_flags_failure", return(ULINT_UNDEFINED);); @@ -112,27 +107,6 @@ dict_tf_verify_flags( return (FALSE); } - if (page_encryption != fsp_page_encryption) { - fprintf(stderr, - "InnoDB: Error: table flags has page_encryption %ld" - " in the data dictionary\n" - "InnoDB: but the flags in file has page_encryption %ld\n", - page_encryption, fsp_page_encryption); - - return (FALSE); - } - - if (page_encryption_key != fsp_page_encryption_key) { - fprintf(stderr, - "InnoDB: Error: table flags has page_encryption_key %ld" - " in the data dictionary\n" - "InnoDB: but the flags in file has page_encryption_key %ld\n", - page_encryption_key, fsp_page_encryption_key); - - return (FALSE); - } - - return(TRUE); } diff --git a/storage/xtradb/include/fil0crypt.h b/storage/xtradb/include/fil0crypt.h new file mode 100644 index 00000000000..9d02034e4b2 --- /dev/null +++ b/storage/xtradb/include/fil0crypt.h @@ -0,0 +1,394 @@ +/***************************************************************************** + +Copyright (c) 2015, MariaDB Corporation. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA + +*****************************************************************************/ + +/**************************************************//** +@file include/fil0crypt.h +The low-level file system encryption support functions + +Created 04/01/2015 Jan Lindström +*******************************************************/ + +#ifndef fil0crypt_h +#define fil0crypt_h + +#define FIL_MSG_CANNOT_DECRYPT "can not decrypt" +#define FIL_ENCRYPTION_WRONG_KEY 1 +#define FIL_ENCRYPTION_WRONG_PAGE_TYPE 2 +#define FIL_ENCRYPTION_ERROR 3 +#define FIL_ENCRYPTION_KEY_MISSING 4 +#define FIL_ENCRYPTION_OK 0 +#define FIL_ENCRYPTION_WILL_NOT_ENCRYPT 5 + +/* This key will be used if nothing else is given */ +#define FIL_DEFAULT_ENCRYPTION_KEY 1 + +/** + * CRYPT_SCHEME_UNENCRYPTED + * + * Used as intermediate state when convering a space from unencrypted + * to encrypted + */ +/** + * CRYPT_SCHEME_1 + * + * AES_CTR / AES_CBC: + * L = AES_ECB(KEY, IV) + * CRYPT(PAGE) = AES_CRT(KEY=L, IV=C, PAGE) + */ + +#define CRYPT_SCHEME_1 1 +#define CRYPT_SCHEME_1_IV_LEN 16 +#define CRYPT_SCHEME_UNENCRYPTED 0 + +/* Currently supported encryption methods */ +typedef enum { + CRYPT_SCHEME_1_UNENCRYPTED = 0, /*keys[0].key_version)) { + /* accessing table would surely fail, because no key or no key provider available */ + return FIL_ENCRYPTION_KEY_MISSING; + } + } + } else { + ulint key = mach_read_from_4(buf + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); + if (!has_encryption_key(key)) { + return FIL_ENCRYPTION_KEY_MISSING; + } + } + return 0; +} diff --git a/storage/xtradb/include/fil0fil.h b/storage/xtradb/include/fil0fil.h index 2e92c2b09b2..91653c4bb7d 100644 --- a/storage/xtradb/include/fil0fil.h +++ b/storage/xtradb/include/fil0fil.h @@ -26,7 +26,6 @@ Created 10/25/1995 Heikki Tuuri #ifndef fil0fil_h #define fil0fil_h -#define MSG_CANNOT_DECRYPT "can not decrypt" #include "univ.i" #ifndef UNIV_INNOCHECKSUM @@ -132,24 +131,6 @@ extern fil_addr_t fil_addr_null; used to encrypt the page + 32-bit checksum or 64 bits of zero if no encryption */ -/** If page type is FIL_PAGE_COMPRESSED then the 8 bytes starting at -FIL_PAGE_FILE_FLUSH_LSN are broken down as follows: */ - -/** Control information version format (u8) */ -static const ulint FIL_PAGE_VERSION = FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION; - -/** Compression algorithm (u8) */ -static const ulint FIL_PAGE_ALGORITHM_V1 = FIL_PAGE_VERSION + 1; - -/** Original page type (u16) */ -static const ulint FIL_PAGE_ORIGINAL_TYPE_V1 = FIL_PAGE_ALGORITHM_V1 + 1; - -/** Original data size in bytes (u16)*/ -static const ulint FIL_PAGE_ORIGINAL_SIZE_V1 = FIL_PAGE_ORIGINAL_TYPE_V1 + 2; - -/** Size after compression (u16)*/ -static const ulint FIL_PAGE_COMPRESS_SIZE_V1 = FIL_PAGE_ORIGINAL_SIZE_V1 + 2; - #define FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID 34 /*!< starting from 4.1.x this contains the space id of the page */ #define FIL_PAGE_SPACE_ID FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID @@ -169,10 +150,7 @@ static const ulint FIL_PAGE_COMPRESS_SIZE_V1 = FIL_PAGE_ORIGINAL_SIZE_V1 + 2; /* @} */ /** File page types (values of FIL_PAGE_TYPE) @{ */ -#define FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED 35631 /* page compressed + - encrypted page */ #define FIL_PAGE_PAGE_COMPRESSED 34354 /*!< Page compressed page */ -#define FIL_PAGE_PAGE_ENCRYPTED 34355 /*!< Page encrypted page */ #define FIL_PAGE_INDEX 17855 /*!< B-tree node */ #define FIL_PAGE_UNDO_LOG 2 /*!< Undo log page */ #define FIL_PAGE_INODE 3 /*!< Index node */ @@ -199,6 +177,9 @@ static const ulint FIL_PAGE_COMPRESS_SIZE_V1 = FIL_PAGE_ORIGINAL_SIZE_V1 + 2; #define FIL_LOG 502 /*!< redo log */ /* @} */ +/* structure containing encryption specification */ +typedef struct fil_space_crypt_struct fil_space_crypt_t; + /** The number of fsyncs done to the log */ extern ulint fil_n_log_flushes; @@ -210,9 +191,6 @@ extern ulint fil_n_pending_tablespace_flushes; /** Number of files currently open */ extern ulint fil_n_file_opened; -/* structure containing encryption specification */ -typedef struct fil_space_crypt_struct fil_space_crypt_t; - struct fsp_open_info { ibool success; /*!< Has the tablespace been opened? */ const char* check_msg; /*!< fil_check_first_page() message */ @@ -963,9 +941,6 @@ fil_space_get_n_reserved_extents( Reads or writes data. This operation is asynchronous (aio). @return DB_SUCCESS, or DB_TABLESPACE_DELETED if we are trying to do i/o on a tablespace which does not exist */ -#define fil_io(type, sync, space_id, zip_size, block_offset, byte_offset, len, buf, message, write_size, lsn, encrypt) \ - _fil_io(type, sync, space_id, zip_size, block_offset, byte_offset, len, buf, message, write_size, NULL, lsn, encrypt) - UNIV_INTERN dberr_t _fil_io( @@ -1000,11 +975,14 @@ _fil_io( operation for this page and if initialized we do not trim again if actual page size does not decrease. */ - trx_t* trx, - lsn_t lsn, /*!< in: lsn of the newest modification */ - bool encrypt_later) /*!< in: encrypt later ? */ + trx_t* trx) /*!< in: trx */ __attribute__((nonnull(8))); + +#define fil_io(type, sync, space_id, zip_size, block_offset, byte_offset, len, buf, message, write_size) \ + _fil_io(type, sync, space_id, zip_size, block_offset, byte_offset, len, buf, message, write_size, NULL) + + /**********************************************************************//** Waits for an aio operation to complete. This function is used to write the handler for completed requests. The aio array of pending requests is divided @@ -1298,12 +1276,6 @@ fil_space_t* fil_space_get_by_id( /*================*/ ulint id); /*!< in: space id */ -/*******************************************************************//** -Return space name */ -char* -fil_space_name( -/*===========*/ - fil_space_t* space); /*!< in: space */ /****************************************************************** Get id of first tablespace or ULINT_UNDEFINED if none */ @@ -1318,253 +1290,16 @@ ulint fil_get_next_space( ulint id); /*!< in: space id */ -/********************************************************************* -Init global resources needed for tablespace encryption/decryption */ -void -fil_space_crypt_init(); - -/********************************************************************* -Cleanup global resources needed for tablespace encryption/decryption */ -void -fil_space_crypt_cleanup(); - -/********************************************************************* -Create crypt data, i.e data that is used for a single tablespace */ -fil_space_crypt_t * -fil_space_create_crypt_data(); - -/********************************************************************* -Destroy crypt data */ -UNIV_INTERN -void -fil_space_destroy_crypt_data( -/*=========================*/ - fil_space_crypt_t **crypt_data); /*!< in/out: crypt data */ - -/********************************************************************* -Get crypt data for a space*/ -fil_space_crypt_t * -fil_space_get_crypt_data( -/*======================*/ - ulint space); /*!< in: tablespace id */ - -/********************************************************************* -Set crypt data for a space*/ -void -fil_space_set_crypt_data( -/*======================*/ - ulint space, /*!< in: tablespace id */ - fil_space_crypt_t* crypt_data); /*!< in: crypt data */ - -/********************************************************************* -Compare crypt data*/ -int -fil_space_crypt_compare( -/*======================*/ - const fil_space_crypt_t* crypt_data1, /*!< in: crypt data */ - const fil_space_crypt_t* crypt_data2); /*!< in: crypt data */ - -/********************************************************************* -Read crypt data from buffer page */ -fil_space_crypt_t * -fil_space_read_crypt_data( -/*======================*/ - ulint space, /*!< in: tablespace id */ - const byte* page, /*!< in: buffer page */ - ulint offset); /*!< in: offset where crypt data is stored */ - -/********************************************************************* -Write crypt data to buffer page */ -void -fil_space_write_crypt_data( -/*=======================*/ - ulint space, /*!< in: tablespace id */ - byte* page, /*!< in: buffer page */ - ulint offset, /*!< in: offset where to store data */ - ulint maxsize, /*!< in: max space available to store crypt data in */ - mtr_t * mtr); /*!< in: mini-transaction */ - -/********************************************************************* -Clear crypt data from page 0 (used for import tablespace) */ -void -fil_space_clear_crypt_data( -/*======================*/ - byte* page, /*!< in: buffer page */ - ulint offset); /*!< in: offset where crypt data is stored */ - -/********************************************************************* -Parse crypt data log record */ -byte* -fil_parse_write_crypt_data( -/*=======================*/ - byte* ptr, /*!< in: start of log record */ - byte* end_ptr, /*!< in: end of log record */ - buf_block_t*); /*!< in: buffer page to apply record to */ - -/********************************************************************* -Check if extra buffer shall be allocated for decrypting after read */ -UNIV_INTERN -bool -fil_space_check_encryption_read( -/*==============================*/ - ulint space); /*!< in: tablespace id */ - -/********************************************************************* -Check if page shall be encrypted before write */ -UNIV_INTERN -bool -fil_space_check_encryption_write( -/*==============================*/ - ulint space); /*!< in: tablespace id */ - -/********************************************************************* -Encrypt buffer page */ -void -fil_space_encrypt( -/*===============*/ - ulint space, /*!< in: tablespace id */ - ulint offset, /*!< in: page no */ - lsn_t lsn, /*!< in: page lsn */ - const byte* src_frame,/*!< in: page frame */ - ulint size, /*!< in: size of data to encrypt */ - byte* dst_frame, /*!< in: where to encrypt to */ - ulint page_encryption_key); /*!< in: page encryption key id if page - encrypted */ - -/********************************************************************* -Decrypt buffer page */ -void -fil_space_decrypt( -/*===============*/ - ulint space, /*!< in: tablespace id */ - const byte* src_frame,/*!< in: page frame */ - ulint page_size, /*!< in: size of data to encrypt */ - byte* dst_frame); /*!< in: where to decrypt to */ - - -/********************************************************************* -Decrypt buffer page -@return true if page was encrypted */ -bool -fil_space_decrypt( -/*===============*/ - fil_space_crypt_t* crypt_data, /*!< in: crypt data */ - const byte* src_frame,/*!< in: page frame */ - ulint page_size, /*!< in: page size */ - byte* dst_frame); /*!< in: where to decrypt to */ - -/********************************************************************* -fil_space_verify_crypt_checksum -NOTE: currently this function can only be run in single threaded mode -as it modifies srv_checksum_algorithm (temporarily) -@return true if page is encrypted AND OK, false otherwise */ -bool -fil_space_verify_crypt_checksum( -/*===============*/ - const byte* src_frame,/*!< in: page frame */ - ulint zip_size); /*!< in: size of data to encrypt */ - -/********************************************************************* -Init threads for key rotation */ -void -fil_crypt_threads_init(); - -/********************************************************************* -Set thread count (e.g start or stops threads) used for key rotation */ -void -fil_crypt_set_thread_cnt( -/*=====================*/ - uint new_cnt); /*!< in: requested #threads */ - -/********************************************************************* -End threads for key rotation */ -void -fil_crypt_threads_end(); - -/********************************************************************* -Cleanup resources for threads for key rotation */ -void -fil_crypt_threads_cleanup(); - -/********************************************************************* -Set rotate key age */ -void -fil_crypt_set_rotate_key_age( -/*=====================*/ - uint rotate_age); /*!< in: requested rotate age */ - -/********************************************************************* -Set rotation threads iops */ -void -fil_crypt_set_rotation_iops( -/*=====================*/ - uint iops); /*!< in: requested iops */ - -/********************************************************************* -Mark a space as closing */ -UNIV_INTERN -void -fil_space_crypt_mark_space_closing( -/*===============*/ - ulint space); /*!< in: tablespace id */ - -/********************************************************************* -Wait for crypt threads to stop accessing space */ -UNIV_INTERN -void -fil_space_crypt_close_tablespace( -/*===============*/ - ulint space); /*!< in: tablespace id */ - -/** Struct for retreiving info about encryption */ -struct fil_space_crypt_status_t { - ulint space; /*!< tablespace id */ - ulint scheme; /*!< encryption scheme */ - uint min_key_version; /*!< min key version */ - uint current_key_version;/*!< current key version */ - uint keyserver_requests;/*!< no of key requests to key server */ - bool rotating; /*!< is key rotation ongoing */ - bool flushing; /*!< is flush at end of rotation ongoing */ - ulint rotate_next_page_number; /*!< next page if key rotating */ - ulint rotate_max_page_number; /*!< max page if key rotating */ -}; - -/********************************************************************* -Get crypt status for a space -@return 0 if crypt data found */ -int -fil_space_crypt_get_status( -/*==================*/ - ulint id, /*!< in: space id */ - struct fil_space_crypt_status_t * status); /*!< out: status */ - -/** Struct for retreiving statistics about encryption key rotation */ -struct fil_crypt_stat_t { - ulint pages_read_from_cache; - ulint pages_read_from_disk; - ulint pages_modified; - ulint pages_flushed; - ulint estimated_iops; -}; - -/********************************************************************* -Get crypt rotation statistics */ -void -fil_crypt_total_stat( -/*==================*/ - fil_crypt_stat_t* stat); /*!< out: crypt stat */ - #endif /*******************************************************************//** Return space flags */ +UNIV_INLINE ulint fil_space_flags( /*===========*/ fil_space_t* space); /*!< in: space */ - - /****************************************************************//** Does error handling when a file operation fails. @return TRUE if we should retry the operation */ @@ -1580,30 +1315,14 @@ os_file_handle_error_no_exit( /*******************************************************************//** Return page type name */ +UNIV_INLINE const char* fil_get_page_type_name( /*===================*/ ulint page_type); /*!< in: FIL_PAGE_TYPE */ -/** Struct for retreiving info about scrubbing */ -struct fil_space_scrub_status_t { - ulint space; /*!< tablespace id */ - bool compressed; /*!< is space compressed */ - time_t last_scrub_completed; /*!< when was last scrub completed */ - bool scrubbing; /*!< is scrubbing ongoing */ - time_t current_scrub_started; /*!< when started current scrubbing */ - ulint current_scrub_active_threads; /*!< current scrub active threads */ - ulint current_scrub_page_number; /*!< current scrub page no */ - ulint current_scrub_max_page_number; /*!< current scrub max page no */ -}; - -/********************************************************************* -Get scrub status for a space -@return 0 if no scrub info found */ -int -fil_space_get_scrub_status( -/*==================*/ - ulint id, /*!< in: space id */ - struct fil_space_scrub_status_t * status); /*!< out: status */ +#ifndef UNIV_NONINL +#include "fil0fil.ic" +#endif #endif /* fil0fil_h */ diff --git a/storage/xtradb/include/fil0fil.ic b/storage/xtradb/include/fil0fil.ic new file mode 100644 index 00000000000..b1e65e6dddb --- /dev/null +++ b/storage/xtradb/include/fil0fil.ic @@ -0,0 +1,108 @@ +/***************************************************************************** + +Copyright (c) 2015, MariaDB Corporation. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA + +*****************************************************************************/ + +/**************************************************//** +@file include/fil0fil.ic +The low-level file system support functions + +Created 31/03/2015 Jan Lindström +*******************************************************/ + +#ifndef fil0fil_ic +#define fil0fil_ic + +/*******************************************************************//** +Return space name */ +UNIV_INLINE +char* +fil_space_name( +/*===========*/ + fil_space_t* space) /*!< in: space */ +{ + return (space->name); +} + +/*******************************************************************//** +Return space flags */ +UNIV_INLINE +ulint +fil_space_flags( +/*===========*/ + fil_space_t* space) /*!< in: space */ +{ + return (space->flags); +} + +/*******************************************************************//** +Return page type name */ +UNIV_INLINE +const char* +fil_get_page_type_name( +/*===================*/ + ulint page_type) /*!< in: FIL_PAGE_TYPE */ +{ + switch(page_type) { + case FIL_PAGE_PAGE_COMPRESSED: + return (const char*)"PAGE_COMPRESSED"; + case FIL_PAGE_INDEX: + return (const char*)"INDEX"; + case FIL_PAGE_UNDO_LOG: + return (const char*)"UNDO LOG"; + case FIL_PAGE_INODE: + return (const char*)"INODE"; + case FIL_PAGE_IBUF_FREE_LIST: + return (const char*)"IBUF_FREE_LIST"; + case FIL_PAGE_TYPE_ALLOCATED: + return (const char*)"ALLOCATED"; + case FIL_PAGE_IBUF_BITMAP: + return (const char*)"IBUF_BITMAP"; + case FIL_PAGE_TYPE_SYS: + return (const char*)"SYS"; + case FIL_PAGE_TYPE_TRX_SYS: + return (const char*)"TRX_SYS"; + case FIL_PAGE_TYPE_FSP_HDR: + return (const char*)"FSP_HDR"; + case FIL_PAGE_TYPE_XDES: + return (const char*)"XDES"; + case FIL_PAGE_TYPE_BLOB: + return (const char*)"BLOB"; + case FIL_PAGE_TYPE_ZBLOB: + return (const char*)"ZBLOB"; + case FIL_PAGE_TYPE_ZBLOB2: + return (const char*)"ZBLOB2"; + case FIL_PAGE_TYPE_COMPRESSED: + return (const char*)"ORACLE PAGE COMPRESSED"; + default: + return (const char*)"PAGE TYPE CORRUPTED"; + } +} + +/****************************************************************//** +Get block size from fil node +@return block size*/ +UNIV_INLINE +ulint +fil_node_get_block_size( +/*====================*/ + fil_node_t* node) /*!< in: Node where to get block + size */ +{ + return (node->file_block_size); +} + +#endif /* fil0fil_ic */ diff --git a/storage/xtradb/include/fil0pageencryption.h b/storage/xtradb/include/fil0pageencryption.h deleted file mode 100644 index fa2b1a5c592..00000000000 --- a/storage/xtradb/include/fil0pageencryption.h +++ /dev/null @@ -1,79 +0,0 @@ -/***************************************************************************** - -Copyright (C) 2014 eperi GmbH. All Rights Reserved. - -This program is free software; you can redistribute it and/or modify it under -the terms of the GNU General Public License as published by the Free Software -Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, but WITHOUT -ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - -You should have received a copy of the GNU General Public License along with -this program; if not, write to the Free Software Foundation, Inc., -51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - -*****************************************************************************/ - -#ifndef fil0pageencryption_h -#define fil0pageencryption_h - -#define PAGE_ENCRYPTION_WRONG_KEY 1 -#define PAGE_ENCRYPTION_WRONG_PAGE_TYPE 2 -#define PAGE_ENCRYPTION_ERROR 3 -#define PAGE_ENCRYPTION_KEY_MISSING 4 -#define PAGE_ENCRYPTION_OK 0 -#define PAGE_ENCRYPTION_WILL_NOT_ENCRYPT 5 - -/* This key will be used if nothing else is given */ -#define DEFAULT_ENCRYPTION_KEY 1 - -#include "fsp0fsp.h" -#include "fsp0pageencryption.h" - -/******************************************************************//** -@file include/fil0pageencryption.h -Helper functions for encryption/decryption page data on to table space. - -Created 08/25/2014 -***********************************************************************/ - -/*******************************************************************//** -Find out whether the page is page encrypted. -Returns the page encryption flag of the space, or false if the space -is not encrypted. The tablespace must be cached in the memory cache. -@return true if page encrypted, false if not or space not found */ -ibool -fil_space_is_page_encrypted( -/*========================*/ - ulint id); /*!< in: space id */ - -/*******************************************************************//** -Find out whether the page is page encrypted -@return true if page is page encrypted, false if not */ -UNIV_INLINE -ibool -fil_page_is_encrypted( -/*==================*/ - const byte *buf); /*!< in: page */ -/*******************************************************************//** -Find out whether the page is page compressed and then encrypted -@return true if page is page compressed+encrypted, false if not */ -UNIV_INLINE -ibool -fil_page_is_compressed_encrypted( -/*=============================*/ - const byte *buf); /*!< in: page */ - -/*******************************************************************//** -Find out whether the page can be decrypted -@return true if page can be decrypted, false if not. */ -UNIV_INLINE -ulint -fil_page_encryption_status( -/*=======================*/ - const byte *buf); /*!< in: page */ - - -#endif // fil0pageencryption_h diff --git a/storage/xtradb/include/fsp0fsp.h b/storage/xtradb/include/fsp0fsp.h index 57e5b9490b1..3d3d482a6c0 100644 --- a/storage/xtradb/include/fsp0fsp.h +++ b/storage/xtradb/include/fsp0fsp.h @@ -58,10 +58,6 @@ is found in a remote location, not the default data directory. */ #define FSP_FLAGS_WIDTH_PAGE_COMPRESSION 1 #define FSP_FLAGS_WIDTH_PAGE_COMPRESSION_LEVEL 4 -/** Number of flag bits used to indicate the page compression and compression level */ -#define FSP_FLAGS_WIDTH_PAGE_ENCRYPTION 1 -#define FSP_FLAGS_WIDTH_PAGE_ENCRYPTION_KEY 8 - /** Number of flag bits used to indicate atomic writes for this tablespace */ #define FSP_FLAGS_WIDTH_ATOMIC_WRITES 2 @@ -73,9 +69,7 @@ is found in a remote location, not the default data directory. */ + FSP_FLAGS_WIDTH_DATA_DIR \ + FSP_FLAGS_WIDTH_PAGE_COMPRESSION \ + FSP_FLAGS_WIDTH_PAGE_COMPRESSION_LEVEL \ - + FSP_FLAGS_WIDTH_ATOMIC_WRITES \ - + FSP_FLAGS_WIDTH_PAGE_ENCRYPTION \ - + FSP_FLAGS_WIDTH_PAGE_ENCRYPTION_KEY) + + FSP_FLAGS_WIDTH_ATOMIC_WRITES ) /** A mask of all the known/used bits in tablespace flags */ #define FSP_FLAGS_MASK (~(~0 << FSP_FLAGS_WIDTH)) @@ -99,15 +93,9 @@ dictionary */ /** Zero relative shift position of the ATOMIC_WRITES field */ #define FSP_FLAGS_POS_ATOMIC_WRITES (FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL \ + FSP_FLAGS_WIDTH_PAGE_COMPRESSION_LEVEL) -/** Zero relative shift position of the PAGE_ENCRYPTION field */ -#define FSP_FLAGS_POS_PAGE_ENCRYPTION (FSP_FLAGS_POS_ATOMIC_WRITES \ - + FSP_FLAGS_WIDTH_ATOMIC_WRITES) -/** Zero relative shift position of the PAGE_ENCRYPTION_KEY field */ -#define FSP_FLAGS_POS_PAGE_ENCRYPTION_KEY (FSP_FLAGS_POS_PAGE_ENCRYPTION \ - + FSP_FLAGS_WIDTH_PAGE_ENCRYPTION) - /** Zero relative shift position of the PAGE_SSIZE field */ -#define FSP_FLAGS_POS_PAGE_SSIZE (FSP_FLAGS_POS_PAGE_ENCRYPTION_KEY \ - + FSP_FLAGS_WIDTH_PAGE_ENCRYPTION_KEY) +/** Zero relative shift position of the PAGE_SSIZE field */ +#define FSP_FLAGS_POS_PAGE_SSIZE (FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL \ + + FSP_FLAGS_WIDTH_PAGE_COMPRESSION_LEVEL) /** Zero relative shift position of the start of the DATA DIR bits */ #define FSP_FLAGS_POS_DATA_DIR (FSP_FLAGS_POS_PAGE_SSIZE \ + FSP_FLAGS_WIDTH_PAGE_SSIZE) @@ -143,15 +131,6 @@ dictionary */ #define FSP_FLAGS_MASK_PAGE_COMPRESSION_LEVEL \ ((~(~0 << FSP_FLAGS_WIDTH_PAGE_COMPRESSION_LEVEL)) \ << FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL) -/** Bit mask of the PAGE_ENCRYPTION field */ -#define FSP_FLAGS_MASK_PAGE_ENCRYPTION \ - ((~(~0 << FSP_FLAGS_WIDTH_PAGE_ENCRYPTION)) \ - << FSP_FLAGS_POS_PAGE_ENCRYPTION) -/** Bit mask of the PAGE_ENCRYPTION_KEY field */ -#define FSP_FLAGS_MASK_PAGE_ENCRYPTION_KEY \ - ((~(~0 << FSP_FLAGS_WIDTH_PAGE_ENCRYPTION_KEY)) \ - << FSP_FLAGS_POS_PAGE_ENCRYPTION_KEY) - /** Bit mask of the ATOMIC_WRITES field */ #define FSP_FLAGS_MASK_ATOMIC_WRITES \ ((~(~0 << FSP_FLAGS_WIDTH_ATOMIC_WRITES)) \ @@ -192,14 +171,6 @@ dictionary */ #define FSP_FLAGS_GET_ATOMIC_WRITES(flags) \ ((flags & FSP_FLAGS_MASK_ATOMIC_WRITES) \ >> FSP_FLAGS_POS_ATOMIC_WRITES) -/** Return the value of the PAGE_ENCRYPTION field */ -#define FSP_FLAGS_GET_PAGE_ENCRYPTION(flags) \ - ((flags & FSP_FLAGS_MASK_PAGE_ENCRYPTION) \ - >> FSP_FLAGS_POS_PAGE_ENCRYPTION) -/** Return the value of the PAGE_ENCRYPTION_KEY field */ -#define FSP_FLAGS_GET_PAGE_ENCRYPTION_KEY(flags) \ - ((flags & FSP_FLAGS_MASK_PAGE_ENCRYPTION_KEY) \ - >> FSP_FLAGS_POS_PAGE_ENCRYPTION_KEY) /** Set a PAGE_SSIZE into the correct bits in a given tablespace flags. */ @@ -216,13 +187,6 @@ tablespace flags. */ #define FSP_FLAGS_SET_PAGE_COMPRESSION_LEVEL(flags, level) \ (flags | (level << FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL)) -/** Set a PAGE_ENCRYPTION into the correct bits in a given tablespace flags. */ -#define FSP_FLAGS_SET_PAGE_ENCRYPTION(flags, encryption) \ - (flags | (encryption << FSP_FLAGS_POS_PAGE_ENCRYPTION)) -/** Set a PAGE_ENCRYPTION_KEY into the correct bits in a given tablespace flags. */ -#define FSP_FLAGS_SET_PAGE_ENCRYPTION_KEY(flags, encryption_key) \ - (flags | (encryption_key << FSP_FLAGS_POS_PAGE_ENCRYPTION_KEY)) - /** Set a ATOMIC_WRITES into the correct bits in a given tablespace flags. */ #define FSP_FLAGS_SET_ATOMIC_WRITES(flags, atomics) \ diff --git a/storage/xtradb/include/fsp0pageencryption.h b/storage/xtradb/include/fsp0pageencryption.h deleted file mode 100644 index 631aa72211c..00000000000 --- a/storage/xtradb/include/fsp0pageencryption.h +++ /dev/null @@ -1,66 +0,0 @@ -/***************************************************************************** - - Copyright (C) 2014 eperi GmbH. All Rights Reserved. - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; version 2 of the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ - -/******************************************************************/ - -/******************************************************************//** -@file include/fsp0pageencryption.h -Helper functions for extracting/storing page encryption information to file space. - -Created 08/28/2014 -***********************************************************************/ - -#ifndef FSP0PAGEENCRYPTION_H_ -#define FSP0PAGEENCRYPTION_H_ - -#define FIL_PAGE_ENCRYPTION_AES_128 16 /*!< Encryption algorithm AES-128. */ -#define FIL_PAGE_ENCRYPTION_AES_196 24 /*!< Encryption algorithm AES-196. */ -#define FIL_PAGE_ENCRYPTION_AES_256 32 /*!< Encryption algorithm AES-256. */ - -#define FIL_PAGE_ENCRYPTED_SIZE 2 /*!< Number of bytes used to store - actual payload data size onencrypted - pages. */ - -/********************************************************************//** -Determine if the tablespace is page encrypted from dict_table_t::flags. -@return TRUE if page encrypted, FALSE if not page encrypted */ -UNIV_INLINE -ibool -fsp_flags_is_page_encrypted( -/*=========================*/ - ulint flags); /*!< in: tablespace flags */ - - -/********************************************************************//** -Extract the page encryption key from tablespace flags. -A tablespace has only one physical page encryption key -whether that page is encrypted or not. -@return page encryption key of the file-per-table tablespace, -or zero if the table is not encrypted. */ -UNIV_INLINE -ulint -fsp_flags_get_page_encryption_key( -/*=================================*/ - ulint flags); /*!< in: tablespace flags */ - - -#ifndef UNIV_NONINL -#include "fsp0pageencryption.ic" -#endif - - -#endif /* FSP0PAGEENCRYPTION_H_ */ diff --git a/storage/xtradb/include/fsp0pageencryption.ic b/storage/xtradb/include/fsp0pageencryption.ic deleted file mode 100644 index d3137001fc5..00000000000 --- a/storage/xtradb/include/fsp0pageencryption.ic +++ /dev/null @@ -1,168 +0,0 @@ -/***************************************************************************** - - Copyright (C) 2014 eperi GmbH. All Rights Reserved. - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; version 2 of the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ - -/******************************************************************//** -@file include/fsp0pageencryption.ic -Implementation for helper functions for encrypting/decrypting pages -and atomic writes information to file space. - -Created 08/28/2014 -***********************************************************************/ - -#include "fsp0fsp.h" -#include "fil0pageencryption.h" - -/********************************************************************//** -Determine if the tablespace is page encrypted from dict_table_t::flags. -@return TRUE if page encrypted, FALSE if not page encrypted */ -UNIV_INLINE -ibool -fsp_flags_is_page_encrypted( -/*=========================*/ - ulint flags) /*!< in: tablespace flags */ -{ - return(FSP_FLAGS_GET_PAGE_ENCRYPTION(flags)); -} - -/********************************************************************//** -Extract the page encryption key from tablespace flags. -A tablespace has only one physical page encryption key -whether that page is encrypted or not. -@return page encryption key of the file-per-table tablespace, -or zero if the table is not encrypted. */ -UNIV_INLINE -ulint -fsp_flags_get_page_encryption_key( -/*=================================*/ - ulint flags) /*!< in: tablespace flags */ -{ - return(FSP_FLAGS_GET_PAGE_ENCRYPTION_KEY(flags)); -} - - -/*******************************************************************//** -Returns the page encryption flag of the space, or false if the space -is not encrypted. The tablespace must be cached in the memory cache. -@return true if page encrypted, false if not or space not found */ -UNIV_INLINE -ibool -fil_space_is_page_encrypted( -/*=========================*/ - ulint id) /*!< in: space id */ -{ - ulint flags; - - flags = fil_space_get_flags(id); - - if (flags && flags != ULINT_UNDEFINED) { - - return(fsp_flags_is_page_encrypted(flags)); - } - - return(flags); -} - -/*******************************************************************//** -Returns the page encryption key of the space, or 0 if the space -is not encrypted. The tablespace must be cached in the memory cache. -@return page compression level, ULINT_UNDEFINED if space not found */ -UNIV_INLINE -ulint -fil_space_get_page_encryption_key( -/*=================================*/ - ulint id) /*!< in: space id */ -{ - ulint flags; - - flags = fil_space_get_flags(id); - - if (flags && flags != ULINT_UNDEFINED) { - - return(fsp_flags_get_page_encryption_key(flags)); - } - - return(flags); -} - - - -/*******************************************************************//** -Find out whether the page is page encrypted -@return true if page is page encrypted, false if not */ -UNIV_INLINE -ibool -fil_page_is_encrypted( -/*===================*/ - const byte *buf) /*!< in: page */ -{ - return(mach_read_from_2(buf+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_ENCRYPTED); -} - -/*******************************************************************//** -Find out whether the page is page is first compressed and then encrypted -@return true if page is page compressed+encrypted, false if not */ -UNIV_INLINE -ibool -fil_page_is_compressed_encrypted( -/*=============================*/ - const byte *buf) /*!< in: page */ -{ - return(mach_read_from_2(buf+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED); -} - -/*******************************************************************//** -Find out whether the page can be decrypted. -This is the case, if the page is already decrypted and is not the first page of the table space. -If the page is already decrypted it is not of the FIL_PAGE_PAGE_ENCRYPTED type. -if it is the first page of the table space, it is assumed that a page can be decrypted if the -key found in the flags (part of the 1st page) can be read from the key provider. -The case, if the key changed, is currently not caught. -The function for decrypting the page should already be executed before this. -@return PAGE_ENCRYPTION_KEY_MISSING if key provider is available, but key is not available - PAGE_ENCRYPTION_ERROR if other error occurred - 0 if decryption should be possible -*/ -UNIV_INLINE -ulint -fil_page_encryption_status( -/*===================*/ - const byte *buf) /*!< in: page */ -{ - ulint page_type = mach_read_from_2(buf+FIL_PAGE_TYPE); - - if (page_type == FIL_PAGE_TYPE_FSP_HDR) { - ulint flags = mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + buf); - if (fsp_flags_is_page_encrypted(flags)) { - if (!has_encryption_key(fsp_flags_get_page_encryption_key(flags))) { - /* accessing table would surely fail, because no key or no key provider available */ - if (!has_encryption_key(fsp_flags_get_page_encryption_key(flags))) { - return PAGE_ENCRYPTION_KEY_MISSING; - } - return PAGE_ENCRYPTION_ERROR; - } - } - } - - if(page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { - ulint key = mach_read_from_4(buf + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); - if (!has_encryption_key(key)) { - return PAGE_ENCRYPTION_KEY_MISSING; - } - return PAGE_ENCRYPTION_ERROR; - } - return 0; -} diff --git a/storage/xtradb/include/os0file.h b/storage/xtradb/include/os0file.h index f93bf026903..154adf085b8 100644 --- a/storage/xtradb/include/os0file.h +++ b/storage/xtradb/include/os0file.h @@ -323,24 +323,21 @@ The wrapper functions have the prefix of "innodb_". */ # define os_aio(type, mode, name, file, buf, offset, \ n, message1, message2, space_id, \ - trx, page_compressed, page_compression_level, write_size, \ - page_encryption, page_encryption_key, lsn, encrypt) \ + trx, write_size) \ pfs_os_aio_func(type, mode, name, file, buf, offset, \ - n, message1, message2, space_id, trx, \ - page_compressed, page_compression_level, write_size, \ - page_encryption, page_encryption_key, lsn, encrypt, \ + n, message1, message2, space_id, trx, write_size, \ __FILE__, __LINE__) -# define os_file_read(file, buf, offset, n, compressed) \ - pfs_os_file_read_func(file, buf, offset, n, NULL, compressed, \ +# define os_file_read(file, buf, offset, n) \ + pfs_os_file_read_func(file, buf, offset, n, NULL, \ __FILE__, __LINE__) -# define os_file_read_trx(file, buf, offset, n, trx, compressed) \ - pfs_os_file_read_func(file, buf, offset, n, trx, compressed, \ +# define os_file_read_trx(file, buf, offset, n, trx) \ + pfs_os_file_read_func(file, buf, offset, n, trx, \ __FILE__, __LINE__) -# define os_file_read_no_error_handling(file, buf, offset, n, compressed) \ - pfs_os_file_read_no_error_handling_func(file, buf, offset, n, compressed, \ +# define os_file_read_no_error_handling(file, buf, offset, n) \ + pfs_os_file_read_no_error_handling_func(file, buf, offset, n, \ __FILE__, __LINE__) # define os_file_write(name, file, buf, offset, n) \ @@ -376,22 +373,18 @@ to original un-instrumented file I/O APIs */ # define os_file_close(file) os_file_close_func(file) # define os_aio(type, mode, name, file, buf, offset, n, message1, \ - message2, space_id, trx, \ - page_compressed, page_compression_level, write_size, \ - page_encryption, page_encryption_key, lsn, encrypt) \ + message2, space_id, trx, write_size) \ os_aio_func(type, mode, name, file, buf, offset, n, \ - message1, message2, space_id, trx, \ - page_compressed, page_compression_level, write_size, \ - page_encryption, page_encryption_key, lsn, encrypt) + message1, message2, space_id, trx, write_size) -# define os_file_read(file, buf, offset, n, compressed) \ - os_file_read_func(file, buf, offset, n, NULL, compressed) +# define os_file_read(file, buf, offset, n) \ + os_file_read_func(file, buf, offset, n, NULL) -# define os_file_read_trx(file, buf, offset, n, trx, compressed) \ - os_file_read_func(file, buf, offset, n, trx, compressed) +# define os_file_read_trx(file, buf, offset, n, trx) \ + os_file_read_func(file, buf, offset, n, trx) -# define os_file_read_no_error_handling(file, buf, offset, n, compressed) \ - os_file_read_no_error_handling_func(file, buf, offset, n, compressed) +# define os_file_read_no_error_handling(file, buf, offset, n) \ + os_file_read_no_error_handling_func(file, buf, offset, n) # define os_file_write(name, file, buf, offset, n) \ os_file_write_func(name, file, buf, offset, n) @@ -747,8 +740,6 @@ pfs_os_file_read_func( os_offset_t offset, /*!< in: file offset where to read */ ulint n, /*!< in: number of bytes to read */ trx_t* trx, /*!< in: trx */ - ibool compressed, /*!< in: is this file space - compressed ? */ const char* src_file,/*!< in: file name where func invoked */ ulint src_line);/*!< in: line where the func invoked */ @@ -767,8 +758,6 @@ pfs_os_file_read_no_error_handling_func( void* buf, /*!< in: buffer where to read */ os_offset_t offset, /*!< in: file offset where to read */ ulint n, /*!< in: number of bytes to read */ - ibool compressed, /*!< in: is this file space - compressed ? */ const char* src_file,/*!< in: file name where func invoked */ ulint src_line);/*!< in: line where the func invoked */ @@ -801,21 +790,11 @@ pfs_os_aio_func( OS_AIO_SYNC */ ulint space_id, trx_t* trx, - ibool page_compression, /*!< in: is page compression used - on this file space */ - ulint page_compression_level, /*!< page compression - level to be used */ ulint* write_size,/*!< in/out: Actual write size initialized after fist successfull trim operation for this page and if initialized we do not trim again if actual page size does not decrease. */ - ibool page_encryption, /*!< in: is page encryption used - on this file space */ - ulint page_encryption_key, /*!< in: page encryption - key to be used */ - lsn_t lsn, /*!< in: lsn of the newest modification */ - bool encrypt_later, /*!< in: should we encrypt ? */ const char* src_file,/*!< in: file name where func invoked */ ulint src_line);/*!< in: line where the func invoked */ /*******************************************************************//** @@ -979,9 +958,7 @@ os_file_read_func( void* buf, /*!< in: buffer where to read */ os_offset_t offset, /*!< in: file offset where to read */ ulint n, /*!< in: number of bytes to read */ - trx_t* trx, /*!< in: trx */ - ibool compressed); /*!< in: is this file space - compressed ? */ + trx_t* trx); /*!< in: trx */ /*******************************************************************//** Rewind file to its start, read at most size - 1 bytes from it to str, and NUL-terminate str. All errors are silently ignored. This function is @@ -1006,9 +983,7 @@ os_file_read_no_error_handling_func( os_file_t file, /*!< in: handle to a file */ void* buf, /*!< in: buffer where to read */ os_offset_t offset, /*!< in: file offset where to read */ - ulint n, /*!< in: number of bytes to read */ - ibool compressed); /*!< in: is this file space - compressed ? */ + ulint n); /*!< in: number of bytes to read */ /*******************************************************************//** NOTE! Use the corresponding macro os_file_write(), not directly this @@ -1194,21 +1169,11 @@ os_aio_func( OS_AIO_SYNC */ ulint space_id, trx_t* trx, - ibool page_compression, /*!< in: is page compression used - on this file space */ - ulint page_compression_level, /*!< page compression - level to be used */ - ulint* write_size,/*!< in/out: Actual write size initialized + ulint* write_size);/*!< in/out: Actual write size initialized after fist successfull trim operation for this page and if initialized we do not trim again if actual page size does not decrease. */ - ibool page_encryption, /*!< in: is page encryption used - on this file space */ - ulint page_encryption_key, /*!< in: page encryption key - to be used */ - lsn_t lsn, /*!< in: lsn of the newest modification */ - bool encrypt_later); /*!< in: should we encrypt ? */ /************************************************************************//** Wakes up all async i/o threads so that they know to exit themselves in shutdown. */ diff --git a/storage/xtradb/include/os0file.ic b/storage/xtradb/include/os0file.ic index 39599ba2047..b25f1577362 100644 --- a/storage/xtradb/include/os0file.ic +++ b/storage/xtradb/include/os0file.ic @@ -220,23 +220,11 @@ pfs_os_aio_func( OS_AIO_SYNC */ ulint space_id, trx_t* trx, - ibool page_compression, /*!< in: is page compression used - on this file space */ - ulint page_compression_level, /*!< in: page compression - level to be used */ ulint* write_size,/*!< in/out: Actual write size initialized after fist successfull trim operation for this page and if initialized we do not trim again if actual page size does not decrease. */ - ibool page_encryption, /*!< in: is page encryption used - on this file space */ - ulint page_encryption_key, /*!< in: page encryption - key to be used */ - - lsn_t lsn, /*!< in: lsn of the newest - modification */ - bool encrypt_later, /*!< in: encrypt later ? */ const char* src_file,/*!< in: file name where func invoked */ ulint src_line)/*!< in: line where the func invoked */ { @@ -253,8 +241,7 @@ pfs_os_aio_func( result = os_aio_func(type, mode, name, file, buf, offset, n, message1, message2, space_id, trx, - page_compression, page_compression_level, write_size , - page_encryption, page_encryption_key, lsn, encrypt_later); + write_size); register_pfs_file_io_end(locker, n); @@ -276,8 +263,6 @@ pfs_os_file_read_func( os_offset_t offset, /*!< in: file offset where to read */ ulint n, /*!< in: number of bytes to read */ trx_t* trx, - ibool compressed, /*!< in: is this file space - compressed ? */ const char* src_file,/*!< in: file name where func invoked */ ulint src_line)/*!< in: line where the func invoked */ { @@ -288,7 +273,7 @@ pfs_os_file_read_func( register_pfs_file_io_begin(&state, locker, file, n, PSI_FILE_READ, src_file, src_line); - result = os_file_read_func(file, buf, offset, n, trx, compressed); + result = os_file_read_func(file, buf, offset, n, trx); register_pfs_file_io_end(locker, n); @@ -311,8 +296,6 @@ pfs_os_file_read_no_error_handling_func( void* buf, /*!< in: buffer where to read */ os_offset_t offset, /*!< in: file offset where to read */ ulint n, /*!< in: number of bytes to read */ - ibool compressed, /*!< in: is this file space - compressed ? */ const char* src_file,/*!< in: file name where func invoked */ ulint src_line)/*!< in: line where the func invoked */ { @@ -323,7 +306,7 @@ pfs_os_file_read_no_error_handling_func( register_pfs_file_io_begin(&state, locker, file, n, PSI_FILE_READ, src_file, src_line); - result = os_file_read_no_error_handling_func(file, buf, offset, n, compressed); + result = os_file_read_no_error_handling_func(file, buf, offset, n); register_pfs_file_io_end(locker, n); diff --git a/storage/xtradb/log/log0log.cc b/storage/xtradb/log/log0log.cc index 37cf18c1a0d..9b33c7858bc 100644 --- a/storage/xtradb/log/log0log.cc +++ b/storage/xtradb/log/log0log.cc @@ -1384,7 +1384,7 @@ log_group_file_header_flush( (ulint) (dest_offset / UNIV_PAGE_SIZE), (ulint) (dest_offset % UNIV_PAGE_SIZE), OS_FILE_LOG_BLOCK_SIZE, - buf, group, 0, 0, false); + buf, group, 0); srv_stats.os_log_pending_writes.dec(); } @@ -1551,7 +1551,7 @@ loop: fil_io(OS_FILE_WRITE | OS_FILE_LOG, true, group->space_id, 0, (ulint) (next_offset / UNIV_PAGE_SIZE), (ulint) (next_offset % UNIV_PAGE_SIZE), write_len, buf, - group, 0, 0, false); + group, 0); srv_stats.os_log_pending_writes.dec(); @@ -2144,7 +2144,7 @@ log_group_checkpoint( write_offset / UNIV_PAGE_SIZE, write_offset % UNIV_PAGE_SIZE, OS_FILE_LOG_BLOCK_SIZE, - buf, ((byte*) group + 1), 0, 0, false); + buf, ((byte*) group + 1), 0); ut_ad(((ulint) group & 0x1UL) == 0); } @@ -2226,7 +2226,7 @@ log_group_read_checkpoint_info( fil_io(OS_FILE_READ | OS_FILE_LOG, true, group->space_id, 0, field / UNIV_PAGE_SIZE, field % UNIV_PAGE_SIZE, - OS_FILE_LOG_BLOCK_SIZE, log_sys->checkpoint_buf, NULL, 0, 0, false); + OS_FILE_LOG_BLOCK_SIZE, log_sys->checkpoint_buf, NULL, 0); } /******************************************************//** @@ -2639,7 +2639,7 @@ loop: fil_io(OS_FILE_READ | OS_FILE_LOG, sync, group->space_id, 0, (ulint) (source_offset / UNIV_PAGE_SIZE), (ulint) (source_offset % UNIV_PAGE_SIZE), - len, buf, (type == LOG_ARCHIVE) ? &log_archive_io : NULL, 0, 0, false); + len, buf, (type == LOG_ARCHIVE) ? &log_archive_io : NULL, 0); if (recv_sys->recv_log_crypt_ver != UNENCRYPTED_KEY_VER && !log_group_decrypt_after_read(group, buf, len)) @@ -2771,7 +2771,7 @@ log_group_archive_file_header_write( dest_offset / UNIV_PAGE_SIZE, dest_offset % UNIV_PAGE_SIZE, 2 * OS_FILE_LOG_BLOCK_SIZE, - buf, &log_archive_io, 0, 0, false); + buf, &log_archive_io, 0); } /******************************************************//** @@ -2808,7 +2808,7 @@ log_group_archive_completed_header_write( dest_offset % UNIV_PAGE_SIZE, OS_FILE_LOG_BLOCK_SIZE, buf + LOG_FILE_ARCH_COMPLETED, - &log_archive_io, 0, 0, false); + &log_archive_io, 0); } /******************************************************//** @@ -2953,7 +2953,7 @@ loop: (ulint) (next_offset / UNIV_PAGE_SIZE), (ulint) (next_offset % UNIV_PAGE_SIZE), ut_calc_align(len, OS_FILE_LOG_BLOCK_SIZE), buf, - &log_archive_io, 0, 0, false); + &log_archive_io, 0); start_lsn += len; next_offset += len; diff --git a/storage/xtradb/log/log0online.cc b/storage/xtradb/log/log0online.cc index 60ae4a50626..e18677973b0 100644 --- a/storage/xtradb/log/log0online.cc +++ b/storage/xtradb/log/log0online.cc @@ -283,7 +283,7 @@ log_online_read_bitmap_page( ut_a(bitmap_file->offset % MODIFIED_PAGE_BLOCK_SIZE == 0); success = os_file_read(bitmap_file->file, page, bitmap_file->offset, - MODIFIED_PAGE_BLOCK_SIZE, FALSE); + MODIFIED_PAGE_BLOCK_SIZE); if (UNIV_UNLIKELY(!success)) { diff --git a/storage/xtradb/log/log0recv.cc b/storage/xtradb/log/log0recv.cc index c7fb3be9625..46f5954d44f 100644 --- a/storage/xtradb/log/log0recv.cc +++ b/storage/xtradb/log/log0recv.cc @@ -56,6 +56,7 @@ Created 9/20/1997 Heikki Tuuri #include "trx0undo.h" #include "trx0rec.h" #include "fil0fil.h" +#include "fil0crypt.h" #ifndef UNIV_HOTBACKUP # include "buf0rea.h" # include "srv0srv.h" @@ -3177,7 +3178,7 @@ recv_recovery_from_checkpoint_start_func( fil_io(OS_FILE_READ | OS_FILE_LOG, true, max_cp_group->space_id, 0, 0, 0, LOG_FILE_HDR_SIZE, - log_hdr_buf, max_cp_group, 0, 0, false); + log_hdr_buf, max_cp_group, 0); if (0 == ut_memcmp(log_hdr_buf + LOG_FILE_WAS_CREATED_BY_HOT_BACKUP, (byte*)"ibbackup", (sizeof "ibbackup") - 1)) { @@ -3208,7 +3209,7 @@ recv_recovery_from_checkpoint_start_func( fil_io(OS_FILE_WRITE | OS_FILE_LOG, true, max_cp_group->space_id, 0, 0, 0, OS_FILE_LOG_BLOCK_SIZE, - log_hdr_buf, max_cp_group, 0, 0, false); + log_hdr_buf, max_cp_group, 0); } log_hdr_log_block_size @@ -3871,7 +3872,7 @@ ask_again: /* Read the archive file header */ fil_io(OS_FILE_READ | OS_FILE_LOG, true, group->archive_space_id, 0, 0, 0, - LOG_FILE_HDR_SIZE, buf, NULL, 0, 0, false); + LOG_FILE_HDR_SIZE, buf, NULL, 0); /* Check if the archive file header is consistent */ @@ -3945,7 +3946,7 @@ ask_again: fil_io(OS_FILE_READ | OS_FILE_LOG, true, group->archive_space_id, 0, read_offset / UNIV_PAGE_SIZE, - read_offset % UNIV_PAGE_SIZE, len, buf, NULL, 0, 0, false); + read_offset % UNIV_PAGE_SIZE, len, buf, NULL, 0); ret = recv_scan_log_recs( (buf_pool_get_n_pages() diff --git a/storage/xtradb/os/os0file.cc b/storage/xtradb/os/os0file.cc index d7d224dc6dd..75750df5639 100644 --- a/storage/xtradb/os/os0file.cc +++ b/storage/xtradb/os/os0file.cc @@ -45,7 +45,6 @@ Created 10/21/1995 Heikki Tuuri #include "fil0fil.h" #include "fsp0fsp.h" #include "fil0pagecompress.h" -#include "fil0pageencryption.h" #include "buf0buf.h" #include "btr0types.h" #include "trx0trx.h" @@ -238,21 +237,6 @@ struct os_aio_slot_t{ completed */ ulint bitmap; - byte* page_compression_page; /*!< Memory allocated for - page compressed page and - freed after the write - has been completed */ - - byte* page_encryption_page; /*!< Memory allocated for - page encrypted page and - freed after the write - has been completed */ - - ibool page_compression; - ulint page_compression_level; - - ibool page_encryption; - ulint page_encryption_key; ulint* write_size; /*!< Actual write size initialized after fist successfull trim @@ -260,31 +244,13 @@ struct os_aio_slot_t{ initialized we do not trim again if actual page size does not decrease. */ - byte* page_buf; /*!< Actual page buffer for - page compressed pages, do not - free this */ - - byte* page_buf2; /*!< Actual page buffer for - page encrypted pages, do not - free this */ - byte* tmp_encryption_buf; /*!< a temporal buffer used by page encryption */ - - ibool page_compression_success; - /*!< TRUE if page compression was successfull, false if not */ - ibool page_encryption_success; - /*!< TRUE if page encryption was successfull, false if not */ - - lsn_t lsn; /* lsn of the newest modification */ - ulint file_block_size;/*!< file block size */ - bool encrypt_later; /*!< should we encrypt the page */ #ifdef LINUX_NATIVE_AIO struct iocb control; /* Linux control block for aio */ int n_bytes; /* bytes written/read. */ int ret; /* AIO return code */ #endif /* WIN_ASYNC_IO */ - byte *lzo_mem; /* Temporal memory used by LZO */ }; /** The asynchronous i/o array structure */ @@ -401,39 +367,6 @@ os_file_trim( /*=========*/ os_aio_slot_t* slot); /*!< in: slot structure */ -/**********************************************************************//** -Allocate memory for temporal buffer used for page compression. This -buffer is freed later. */ -UNIV_INTERN -void -os_slot_alloc_page_buf( -/*===================*/ - os_aio_slot_t* slot); /*!< in: slot structure */ - -#ifdef HAVE_LZO -/**********************************************************************//** -Allocate memory for temporal memory used for page compression when -LZO compression method is used */ -UNIV_INTERN -void -os_slot_alloc_lzo_mem( -/*===================*/ - os_aio_slot_t* slot); /*!< in: slot structure */ -#endif - -/**********************************************************************//** -Allocate memory for temporal buffer used for page encryption. This -buffer is freed later. */ -UNIV_INTERN -void -os_slot_alloc_page_buf2( - os_aio_slot_t* slot); /*!< in: slot structure */ -/**********************************************************************//** -Allocate memory for temporal buffer used for page encryption. */ -UNIV_INTERN -void -os_slot_alloc_tmp_encryption_buf( - os_aio_slot_t* slot); /*!< in: slot structure */ /****************************************************************//** Does error handling when a file operation fails. @return TRUE if we should retry the operation */ @@ -3114,9 +3047,7 @@ os_file_read_func( void* buf, /*!< in: buffer where to read */ os_offset_t offset, /*!< in: file offset where to read */ ulint n, /*!< in: number of bytes to read */ - trx_t* trx, - ibool compressed) /*!< in: is this file space - compressed ? */ + trx_t* trx) { #ifdef __WIN__ BOOL ret; @@ -3212,9 +3143,7 @@ os_file_read_no_error_handling_func( os_file_t file, /*!< in: handle to a file */ void* buf, /*!< in: buffer where to read */ os_offset_t offset, /*!< in: file offset where to read */ - ulint n, /*!< in: number of bytes to read */ - ibool compressed) /*!< in: is this file space - compressed ? */ + ulint n) /*!< in: number of bytes to read */ { #ifdef __WIN__ BOOL ret; @@ -4275,8 +4204,6 @@ os_aio_array_free( /*==============*/ os_aio_array_t*& array) /*!< in, own: array to free */ { - ulint i; - os_mutex_free(array->mutex); os_event_free(array->not_full); os_event_free(array->is_empty); @@ -4288,31 +4215,6 @@ os_aio_array_free( } #endif /* LINUX_NATIVE_AIO */ - for (i = 0; i < array->n_slots; i++) { - os_aio_slot_t* slot = os_aio_array_get_nth_slot(array, i); - - if (slot->page_compression_page) { - ut_free(slot->page_compression_page); - slot->page_compression_page = NULL; - } - - if (slot->lzo_mem) { - ut_free(slot->lzo_mem); - slot->lzo_mem = NULL; - } - - if (slot->page_encryption_page) { - ut_free(slot->page_encryption_page); - slot->page_encryption_page = NULL; - } - - if (slot->tmp_encryption_buf) { - ut_free(slot->tmp_encryption_buf); - slot->tmp_encryption_buf = NULL; - } - } - - ut_free(array->slots); ut_free(array); @@ -4658,22 +4560,11 @@ os_aio_array_reserve_slot( os_offset_t offset, /*!< in: file offset */ ulint len, /*!< in: length of the block to read or write */ ulint space_id, - ibool page_compression, /*!< in: is page compression used - on this file space */ - ulint page_compression_level, /*!< page compression - level to be used */ - ibool page_encryption, /*!< in: is page encryption used - on this file space */ - ulint page_encryption_key, /*!< page encryption key - to be used */ - ulint* write_size,/*!< in/out: Actual write size initialized + ulint* write_size)/*!< in/out: Actual write size initialized after first successfull trim operation for this page and if initialized we do not trim again if actual page size does not decrease. */ - lsn_t lsn, /*!< in: lsn of the newest - modification */ - bool encrypt_later) /*!< in: should we encrypt the page */ { os_aio_slot_t* slot = NULL; #ifdef WIN_ASYNC_IO @@ -4762,96 +4653,13 @@ found: slot->type = type; slot->buf = static_cast(buf); slot->offset = offset; - slot->lsn = lsn; slot->io_already_done = FALSE; slot->space_id = space_id; - slot->page_compression_success = FALSE; - slot->page_encryption_success = FALSE; - slot->write_size = write_size; - slot->page_compression_level = page_compression_level; - slot->page_compression = page_compression; - slot->page_encryption_key = page_encryption_key; - slot->page_encryption = page_encryption; - slot->encrypt_later = encrypt_later; if (message1) { slot->file_block_size = fil_node_get_block_size(message1); } - - /* If the space is page compressed and this is write operation - then we encrypt the page */ - if (message1 && type == OS_FILE_WRITE && page_compression) { - ulint real_len = len; - byte* tmp = NULL; - - /* Release the array mutex while encrypting */ - os_mutex_exit(array->mutex); - - // We allocate memory for page compressed buffer if and only - // if it is not yet allocated. - os_slot_alloc_page_buf(slot); - -#ifdef HAVE_LZO - if (innodb_compression_algorithm == 3) { - os_slot_alloc_lzo_mem(slot); - } -#endif - - /* Call page compression */ - tmp = fil_compress_page( - fil_node_get_space_id(slot->message1), - (byte *)buf, - slot->page_buf, - len, - page_compression_level, - fil_node_get_block_size(slot->message1), - &real_len, - slot->lzo_mem - ); - - /* If compression succeeded, set up the length and buffer */ - if (tmp != buf) { - len = real_len; - buf = slot->page_buf; - slot->len = real_len; - slot->page_compression_success = TRUE; - } else { - slot->page_compression_success = FALSE; - } - - /* Take array mutex back, not sure if this is really needed - below */ - os_mutex_enter(array->mutex); - - } - - /* If the space is page encryption and this is write operation - then we encrypt the page */ - if (message1 && type == OS_FILE_WRITE && (page_encryption || encrypt_later)) { - /* Release the array mutex while encrypting */ - os_mutex_exit(array->mutex); - - // We allocate memory for page encrypted buffer if and only - // if it is not yet allocated. - os_slot_alloc_page_buf2(slot); - - fil_space_encrypt( - fil_node_get_space_id(slot->message1), - slot->offset, - slot->lsn, - (byte *)buf, - slot->len, - slot->page_buf2, - slot->page_encryption_key); - - slot->page_encryption_success = TRUE; - buf = slot->page_buf2; - - /* Take array mutex back */ - os_mutex_enter(array->mutex); - } - slot->buf = (byte *)buf; #ifdef WIN_ASYNC_IO @@ -5130,22 +4938,11 @@ os_aio_func( OS_AIO_SYNC */ ulint space_id, trx_t* trx, - ibool page_compression, /*!< in: is page compression used - on this file space */ - ulint page_compression_level, /*!< page compression - level to be used */ - ulint* write_size,/*!< in/out: Actual write size initialized + ulint* write_size)/*!< in/out: Actual write size initialized after fist successfull trim operation for this page and if initialized we do not trim again if actual page size does not decrease. */ - ibool page_encryption, /*!< in: is page encryption used - on this file space */ - ulint page_encryption_key, /*!< in: page encryption key - to be used */ - lsn_t lsn, /*!< in: lsn of the newest modification */ - bool encrypt_later) /*!< in: should we encrypt before - writing the page */ { os_aio_array_t* array; os_aio_slot_t* slot; @@ -5178,8 +4975,7 @@ os_aio_func( no need to use an i/o-handler thread */ if (type == OS_FILE_READ) { - ret = os_file_read_func(file, buf, offset, n, trx, - page_compression); + ret = os_file_read_func(file, buf, offset, n, trx); } else { ut_ad(!srv_read_only_mode); ut_a(type == OS_FILE_WRITE); @@ -5247,9 +5043,7 @@ try_again: slot = os_aio_array_reserve_slot(type, array, message1, message2, file, name, buf, offset, n, space_id, - page_compression, page_compression_level, - page_encryption, page_encryption_key, - write_size, lsn, encrypt_later); + write_size); if (type == OS_FILE_READ) { if (srv_use_native_aio) { @@ -5278,15 +5072,8 @@ try_again: if (srv_use_native_aio) { os_n_file_writes++; #ifdef WIN_ASYNC_IO - if (page_encryption && slot->page_encryption_success) { - buffer = slot->page_buf2; - n = slot->len; - } else if (page_compression && slot->page_compression_success) { - buffer = slot->page_buf; - n = slot->len; - } else { - buffer = buf; - } + n = slot->len; + buffer = buf; ret = WriteFile(file, buffer, (DWORD) n, &len, &(slot->control)); @@ -5445,22 +5232,12 @@ os_aio_windows_handle( switch (slot->type) { case OS_FILE_WRITE: - if (slot->message1 && slot->page_encryption && slot->page_encryption_success) { - ret_val = os_file_write(slot->name, slot->file, slot->page_buf2, - slot->offset, slot->len); - } else { - if (slot->message1 && slot->page_compression && slot->page_compression_success) { - ret_val = os_file_write(slot->name, slot->file, slot->page_buf, + ret_val = os_file_write(slot->name, slot->file, slot->buf, slot->offset, slot->len); - } else { - ret_val = os_file_write(slot->name, slot->file, slot->buf, - slot->offset, slot->len); - } - } break; case OS_FILE_READ: ret_val = os_file_read(slot->file, slot->buf, - slot->offset, slot->len, slot->page_compression); + slot->offset, slot->len); break; default: ut_error; @@ -5485,45 +5262,10 @@ os_aio_windows_handle( ret_val = ret && len == slot->len; } - if (slot->type == OS_FILE_READ) { - if (fil_page_is_compressed_encrypted(slot->buf) || - fil_page_is_encrypted(slot->buf)) { - ut_ad(slot->message1 != NULL); - os_slot_alloc_page_buf2(slot); - os_slot_alloc_tmp_encryption_buf(slot); - - // Decrypt the data - fil_space_decrypt( - fil_node_get_space_id(slot->message1), - slot->buf, - slot->len, - slot->page_buf2); - // Copy decrypted buffer back to buf - memcpy(slot->buf, slot->page_buf2, slot->len); - } - - if (fil_page_is_compressed(slot->buf)) { - /* We allocate memory for page compressed buffer if - and only if it is not yet allocated. */ - os_slot_alloc_page_buf(slot); - -#ifdef HAVE_LZO - if (fil_page_is_lzo_compressed(slot->buf)) { - os_slot_alloc_lzo_mem(slot); - } -#endif - fil_decompress_page(slot->page_buf, slot->buf, - slot->len, slot->write_size); - } - } else { - /* OS_FILE_WRITE */ - if (slot->page_compression_success && - (fil_page_is_compressed(slot->page_buf) || - fil_page_is_compressed_encrypted(slot->buf))) { - if (srv_use_trim && os_fallocate_failed == FALSE) { - // Deallocate unused blocks from file system - os_file_trim(slot); - } + if (slot->type == OS_FILE_WRITE) { + if (srv_use_trim && os_fallocate_failed == FALSE) { + // Deallocate unused blocks from file system + os_file_trim(slot); } } @@ -5616,48 +5358,10 @@ retry: /* We have not overstepped to next segment. */ ut_a(slot->pos < end_pos); - if (slot->type == OS_FILE_READ) { - /* If the page is page encrypted we decrypt */ - if (fil_page_is_compressed_encrypted(slot->buf) || - fil_page_is_encrypted(slot->buf)) { - os_slot_alloc_page_buf2(slot); - os_slot_alloc_tmp_encryption_buf(slot); - ut_ad(slot->message1 != NULL); - - // Decrypt the data - fil_space_decrypt(fil_node_get_space_id(slot->message1), - slot->buf, - slot->len, - slot->page_buf2); - // Copy decrypted buffer back to buf - memcpy(slot->buf, slot->page_buf2, slot->len); - } - - /* If the table is page compressed and this - is read, we decompress before we announce - the read is complete. For writes, we free - the compressed page. */ - if (fil_page_is_compressed(slot->buf)) { - // We allocate memory for page compressed buffer if and only - // if it is not yet allocated. - os_slot_alloc_page_buf(slot); -#ifdef HAVE_LZO - if (fil_page_is_lzo_compressed(slot->buf)) { - os_slot_alloc_lzo_mem(slot); - } -#endif - fil_decompress_page(slot->page_buf, slot->buf, slot->len, slot->write_size); - } - } else { - /* OS_FILE_WRITE */ - if (slot->page_compression_success && - (fil_page_is_compressed(slot->page_buf) || - fil_page_is_compressed_encrypted(slot->buf))) { - ut_ad(slot->page_compression_page); - if (srv_use_trim && os_fallocate_failed == FALSE) { - // Deallocate unused blocks from file system - os_file_trim(slot); - } + if (slot->type == OS_FILE_WRITE) { + if (srv_use_trim && os_fallocate_failed == FALSE) { + // Deallocate unused blocks from file system + os_file_trim(slot); } } @@ -6140,8 +5844,7 @@ consecutive_loop: } else { ret = os_file_read( aio_slot->file, combined_buf, - aio_slot->offset, total_len, - aio_slot->page_compression); + aio_slot->offset, total_len); } srv_set_io_thread_op_info(global_segment, "file i/o done"); @@ -6714,91 +6417,6 @@ os_file_trim( } -/**********************************************************************//** -Allocate memory for temporal buffer used for page encryption. This -buffer is freed later. */ -UNIV_INTERN -void -os_slot_alloc_page_buf2( -/*===================*/ - os_aio_slot_t* slot) /*!< in: slot structure */ -{ - ut_a(slot != NULL); - - if(slot->page_buf2 == NULL) { - byte* cbuf2; - byte* cbuf; - - cbuf2 = static_cast(ut_malloc(UNIV_PAGE_SIZE*2)); - cbuf = static_cast(ut_align(cbuf2, UNIV_PAGE_SIZE)); - slot->page_encryption_page = static_cast(cbuf2); - slot->page_buf2 = static_cast(cbuf); - memset(slot->page_encryption_page, 0, UNIV_PAGE_SIZE*2); - } -} - -/**********************************************************************//** -Allocate memory for temporal buffer used for page compression. This -buffer is freed later. */ -UNIV_INTERN -void -os_slot_alloc_page_buf( -/*===================*/ - os_aio_slot_t* slot) /*!< in: slot structure */ -{ - ut_a(slot != NULL); - if (slot->page_buf == NULL) { - byte* cbuf2; - byte* cbuf; - ulint asize = UNIV_PAGE_SIZE; -#ifdef HAVE_SNAPPY - asize += snappy_max_compressed_length(asize) - UNIV_PAGE_SIZE; -#endif - /* We allocate extra to avoid memory overwrite on - compression */ - cbuf2 = static_cast(ut_malloc(asize*2)); - cbuf = static_cast(ut_align(cbuf2, UNIV_PAGE_SIZE)); - slot->page_compression_page = static_cast(cbuf2); - slot->page_buf = static_cast(cbuf); - ut_a(slot->page_buf != NULL); - memset(slot->page_compression_page, 0, asize*2); - } -} - -#ifdef HAVE_LZO -/**********************************************************************//** -Allocate memory for temporal memory used for page compression when -LZO compression method is used */ -UNIV_INTERN -void -os_slot_alloc_lzo_mem( -/*===================*/ - os_aio_slot_t* slot) /*!< in: slot structure */ -{ - ut_a(slot != NULL); - if(slot->lzo_mem == NULL) { - slot->lzo_mem = static_cast(ut_malloc(LZO1X_1_15_MEM_COMPRESS)); - ut_a(slot->lzo_mem != NULL); - memset(slot->lzo_mem, 0, LZO1X_1_15_MEM_COMPRESS); - } -} -#endif - -/**********************************************************************//** -Allocate memory for temporal buffer used for page encryption. */ -UNIV_INTERN -void -os_slot_alloc_tmp_encryption_buf( -/*=============================*/ - os_aio_slot_t* slot) /*!< in: slot structure */ -{ - ut_a(slot != NULL); - if (slot->tmp_encryption_buf == NULL) { - slot->tmp_encryption_buf = static_cast(ut_malloc(64)); - memset(slot->tmp_encryption_buf, 0, 64); - } -} - /***********************************************************************//** Try to get number of bytes per sector from file system. @return file block size */ diff --git a/storage/xtradb/row/row0log.cc b/storage/xtradb/row/row0log.cc index 5a086ccebb4..ac0e75929a4 100644 --- a/storage/xtradb/row/row0log.cc +++ b/storage/xtradb/row/row0log.cc @@ -2552,7 +2552,7 @@ all_done: success = os_file_read_no_error_handling( OS_FILE_FROM_FD(index->online_log->fd), index->online_log->head.block, ofs, - srv_sort_buf_size, FALSE); + srv_sort_buf_size); if (!success) { fprintf(stderr, "InnoDB: unable to read temporary file" @@ -3380,7 +3380,7 @@ all_done: success = os_file_read_no_error_handling( OS_FILE_FROM_FD(index->online_log->fd), index->online_log->head.block, ofs, - srv_sort_buf_size, FALSE); + srv_sort_buf_size); if (!success) { fprintf(stderr, "InnoDB: unable to read temporary file" diff --git a/storage/xtradb/row/row0merge.cc b/storage/xtradb/row/row0merge.cc index 61494bed68a..fc08a4c6f07 100644 --- a/storage/xtradb/row/row0merge.cc +++ b/storage/xtradb/row/row0merge.cc @@ -786,7 +786,7 @@ row_merge_read( #endif /* UNIV_DEBUG */ success = os_file_read_no_error_handling(OS_FILE_FROM_FD(fd), buf, - ofs, srv_sort_buf_size, FALSE); + ofs, srv_sort_buf_size); #ifdef POSIX_FADV_DONTNEED /* Each block is read exactly once. Free up the file cache. */ diff --git a/storage/xtradb/row/row0mysql.cc b/storage/xtradb/row/row0mysql.cc index 5b2b9ac1503..5929bb21f7b 100644 --- a/storage/xtradb/row/row0mysql.cc +++ b/storage/xtradb/row/row0mysql.cc @@ -57,6 +57,7 @@ Created 9/17/2000 Heikki Tuuri #include "btr0sea.h" #include "btr0defragment.h" #include "fil0fil.h" +#include "fil0crypt.h" #include "ibuf0ibuf.h" #include "fts0fts.h" #include "fts0types.h" diff --git a/storage/xtradb/srv/srv0srv.cc b/storage/xtradb/srv/srv0srv.cc index 37adca6975b..9396a713cad 100644 --- a/storage/xtradb/srv/srv0srv.cc +++ b/storage/xtradb/srv/srv0srv.cc @@ -75,10 +75,10 @@ Created 10/8/1995 Heikki Tuuri #include "mysql/plugin.h" #include "mysql/service_thd_wait.h" #include "fil0fil.h" +#include "fil0crypt.h" #include "fil0pagecompress.h" #include #include "btr0scrub.h" -#include "fil0pageencryption.h" /* prototypes of new functions added to ha_innodb.cc for kill_idle_transaction */ ibool innobase_thd_is_idle(const void* thd); @@ -671,7 +671,7 @@ second. */ static time_t srv_last_log_flush_time; /** Default encryption key used for page encryption */ -UNIV_INTERN uint srv_default_page_encryption_key = DEFAULT_ENCRYPTION_KEY; +UNIV_INTERN uint srv_default_page_encryption_key = FIL_DEFAULT_ENCRYPTION_KEY; /** Enable semaphore request instrumentation */ UNIV_INTERN my_bool srv_instrument_semaphores = FALSE; diff --git a/storage/xtradb/srv/srv0start.cc b/storage/xtradb/srv/srv0start.cc index ddf261cc236..4d2aee6f08c 100644 --- a/storage/xtradb/srv/srv0start.cc +++ b/storage/xtradb/srv/srv0start.cc @@ -3,7 +3,7 @@ Copyright (c) 1996, 2014, Oracle and/or its affiliates. All rights reserved. Copyright (c) 2008, Google Inc. Copyright (c) 2009, Percona Inc. -Copyright (c) 2013, 2014, SkySQL Ab. All Rights Reserved. +Copyright (c) 2013, 2015, MariaDB Corporation Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -52,6 +52,7 @@ Created 2/16/1996 Heikki Tuuri #include "os0file.h" #include "os0thread.h" #include "fil0fil.h" +#include "fil0crypt.h" #include "fsp0fsp.h" #include "rem0rec.h" #include "mtr0mtr.h" From 0ba9fa35bc119cbded96cd8587427be195603cec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Wed, 1 Apr 2015 19:37:00 +0300 Subject: [PATCH 2/6] InnoDB/XtraDB Encryption cleanup Step 2: -- Introduce temporal memory array to buffer pool where to allocate temporary memory for encryption/compression -- Rename PAGE_ENCRYPTION -> ENCRYPTION -- Rename PAGE_ENCRYPTION_KEY -> ENCRYPTION_KEY -- Rename innodb_default_page_encryption_key -> innodb_default_encryption_key -- Allow enable/disable encryption for tables by changing ENCRYPTION to enum having values DEFAULT, ON, OFF -- In create table store crypt_data if ENCRYPTION is ON or OFF -- Do not crypt tablespaces having ENCRYPTION=OFF -- Store encryption mode to crypt_data and redo-log --- storage/innobase/buf/buf0buf.cc | 338 +++++++++++----------- storage/innobase/buf/buf0dblwr.cc | 4 +- storage/innobase/buf/buf0flu.cc | 6 +- storage/innobase/buf/buf0rea.cc | 3 +- storage/innobase/fil/fil0crypt.cc | 44 ++- storage/innobase/fil/fil0fil.cc | 102 +++++-- storage/innobase/handler/ha_innodb.cc | 68 +++-- storage/innobase/handler/ha_innodb.h | 4 +- storage/innobase/handler/handler0alter.cc | 4 +- storage/innobase/include/buf0buf.h | 72 +++-- storage/innobase/include/buf0buf.ic | 17 +- storage/innobase/include/fil0crypt.h | 15 +- storage/innobase/include/fil0fil.h | 10 + storage/innobase/include/srv0srv.h | 4 +- storage/innobase/os/os0file.cc | 1 - storage/innobase/srv/srv0srv.cc | 4 +- storage/xtradb/buf/buf0buf.cc | 333 +++++++++++---------- storage/xtradb/buf/buf0dblwr.cc | 5 +- storage/xtradb/buf/buf0flu.cc | 6 +- storage/xtradb/buf/buf0rea.cc | 3 +- storage/xtradb/fil/fil0crypt.cc | 44 ++- storage/xtradb/fil/fil0fil.cc | 102 +++++-- storage/xtradb/handler/ha_innodb.cc | 68 +++-- storage/xtradb/handler/ha_innodb.h | 5 +- storage/xtradb/handler/handler0alter.cc | 4 +- storage/xtradb/include/buf0buf.h | 74 +++-- storage/xtradb/include/buf0buf.ic | 17 +- storage/xtradb/include/fil0crypt.h | 15 +- storage/xtradb/include/fil0fil.h | 10 + storage/xtradb/include/srv0srv.h | 4 +- storage/xtradb/srv/srv0srv.cc | 4 +- 31 files changed, 822 insertions(+), 568 deletions(-) diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index b2758e1253f..a4825adbbbb 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -59,6 +59,14 @@ Created 11/5/1995 Heikki Tuuri #include "ut0byte.h" #include +#ifdef HAVE_LZO +#include "lzo/lzo1x.h" +#endif + +/* Number of temporary slots used for encryption/compression +memory allocation before/after I/O operations */ +#define BUF_MAX_TMP_SLOTS 200 + /* IMPLEMENTATION OF THE BUFFER POOL ================================= @@ -1001,14 +1009,11 @@ buf_block_init( block->page.state = BUF_BLOCK_NOT_USED; block->page.buf_fix_count = 0; block->page.io_fix = BUF_IO_NONE; - block->page.crypt_buf = NULL; - block->page.crypt_buf_free = NULL; - block->page.comp_buf = NULL; - block->page.comp_buf_free = NULL; block->page.key_version = 0; - block->page.encrypt_later = false; - + block->page.real_size = 0; + block->page.write_size = 0; block->modify_clock = 0; + block->page.slot = NULL; #if defined UNIV_DEBUG_FILE_ACCESSES || defined UNIV_DEBUG block->page.file_page_was_freed = FALSE; @@ -1361,6 +1366,11 @@ buf_pool_init_instance( /* Initialize the iterator for single page scan search */ new(&buf_pool->single_scan_itr) LRUItr(buf_pool, &buf_pool->mutex); + /* Initialize the temporal memory array and slots */ + buf_pool->tmp_arr = (buf_tmp_array_t *)mem_zalloc(sizeof(buf_tmp_array_t)); + buf_pool->tmp_arr->n_slots = BUF_MAX_TMP_SLOTS; + buf_pool->tmp_arr->slots = (buf_tmp_buffer_t*)mem_zalloc(sizeof(buf_tmp_buffer_t) * BUF_MAX_TMP_SLOTS); + buf_pool_mutex_exit(buf_pool); return(DB_SUCCESS); @@ -1412,6 +1422,25 @@ buf_pool_free_instance( ha_clear(buf_pool->page_hash); hash_table_free(buf_pool->page_hash); hash_table_free(buf_pool->zip_hash); + + /* Free all used temporary slots */ + for(ulint i = 0; i < buf_pool->tmp_arr->n_slots; i++) { + buf_tmp_buffer_t* slot = &buf_pool->tmp_arr->slots[i]; +#ifdef HAVE_LZO + if (slot->lzo_mem) { + ut_free(slot->lzo_mem); + } +#endif + if (slot->crypt_buf_free) { + ut_free(slot->crypt_buf_free); + } + if (slot->comp_buf_free) { + ut_free(slot->comp_buf_free); + } + } + + mem_free(buf_pool->tmp_arr->slots); + mem_free(buf_pool->tmp_arr); } /********************************************************************//** @@ -3494,12 +3523,9 @@ buf_page_init_low( bpage->newest_modification = 0; bpage->oldest_modification = 0; bpage->write_size = 0; - bpage->crypt_buf = NULL; - bpage->crypt_buf_free = NULL; - bpage->comp_buf = NULL; - bpage->comp_buf_free = NULL; bpage->key_version = 0; - bpage->encrypt_later = false; + bpage->real_size = 0; + bpage->slot = NULL; HASH_INVALIDATE(bpage, hash); #if defined UNIV_DEBUG_FILE_ACCESSES || defined UNIV_DEBUG @@ -4366,7 +4392,11 @@ corrupt: } } else { /* io_type == BUF_IO_WRITE */ - buf_page_encrypt_after_write(bpage); + if (bpage->slot) { + /* Mark slot free */ + bpage->slot->reserved = false; + bpage->slot = NULL; + } } buf_pool_mutex_enter(buf_pool); @@ -5602,6 +5632,60 @@ buf_page_init_for_backup_restore( } #endif /* !UNIV_HOTBACKUP */ +/********************************************************************//** +Reserve unused slot from temporary memory array and allocate necessary +temporary memory if not yet allocated. +@return reserved slot */ +buf_tmp_buffer_t* +buf_pool_reserve_tmp_slot( +/*======================*/ + buf_pool_t* buf_pool, /*!< in: buffer pool where to + reserve */ + bool compressed) /*!< in: is file space compressed */ +{ + buf_tmp_buffer_t *free_slot=NULL; + + /* Array is protected by buf_pool mutex */ + buf_pool_mutex_enter(buf_pool); + + for(ulint i = 0; i < buf_pool->tmp_arr->n_slots; i++) { + buf_tmp_buffer_t *slot = &buf_pool->tmp_arr->slots[i]; + + if(slot->reserved == false) { + free_slot = slot; + break; + } + } + + /* We assume that free slot is found */ + ut_a(free_slot != NULL); + free_slot->reserved = true; + /* Now that we have reserved this slot we can release + buf_pool mutex */ + buf_pool_mutex_exit(buf_pool); + + /* Allocate temporary memory for encryption/decryption */ + if (free_slot->crypt_buf_free == NULL) { + free_slot->crypt_buf_free = static_cast(ut_malloc(UNIV_PAGE_SIZE*2)); + free_slot->crypt_buf = static_cast(ut_align(free_slot->crypt_buf_free, UNIV_PAGE_SIZE)); + memset(free_slot->crypt_buf_free, 0, UNIV_PAGE_SIZE *2); + } + + /* For page compressed tables allocate temporary memory for + compression/decompression */ + if (compressed && free_slot->comp_buf_free == NULL) { + free_slot->comp_buf_free = static_cast(ut_malloc(UNIV_PAGE_SIZE*2)); + free_slot->comp_buf = static_cast(ut_align(free_slot->comp_buf_free, UNIV_PAGE_SIZE)); + memset(free_slot->comp_buf_free, 0, UNIV_PAGE_SIZE *2); +#ifdef HAVE_LZO + free_slot->lzo_mem = static_cast(ut_malloc(LZO1X_1_15_MEM_COMPRESS)); + memset(free_slot->lzo_mem, 0, LZO1X_1_15_MEM_COMPRESS); +#endif + } + + return (free_slot); +} + /********************************************************************//** Encrypts a buffer page right before it's flushed to disk */ @@ -5609,11 +5693,17 @@ byte* buf_page_encrypt_before_write( /*==========================*/ buf_page_t* bpage, /*!< in/out: buffer page to be flushed */ - const byte* src_frame) /*!< in: src frame */ + const byte* src_frame, /*!< in: src frame */ + ulint space_id) /*!< in: space id */ { - bpage->encrypt_later = false; + fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space_id); - if (srv_encrypt_tables == FALSE) { + if (crypt_data != NULL && crypt_data->encryption == FIL_SPACE_ENCRYPTION_OFF) { + /* Encryption is disabled */ + return const_cast(src_frame); + } + + if (!srv_encrypt_tables && (crypt_data == NULL || crypt_data->encryption == FIL_SPACE_ENCRYPTION_DEFAULT)) { /* Encryption is disabled */ return const_cast(src_frame); } @@ -5638,103 +5728,63 @@ buf_page_encrypt_before_write( ulint zip_size = buf_page_get_zip_size(bpage); ulint page_size = (zip_size) ? zip_size : UNIV_PAGE_SIZE; + buf_pool_t* buf_pool = buf_pool_from_bpage(bpage); + bool page_compressed = fil_space_is_page_compressed(bpage->space); - /** - * TODO(jonaso): figure out more clever malloc strategy - * - * This implementation does a malloc/free per iop for encrypted - * tablespaces. Alternative strategies that have been considered are - * - * 1) use buf_block_alloc (i.e alloc from buffer pool) - * this does not work as buf_block_alloc will then be called - * when needing to flush a page, which might be triggered - * due to shortage of memory in buffer pool - * 2) allocate a buffer per fil_node_t - * this would break abstraction layers and has therfore not been - * considered a lot. - */ + /* Find free slot from temporary memory array */ + buf_tmp_buffer_t* slot = buf_pool_reserve_tmp_slot(buf_pool, page_compressed); + bpage->slot = slot; - if (bpage->crypt_buf_free == NULL) { - bpage->crypt_buf_free = (byte*)malloc(page_size*2); - // TODO: Is 4k aligment enough ? - bpage->crypt_buf = (byte *)ut_align(bpage->crypt_buf_free, page_size); - } + byte *dst_frame = slot->crypt_buf; - byte *dst_frame = bpage->crypt_buf; - - if (!fil_space_is_page_compressed(bpage->space)) { - // encrypt page content - fil_space_encrypt(bpage->space, bpage->offset, - bpage->newest_modification, - src_frame, zip_size, dst_frame, 0); + if (!page_compressed) { + /* Encrypt page content */ + fil_space_encrypt(bpage->space, + bpage->offset, + bpage->newest_modification, + src_frame, + zip_size, + dst_frame); unsigned key_version = mach_read_from_4(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); ut_ad(key_version == 0 || key_version >= bpage->key_version); bpage->key_version = key_version; + bpage->real_size = zip_size; } else { - /** Compression and encryption is done later at os0file.cc */ - bpage->encrypt_later = true; - dst_frame = (byte *)src_frame; + /* First we compress the page content */ + ulint out_len = 0; + ulint block_size = fil_space_get_block_size(bpage->space, bpage->offset, page_size); + + byte *tmp = fil_compress_page(bpage->space, + (byte *)src_frame, + slot->comp_buf, + page_size, + fil_space_get_page_compression_level(bpage->space), + block_size, + &out_len, +#ifdef HAVE_LZO + slot->lzo_mem +#else + NULL +#endif + ); + + bpage->real_size = out_len; + + /* And then we encrypt the page content */ + fil_space_encrypt(bpage->space, + bpage->offset, + bpage->newest_modification, + tmp, + zip_size, + dst_frame); } // return dst_frame which will be written return dst_frame; } -/********************************************************************//** -Release memory after encrypted page has been written to disk -*/ -ibool -buf_page_encrypt_after_write( -/*=========================*/ - buf_page_t* bpage) /*!< in/out: buffer page flushed */ -{ - if (bpage->crypt_buf_free != NULL) { - free(bpage->crypt_buf_free); - bpage->crypt_buf_free = NULL; - bpage->crypt_buf = NULL; - } - - if (bpage->comp_buf_free != NULL) { - free(bpage->comp_buf_free); - bpage->comp_buf_free = NULL; - bpage->comp_buf = NULL; - } - - return (TRUE); -} - -/********************************************************************//** -Allocates memory to read in an encrypted page -*/ -byte* -buf_page_decrypt_before_read( -/*=========================*/ - buf_page_t* bpage, /*!< in/out: buffer page to be read */ - ulint zip_size) /*!< in: compressed page size, or 0 */ -{ - ulint size = (zip_size) ? zip_size : UNIV_PAGE_SIZE; - - /* - Here we only need to allocate space for not header pages - in case of file space encryption. Table encryption is handled - later. - */ - if (!srv_encrypt_tables || bpage->offset == 0 || - fil_space_check_encryption_read(bpage->space) == false) - return zip_size ? bpage->zip.data : ((buf_block_t*) bpage)->frame; - - if (bpage->crypt_buf_free == NULL) - { - // allocate buffer to read data into - bpage->crypt_buf_free = (byte*)malloc(size*2); - // TODO: Is 4K aligment enough ? - bpage->crypt_buf = (byte*)ut_align(bpage->crypt_buf_free, size); - } - return bpage->crypt_buf; -} - /********************************************************************//** Decrypt page after it has been read from disk */ @@ -5752,96 +5802,58 @@ buf_page_decrypt_after_read( if (bpage->offset == 0) { /* File header pages are not encrypted */ - ut_a(bpage->crypt_buf == NULL); return (TRUE); } - - const byte* src_frame = bpage->crypt_buf != NULL ? - bpage->crypt_buf : dst_frame; - unsigned key_version = - mach_read_from_4(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); + mach_read_from_4(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); bool page_compressed = fil_page_is_compressed(dst_frame); + buf_pool_t* buf_pool = buf_pool_from_bpage(bpage); + if (key_version == 0) { /* the page we read is unencrypted */ - if (fil_page_is_compressed(dst_frame)) { - if (bpage->comp_buf_free == NULL) { - bpage->comp_buf_free = (byte *)malloc(UNIV_PAGE_SIZE*2); - // TODO: is 4k aligment enough ? - bpage->comp_buf = (byte*)ut_align(bpage->comp_buf_free, UNIV_PAGE_SIZE); - } + if (page_compressed) { + /* Find free slot from temporary memory array */ + buf_tmp_buffer_t* slot = buf_pool_reserve_tmp_slot(buf_pool, page_compressed); - fil_decompress_page(bpage->comp_buf, dst_frame, size, NULL); - } else { - if (dst_frame != src_frame) { - /* but we had allocated a crypt_buf */ - // TODO: Can this be avoided ? - memcpy(dst_frame, src_frame, size); - } + fil_decompress_page(slot->comp_buf, + dst_frame, + size, + &bpage->write_size); + + /* Mark this slot as free */ + slot->reserved = false; } } else { /* the page we read is encrypted */ - if (dst_frame == src_frame) { - /* but we had NOT allocated a crypt buf - * malloc a buffer, copy page to it - * and then decrypt from that into real page*/ - bpage->crypt_buf_free = (byte *)malloc(UNIV_PAGE_SIZE*2); - // TODO: is 4k aligment enough ? - src_frame = bpage->crypt_buf = (byte*)ut_align(bpage->crypt_buf_free, UNIV_PAGE_SIZE); - memcpy(bpage->crypt_buf, dst_frame, size); - } + /* Find free slot from temporary memory array */ + buf_tmp_buffer_t* slot = buf_pool_reserve_tmp_slot(buf_pool, page_compressed); - /* decrypt from src_frame to dst_frame */ + memcpy(slot->crypt_buf, dst_frame, size); + + /* decrypt from crypt_buf to dst_frame */ fil_space_decrypt(bpage->space, - src_frame, size, dst_frame); + slot->crypt_buf, + size, + dst_frame); /* decompress from dst_frame to comp_buf and then copy to buffer pool */ if (page_compressed) { - if (bpage->comp_buf_free == NULL) { - bpage->comp_buf_free = (byte *)malloc(UNIV_PAGE_SIZE*2); - // TODO: is 4k aligment enough ? - bpage->comp_buf = (byte*)ut_align(bpage->comp_buf_free, UNIV_PAGE_SIZE); - } - - fil_decompress_page(bpage->comp_buf, dst_frame, size, NULL); + fil_decompress_page(slot->comp_buf, + dst_frame, + size, + &bpage->write_size); } + + /* Mark this slot as free */ + slot->reserved = false; } bpage->key_version = key_version; - if (bpage->crypt_buf_free != NULL) { - // free temp page - free(bpage->crypt_buf_free); - bpage->crypt_buf = NULL; - bpage->crypt_buf_free = NULL; - } - - if (bpage->comp_buf_free != NULL) { - // free temp page - free(bpage->comp_buf_free); - bpage->comp_buf = NULL; - bpage->comp_buf_free = NULL; - } - return (TRUE); } - -/********************************************************************//** -Release memory allocated for decryption -*/ -void -buf_page_decrypt_cleanup( -/*=====================*/ - buf_page_t* bpage) /*!< in/out: buffer page */ -{ - if (bpage->crypt_buf != NULL) { - free(bpage->crypt_buf_free); - bpage->crypt_buf = NULL; - bpage->crypt_buf_free = NULL; - } -} diff --git a/storage/innobase/buf/buf0dblwr.cc b/storage/innobase/buf/buf0dblwr.cc index 53ef6ecb316..03681e8fbeb 100644 --- a/storage/innobase/buf/buf0dblwr.cc +++ b/storage/innobase/buf/buf0dblwr.cc @@ -872,7 +872,7 @@ buf_dblwr_write_block_to_datafile( 0, buf_block_get_page_no(block), 0, - UNIV_PAGE_SIZE, + bpage->real_size, frame, (void*) block, (ulint *)&bpage->write_size); @@ -1231,7 +1231,7 @@ retry: TRX_SYS_SPACE, 0, offset, 0, - UNIV_PAGE_SIZE, + bpage->real_size, frame, NULL, 0); diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index b8a34558867..d893d424b02 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -909,7 +909,7 @@ buf_flush_write_block_low( break; } - frame = buf_page_encrypt_before_write(bpage, frame); + frame = buf_page_encrypt_before_write(bpage, frame, space_id); if (!srv_use_doublewrite_buf || !buf_dblwr) { fil_io(OS_FILE_WRITE | OS_AIO_SIMULATED_WAKE_LATER, @@ -918,7 +918,7 @@ buf_flush_write_block_low( zip_size, buf_page_get_page_no(bpage), 0, - zip_size ? zip_size : UNIV_PAGE_SIZE, + zip_size ? zip_size : bpage->real_size, frame, bpage, &bpage->write_size); @@ -938,7 +938,7 @@ buf_flush_write_block_low( zip_size, buf_page_get_page_no(bpage), 0, - zip_size ? zip_size : UNIV_PAGE_SIZE, + zip_size ? zip_size : bpage->real_size, frame, bpage, &bpage->write_size); diff --git a/storage/innobase/buf/buf0rea.cc b/storage/innobase/buf/buf0rea.cc index 5917af7ca9d..b4b474c547f 100644 --- a/storage/innobase/buf/buf0rea.cc +++ b/storage/innobase/buf/buf0rea.cc @@ -176,7 +176,7 @@ buf_read_page_low( ut_ad(buf_page_in_file(bpage)); - byte* frame = buf_page_decrypt_before_read(bpage, zip_size); + byte* frame = zip_size ? bpage->zip.data : ((buf_block_t*) bpage)->frame; if (sync) { thd_wait_begin(NULL, THD_WAIT_DISKIO); @@ -202,7 +202,6 @@ buf_read_page_low( } if (*err != DB_SUCCESS) { - buf_page_decrypt_cleanup(bpage); if (ignore_nonexistent_pages || *err == DB_TABLESPACE_DELETED) { buf_read_page_handle_error(bpage); return(0); diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index aab0b1f5568..07636d27879 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -319,6 +319,7 @@ fil_space_create_crypt_data() &crypt_data->mutex, SYNC_NO_ORDER_CHECK); crypt_data->iv_length = iv_length; my_random_bytes(crypt_data->iv, iv_length); + crypt_data->encryption = FIL_SPACE_ENCRYPTION_DEFAULT; return crypt_data; } @@ -421,6 +422,9 @@ fil_space_read_crypt_data( uint min_key_version = mach_read_from_4 (page + offset + MAGIC_SZ + 2 + iv_length); + fil_encryption_t encryption = (fil_encryption_t)mach_read_from_1( + page + offset + MAGIC_SZ + 2 + iv_length + 4); + const uint sz = sizeof(fil_space_crypt_t) + iv_length; fil_space_crypt_t* crypt_data = static_cast( malloc(sz)); @@ -429,6 +433,7 @@ fil_space_read_crypt_data( crypt_data->type = type; crypt_data->min_key_version = min_key_version; crypt_data->page0_offset = offset; + crypt_data->encryption = encryption; mutex_create(fil_crypt_data_mutex_key, &crypt_data->mutex, SYNC_NO_ORDER_CHECK); crypt_data->iv_length = iv_length; @@ -474,8 +479,9 @@ fil_space_write_crypt_data_low( page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); const uint len = crypt_data->iv_length; const uint min_key_version = crypt_data->min_key_version; + const fil_encryption_t encryption = crypt_data->encryption; crypt_data->page0_offset = offset; - ut_a(2 + len + 4 + MAGIC_SZ < maxsize); + ut_a(2 + len + 4 + 1 + MAGIC_SZ < maxsize); /* redo log this as bytewise updates to page 0 @@ -489,8 +495,10 @@ fil_space_write_crypt_data_low( mtr); mlog_write_ulint(page + offset + MAGIC_SZ + 2 + len, min_key_version, MLOG_4BYTES, mtr); + mlog_write_ulint(page + offset + MAGIC_SZ + 2 + len + 4, encryption, + MLOG_1BYTE, mtr); - byte* log_ptr = mlog_open(mtr, 11 + 12 + len); + byte* log_ptr = mlog_open(mtr, 11 + 13 + len); if (log_ptr != NULL) { log_ptr = mlog_write_initial_log_record_fast( @@ -507,6 +515,8 @@ fil_space_write_crypt_data_low( log_ptr += 1; mach_write_to_4(log_ptr, min_key_version); log_ptr += 4; + mach_write_to_1(log_ptr, encryption); + log_ptr += 1; mlog_close(mtr, log_ptr); mlog_catenate_string(mtr, crypt_data->iv, len); @@ -555,7 +565,8 @@ fil_parse_write_crypt_data( 2 + // size of offset 1 + // size of type 1 + // size of iv-len - 4; // size of min_key_version + 4 + // size of min_key_version + 1; // fil_encryption_t if (end_ptr - ptr < entry_size){ return NULL; @@ -582,9 +593,17 @@ fil_parse_write_crypt_data( return NULL; } + fil_encryption_t encryption = (fil_encryption_t)mach_read_from_1(ptr); + ptr +=1; + + if (end_ptr - ptr < len) { + return NULL; + } + fil_space_crypt_t* crypt_data = fil_space_create_crypt_data(); crypt_data->page0_offset = offset; crypt_data->min_key_version = min_key_version; + crypt_data->encryption = encryption; memcpy(crypt_data->iv, ptr, len); ptr += len; @@ -610,7 +629,8 @@ fil_space_clear_crypt_data( 1 + // type 1 + // len len + // iv - 4; // min key version + 4 + // min key version + 1; // fil_encryption_t memset(page + offset, 0, size); } @@ -674,13 +694,11 @@ fil_space_encrypt( const byte* src_frame, /*!< in: Source page to be encrypted */ ulint zip_size, /*!< in: compressed size if row_format compressed */ - byte* dst_frame, /*!< in: outbut buffer */ - ulint encryption_key) /*!< in: encryption key id if page - encrypted */ + byte* dst_frame) /*!< in: outbut buffer */ { fil_space_crypt_t* crypt_data=NULL; ulint page_size = (zip_size) ? zip_size : UNIV_PAGE_SIZE; - uint key_version = (uint)encryption_key; + uint key_version; unsigned char key[MY_AES_MAX_KEY_LENGTH]; uint key_length=MY_AES_MAX_KEY_LENGTH; uint aes_method; @@ -699,6 +717,7 @@ fil_space_encrypt( /* Get crypt data from file space */ crypt_data = fil_space_get_crypt_data(space); + key_version = crypt_data->keys[0].key_id; if (crypt_data == NULL) { //TODO: Is this really needed ? @@ -808,6 +827,10 @@ fil_space_check_encryption_read( return false; } + if (crypt_data->encryption == FIL_SPACE_ENCRYPTION_OFF) { + return false; + } + return true; } @@ -1352,6 +1375,11 @@ fil_crypt_space_needs_rotation( mutex_enter(&crypt_data->mutex); do { + if (crypt_data->encryption == FIL_SPACE_ENCRYPTION_OFF) { + /* This space is unencrypted by user request */ + break; + } + /* prevent threads from starting to rotate space */ if (crypt_data->rotate_state.starting) { /* recheck this space later */ diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 9d37de6d63b..9d0bb03c0e9 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -5543,6 +5543,74 @@ fil_report_invalid_page_access( (ulong) byte_offset, (ulong) len, (ulong) type); } +/********************************************************************//** +Find correct node from file space +@return node */ +static +fil_node_t* +fil_space_get_node( + fil_space_t* space, /*!< in: file spage */ + ulint space_id, /*!< in: space id */ + ulint* block_offset, /*!< in/out: offset in number of blocks */ + ulint byte_offset, /*!< in: remainder of offset in bytes; in + aio this must be divisible by the OS block + size */ + ulint len) /*!< in: how many bytes to read or write; this + must not cross a file boundary; in aio this + must be a block size multiple */ +{ + fil_node_t* node; + ut_ad(mutex_own(&fil_system->mutex)); + + node = UT_LIST_GET_FIRST(space->chain); + + for (;;) { + if (node == NULL) { + return(NULL); + } else if (fil_is_user_tablespace_id(space->id) + && node->size == 0) { + + /* We do not know the size of a single-table tablespace + before we open the file */ + break; + } else if (node->size > *block_offset) { + /* Found! */ + break; + } else { + *block_offset -= node->size; + node = UT_LIST_GET_NEXT(chain, node); + } + } + + return (node); +} +/********************************************************************//** +Return block size of node in file space +@return file block size */ +UNIV_INTERN +ulint +fil_space_get_block_size( +/*=====================*/ + ulint space_id, + ulint block_offset, + ulint len) +{ + ulint block_size = 512; + fil_space_t* space = fil_space_get_space(space_id); + + if (space) { + mutex_enter(&fil_system->mutex); + fil_node_t* node = fil_space_get_node(space, space_id, &block_offset, 0, len); + mutex_exit(&fil_system->mutex); + + if (node) { + block_size = node->file_block_size; + } + } + + return block_size; +} + /********************************************************************//** Reads or writes data. This operation is asynchronous (aio). @return DB_SUCCESS, or DB_TABLESPACE_DELETED if we are trying to do @@ -5589,7 +5657,7 @@ fil_io( ulint is_log; ulint wake_later; os_offset_t offset; - ibool ignore_nonexistent_pages; + bool ignore_nonexistent_pages; is_log = type & OS_FILE_LOG; type = type & ~OS_FILE_LOG; @@ -5674,34 +5742,18 @@ fil_io( ut_ad(mode != OS_AIO_IBUF || space->purpose == FIL_TABLESPACE); - node = UT_LIST_GET_FIRST(space->chain); + node = fil_space_get_node(space, space_id, &block_offset, byte_offset, len); - for (;;) { - if (node == NULL) { - if (ignore_nonexistent_pages) { - mutex_exit(&fil_system->mutex); - return(DB_ERROR); - } - - fil_report_invalid_page_access( + if (!node) { + if (ignore_nonexistent_pages) { + mutex_exit(&fil_system->mutex); + return(DB_ERROR); + } + fil_report_invalid_page_access( block_offset, space_id, space->name, byte_offset, len, type); - ut_error; - - } else if (fil_is_user_tablespace_id(space->id) - && node->size == 0) { - - /* We do not know the size of a single-table tablespace - before we open the file */ - break; - } else if (node->size > block_offset) { - /* Found! */ - break; - } else { - block_offset -= node->size; - node = UT_LIST_GET_NEXT(chain, node); - } + ut_error; } /* Open file if closed */ diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 311fdee4d3b..bd2d49995a0 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -566,10 +566,10 @@ ha_create_table_option innodb_table_option_list[]= HA_TOPTION_NUMBER("PAGE_COMPRESSION_LEVEL", page_compression_level, 0, 1, 9, 1), /* With this option user can enable atomic writes feature for this table */ HA_TOPTION_ENUM("ATOMIC_WRITES", atomic_writes, "DEFAULT,ON,OFF", 0), - /* With this option the user can enable page encryption for the table */ - HA_TOPTION_BOOL("PAGE_ENCRYPTION", page_encryption, 0), + /* With this option the user can enable encryption for the table */ + HA_TOPTION_ENUM("ENCRYPTION", encryption, "DEFAULT, ON, OFF", 0), /* With this option the user defines the key identifier using for the encryption */ - HA_TOPTION_NUMBER("PAGE_ENCRYPTION_KEY", page_encryption_key, 0, 1, 255, 1), + HA_TOPTION_NUMBER("ENCRYPTION_KEY", encryption_key, 0, 1, UINT_MAX32, 1), HA_TOPTION_END }; @@ -11285,20 +11285,21 @@ ha_innobase::check_table_options( enum row_type row_format = table->s->row_type; ha_table_option_struct *options= table->s->option_struct; atomic_writes_t awrites = (atomic_writes_t)options->atomic_writes; + fil_encryption_t encrypt = (fil_encryption_t)options->encryption; - if (options->page_encryption) { + if (encrypt == FIL_SPACE_ENCRYPTION_ON) { if (srv_encrypt_tables) { push_warning( thd, Sql_condition::WARN_LEVEL_WARN, HA_WRONG_CREATE_OPTION, - "InnoDB: PAGE_ENCRYPTION not available if innodb_encrypt_tables=ON"); + "InnoDB: ENCRYPTION not available if innodb_encrypt_tables=ON"); return "INNODB_ENCRYPT_TABLES"; } if (!use_tablespace) { push_warning( thd, Sql_condition::WARN_LEVEL_WARN, HA_WRONG_CREATE_OPTION, - "InnoDB: PAGE_ENCRYPTION requires" + "InnoDB: ENCRYPTION requires" " innodb_file_per_table."); return "PAGE_ENCRYPTION"; } @@ -11307,14 +11308,6 @@ ha_innobase::check_table_options( /* Check page compression requirements */ if (options->page_compressed) { - if (srv_encrypt_tables) { - push_warning( - thd, Sql_condition::WARN_LEVEL_WARN, - HA_WRONG_CREATE_OPTION, - "InnoDB: PAGE_COMPRESSION not available if innodb_encrypt_tables=ON"); - return "PAGE_COMPRESSED"; - } - if (row_format == ROW_TYPE_COMPRESSED) { push_warning( thd, Sql_condition::WARN_LEVEL_WARN, @@ -11384,29 +11377,29 @@ ha_innobase::check_table_options( } } - if (options->page_encryption_key != 0) { - if (options->page_encryption == false) { + if (options->encryption_key != 0) { + if (options->encryption == FIL_SPACE_ENCRYPTION_OFF) { /* ignore this to allow alter table without changing page_encryption_key ...*/ } - if (options->page_encryption_key < 1 || options->page_encryption_key > 255) { + if (options->encryption_key < 1) { push_warning_printf( thd, Sql_condition::WARN_LEVEL_WARN, HA_WRONG_CREATE_OPTION, - "InnoDB: invalid PAGE_ENCRYPTION_KEY = %lu." - " Valid values are [1..255]", - options->page_encryption_key); - return "PAGE_ENCRYPTION_KEY"; + "InnoDB: invalid ENCRYPTION_KEY = %lu." + " Valid values are [1..INT32_MAX]", + options->encryption_key); + return "ENCRYPTION_KEY"; } - if (!has_encryption_key(options->page_encryption_key)) { + if (!has_encryption_key(options->encryption_key)) { push_warning_printf( thd, Sql_condition::WARN_LEVEL_WARN, HA_WRONG_CREATE_OPTION, - "InnoDB: PAGE_ENCRYPTION_KEY encryption key %lu not available", - options->page_encryption_key + "InnoDB: ENCRYPTION_KEY encryption key %lu not available", + options->encryption_key ); - return "PAGE_ENCRYPTION_KEY"; + return "ENCRYPTION_KEY"; } } @@ -11467,6 +11460,11 @@ ha_innobase::create( const char* stmt; size_t stmt_len; + /* Cache table options */ + ha_table_option_struct *options= table->s->option_struct; + fil_encryption_t encrypt = (fil_encryption_t)options->encryption; + ulint key_id = (options->encryption_key == 0) ? srv_default_encryption_key : + options->encryption_key; DBUG_ENTER("ha_innobase::create"); @@ -11706,6 +11704,18 @@ ha_innobase::create( innobase_commit_low(trx); + /* If user has requested that table should be encrypted or table + should remain as unencrypted store crypt data */ + if (encrypt == FIL_SPACE_ENCRYPTION_ON || encrypt == FIL_SPACE_ENCRYPTION_OFF) { + ulint maxsize; + ulint zip_size = fil_space_get_zip_size(innobase_table->space); + fil_space_crypt_t* crypt_data = fil_space_create_crypt_data(); + crypt_data->page0_offset = fsp_header_get_crypt_offset(zip_size, &maxsize); + crypt_data->keys[0].key_id = key_id; + crypt_data->encryption = encrypt; + fil_space_set_crypt_data(innobase_table->space, crypt_data); + } + row_mysql_unlock_data_dictionary(trx); /* Flush the log to reduce probability that the .frm files and @@ -19146,12 +19156,12 @@ static MYSQL_SYSVAR_UINT(encryption_rotation_iops, srv_n_fil_crypt_iops, innodb_encryption_rotation_iops_update, srv_n_fil_crypt_iops, 0, UINT_MAX32, 0); -static MYSQL_SYSVAR_UINT(default_page_encryption_key, srv_default_page_encryption_key, +static MYSQL_SYSVAR_UINT(default_encryption_key, srv_default_encryption_key, PLUGIN_VAR_RQCMDARG, - "Encryption key used for page encryption.", + "Default encryption key used for table encryption.", NULL, NULL, - FIL_DEFAULT_ENCRYPTION_KEY, 1, 255, 0); + FIL_DEFAULT_ENCRYPTION_KEY, 1, UINT_MAX32, 0); static MYSQL_SYSVAR_BOOL(scrub_log, srv_scrub_log, PLUGIN_VAR_OPCMDARG | PLUGIN_VAR_READONLY, @@ -19420,7 +19430,7 @@ static struct st_mysql_sys_var* innobase_system_variables[]= { MYSQL_SYSVAR(scrub_log), MYSQL_SYSVAR(scrub_log_interval), MYSQL_SYSVAR(encrypt_log), - MYSQL_SYSVAR(default_page_encryption_key), + MYSQL_SYSVAR(default_encryption_key), /* Scrubing feature */ MYSQL_SYSVAR(immediate_scrub_data_uncompressed), diff --git a/storage/innobase/handler/ha_innodb.h b/storage/innobase/handler/ha_innodb.h index 7807c7ca7e6..2b9f1506339 100644 --- a/storage/innobase/handler/ha_innodb.h +++ b/storage/innobase/handler/ha_innodb.h @@ -70,8 +70,8 @@ struct ha_table_option_struct srv_use_atomic_writes=1. Atomic writes are not used if value OFF.*/ - bool page_encryption; /*!< Flag for an encrypted table */ - int page_encryption_key; /*!< ID of the encryption key */ + uint encryption; /*!< DEFAULT, ON, OFF */ + int encryption_key; /*!< encryption key id*/ }; diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index 94f17e75358..327e4e92d1b 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -278,8 +278,8 @@ ha_innobase::check_if_supported_inplace_alter( DBUG_RETURN(HA_ALTER_INPLACE_NOT_SUPPORTED); } - if (new_options->page_encryption != old_options->page_encryption || - new_options->page_encryption_key != old_options->page_encryption_key) { + if (new_options->encryption != old_options->encryption || + new_options->encryption_key != old_options->encryption_key) { ha_alter_info->unsupported_reason = innobase_get_err_msg( ER_ALTER_OPERATION_NOT_SUPPORTED_REASON); DBUG_RETURN(HA_ALTER_INPLACE_NOT_SUPPORTED); diff --git a/storage/innobase/include/buf0buf.h b/storage/innobase/include/buf0buf.h index 1c84e68167e..d22d344162a 100644 --- a/storage/innobase/include/buf0buf.h +++ b/storage/innobase/include/buf0buf.h @@ -1445,16 +1445,19 @@ buf_flush_update_zip_checksum( The hook that is called just before a page is written to disk. The function encrypts the content of the page and returns a pointer to a frame that will be written instead of the real frame. */ +UNIV_INTERN byte* buf_page_encrypt_before_write( /*==========================*/ buf_page_t* page, /*!< in/out: buffer page to be flushed */ - const byte* frame); + const byte* frame, + ulint space_id); /********************************************************************** The hook that is called after page is written to disk. The function releases any resources needed for encryption that was allocated in buf_page_encrypt_before_write */ +UNIV_INTERN ibool buf_page_encrypt_after_write( /*=========================*/ @@ -1464,6 +1467,7 @@ buf_page_encrypt_after_write( The hook that is called just before a page is read from disk. The function allocates memory that is used to temporarily store disk content before getting decrypted */ +UNIV_INTERN byte* buf_page_decrypt_before_read( /*=========================*/ @@ -1474,19 +1478,35 @@ buf_page_decrypt_before_read( The hook that is called just after a page is read from disk. The function decrypt disk content into buf_page_t and releases the temporary buffer that was allocated in buf_page_decrypt_before_read */ +UNIV_INTERN ibool buf_page_decrypt_after_read( /*========================*/ buf_page_t* page); /*!< in/out: buffer page read from disk */ -/********************************************************************//** -Release memory allocated for page decryption. -Only used in scenarios where read fails, e.g due to tablespace being dropped */ -void -buf_page_decrypt_cleanup( -/*=====================*/ - buf_page_t* page); /*!< in/out: buffer page read from disk */ +/** @brief The temporary memory structure. +NOTE! The definition appears here only for other modules of this +directory (buf) to see it. Do not use from outside! */ + +typedef struct { + bool reserved; /*!< true if this slot is reserved + */ +#ifdef HAVE_LZO + byte* lzo_mem; /*!< Temporal memory used by LZO */ +#endif + byte* crypt_buf; /*!< for encryption the data needs to be + copied to a separate buffer before it's + encrypted&written. this as a page can be + read while it's being flushed */ + byte* crypt_buf_free; /*!< for encryption, allocated buffer + that is then alligned */ + byte* comp_buf; /*!< for compression we need + temporal buffer because page + can be read while it's being flushed */ + byte* comp_buf_free; /*!< for compression, allocated + buffer that is then alligned */ +} buf_tmp_buffer_t; /** The common buffer control block structure for compressed and uncompressed frames */ @@ -1559,20 +1579,15 @@ struct buf_page_t{ operation needed. */ unsigned key_version; /*!< key version for this block */ - byte* crypt_buf; /*!< for encryption the data needs to be - copied to a separate buffer before it's - encrypted&written. this as a page can be - read while it's being flushed */ - byte* crypt_buf_free; /*!< for encryption, allocated buffer - that is then alligned */ - byte* comp_buf; /*!< for compression we need - temporal buffer because page - can be read while it's being flushed */ - byte* comp_buf_free; /*!< for compression, allocated - buffer that is then alligned */ - bool encrypt_later; /*!< should we encrypt the page - at os0file.cc ? */ + ulint real_size; /*!< Real size of the page + Normal pages == UNIV_PAGE_SIZE + page compressed pages, payload + size alligned to sector boundary. + */ + buf_tmp_buffer_t* slot; /*!< Slot for temporary memory + used for encryption/compression + or NULL */ #ifndef UNIV_HOTBACKUP buf_page_t* hash; /*!< node used in chaining to buf_pool->page_hash or @@ -2017,6 +2032,17 @@ struct buf_buddy_stat_t { ib_uint64_t relocated_usec; }; +/** @brief The temporary memory array structure. + +NOTE! The definition appears here only for other modules of this +directory (buf) to see it. Do not use from outside! */ + +typedef struct { + ulint n_slots; /*!< Total number of slots */ + buf_tmp_buffer_t *slots; /*!< Pointer to the slots in the + array */ +} buf_tmp_array_t; + /** @brief The buffer pool structure. NOTE! The definition appears here only for other modules of this @@ -2200,6 +2226,10 @@ struct buf_pool_t{ pool watches. Protected by buf_pool->mutex. */ + buf_tmp_array_t* tmp_arr; + /*!< Array for temporal memory + used in compression and encryption */ + #if BUF_BUDDY_LOW > UNIV_ZIP_SIZE_MIN # error "BUF_BUDDY_LOW > UNIV_ZIP_SIZE_MIN" #endif diff --git a/storage/innobase/include/buf0buf.ic b/storage/innobase/include/buf0buf.ic index 842aa6758cd..53e7445d447 100644 --- a/storage/innobase/include/buf0buf.ic +++ b/storage/innobase/include/buf0buf.ic @@ -1464,28 +1464,15 @@ buf_get_nth_chunk_block( return(chunk->blocks); } -/********************************************************************//** -Get crypt buffer. */ -UNIV_INLINE -byte* -buf_page_get_crypt_buffer( -/*=========================*/ - const buf_page_t* bpage) /*!< in: buffer pool page */ -{ - return bpage->crypt_buf; -} - /********************************************************************//** Get buf frame. */ UNIV_INLINE void * buf_page_get_frame( -/*=========================*/ +/*===============*/ const buf_page_t* bpage) /*!< in: buffer pool page */ { - if (bpage->crypt_buf) { - return buf_page_get_crypt_buffer(bpage); - } else if (bpage->zip.data) { + if (bpage->zip.data) { return bpage->zip.data; } else { return ((buf_block_t*) bpage)->frame; diff --git a/storage/innobase/include/fil0crypt.h b/storage/innobase/include/fil0crypt.h index 9d02034e4b2..359533dc50a 100644 --- a/storage/innobase/include/fil0crypt.h +++ b/storage/innobase/include/fil0crypt.h @@ -37,6 +37,14 @@ Created 04/01/2015 Jan Lindström /* This key will be used if nothing else is given */ #define FIL_DEFAULT_ENCRYPTION_KEY 1 +/** Enum values for encryption table option */ +typedef enum { + FIL_SPACE_ENCRYPTION_DEFAULT = 0, /* Tablespace encrypted if + srv_encrypt_tables = ON */ + FIL_SPACE_ENCRYPTION_ON = 1, /* Tablespace is encrypted always */ + FIL_SPACE_ENCRYPTION_OFF = 2 /* Tablespace is not encrypted */ +} fil_encryption_t; + /** * CRYPT_SCHEME_UNENCRYPTED * @@ -67,6 +75,8 @@ struct key_struct { uint key_version; /*!< Key version used as identifier */ + uint key_id; /*1< Key id used as + identifier */ byte key[MY_AES_MAX_KEY_LENGTH]; /*!< Cached L or key */ uint key_length; /*!< Key length */ }; @@ -95,6 +105,7 @@ struct fil_space_crypt_struct key_struct keys[3]; // cached L = AES_ECB(KEY, IV) uint min_key_version; // min key version for this space ulint page0_offset; // byte offset on page 0 for crypt data + fil_encryption_t encryption; // Encryption setup ib_mutex_t mutex; // mutex protecting following variables bool closing; // is tablespace being closed @@ -227,9 +238,7 @@ fil_space_encrypt( lsn_t lsn, /*!< in: page lsn */ const byte* src_frame,/*!< in: page frame */ ulint size, /*!< in: size of data to encrypt */ - byte* dst_frame, /*!< in: where to encrypt to */ - ulint page_encryption_key); /*!< in: page encryption key id if page - encrypted */ + byte* dst_frame); /*!< in: where to encrypt to */ /********************************************************************* Decrypt buffer page */ diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index 2c6c5804f65..7f2c00eb168 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -1272,6 +1272,16 @@ ulint fil_get_next_space( ulint id); /*!< in: space id */ +/*******************************************************************//** +Returns the block size of the file space +@return block size */ +UNIV_INTERN +ulint +fil_space_get_block_size( +/*=====================*/ + ulint id, /*!< in: space id */ + ulint offset, /*!< in: page offset */ + ulint len); /*!< in: page len */ #endif diff --git a/storage/innobase/include/srv0srv.h b/storage/innobase/include/srv0srv.h index 31db78ae5ee..314bce63c7b 100644 --- a/storage/innobase/include/srv0srv.h +++ b/storage/innobase/include/srv0srv.h @@ -562,8 +562,8 @@ that semaphore times out in InnoDB */ #define DEFAULT_SRV_FATAL_SEMAPHORE_TIMEOUT 600 extern ulong srv_fatal_semaphore_wait_threshold; -/** Default encryption key used for page encryption */ -extern uint srv_default_page_encryption_key; +/** Default encryption key used for tablespace encryption */ +extern uint srv_default_encryption_key; /** Enable semaphore request instrumentation */ extern my_bool srv_instrument_semaphores; diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index 1287ee76819..e17cf161973 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -242,7 +242,6 @@ struct os_aio_slot_t{ int n_bytes; /* bytes written/read. */ int ret; /* AIO return code */ #endif /* WIN_ASYNC_IO */ - byte *lzo_mem; /* Temporal memory used by LZO */ }; /** The asynchronous i/o array structure */ diff --git a/storage/innobase/srv/srv0srv.cc b/storage/innobase/srv/srv0srv.cc index 78954725001..d8c921fb612 100644 --- a/storage/innobase/srv/srv0srv.cc +++ b/storage/innobase/srv/srv0srv.cc @@ -523,8 +523,8 @@ thread ensures that we flush the log files at least once per second. */ static time_t srv_last_log_flush_time; -/** Default encryption key used for page encryption */ -UNIV_INTERN uint srv_default_page_encryption_key = FIL_DEFAULT_ENCRYPTION_KEY; +/** Default encryption key used for tablespace encryption */ +UNIV_INTERN uint srv_default_encryption_key = FIL_DEFAULT_ENCRYPTION_KEY; /** Enable semaphore request instrumentation */ UNIV_INTERN my_bool srv_instrument_semaphores = FALSE; diff --git a/storage/xtradb/buf/buf0buf.cc b/storage/xtradb/buf/buf0buf.cc index 56b0245da93..ab8dbadf0fa 100644 --- a/storage/xtradb/buf/buf0buf.cc +++ b/storage/xtradb/buf/buf0buf.cc @@ -95,6 +95,14 @@ _increment_page_get_statistics(buf_block_t* block, trx_t* trx) return; } +#ifdef HAVE_LZO +#include "lzo/lzo1x.h" +#endif + +/* Number of temporary slots used for encryption/compression +memory allocation before/after I/O operations */ +#define BUF_MAX_TMP_SLOTS 200 + /* IMPLEMENTATION OF THE BUFFER POOL ================================= @@ -1077,15 +1085,11 @@ buf_block_init( block->page.state = BUF_BLOCK_NOT_USED; block->page.buf_fix_count = 0; block->page.io_fix = BUF_IO_NONE; - block->page.crypt_buf = NULL; - block->page.crypt_buf_free = NULL; - block->page.comp_buf = NULL; - block->page.comp_buf_free = NULL; block->page.key_version = 0; - block->page.encrypt_later = false; - - + block->page.real_size = 0; + block->page.write_size = 0; block->modify_clock = 0; + block->page.slot = NULL; #if defined UNIV_DEBUG_FILE_ACCESSES || defined UNIV_DEBUG block->page.file_page_was_freed = FALSE; @@ -1438,6 +1442,11 @@ buf_pool_init_instance( /* All fields are initialized by mem_zalloc(). */ + /* Initialize the temporal memory array and slots */ + buf_pool->tmp_arr = (buf_tmp_array_t *)mem_zalloc(sizeof(buf_tmp_array_t)); + buf_pool->tmp_arr->n_slots = BUF_MAX_TMP_SLOTS; + buf_pool->tmp_arr->slots = (buf_tmp_buffer_t*)mem_zalloc(sizeof(buf_tmp_buffer_t) * BUF_MAX_TMP_SLOTS); + buf_pool->try_LRU_scan = TRUE; return(DB_SUCCESS); @@ -1489,6 +1498,9 @@ buf_pool_free_instance( ha_clear(buf_pool->page_hash); hash_table_free(buf_pool->page_hash); hash_table_free(buf_pool->zip_hash); + + mem_free(buf_pool->tmp_arr->slots); + mem_free(buf_pool->tmp_arr); } /********************************************************************//** @@ -3574,12 +3586,8 @@ buf_page_init_low( bpage->newest_modification = 0; bpage->oldest_modification = 0; bpage->write_size = 0; - bpage->crypt_buf = NULL; - bpage->crypt_buf_free = NULL; - bpage->comp_buf = NULL; - bpage->comp_buf_free = NULL; bpage->key_version = 0; - bpage->encrypt_later = false; + bpage->real_size = 0; HASH_INVALIDATE(bpage, hash); bpage->is_corrupt = FALSE; @@ -4498,7 +4506,11 @@ corrupt: } } else { /* io_type == BUF_IO_WRITE */ - buf_page_encrypt_after_write(bpage); + if (bpage->slot) { + /* Mark slot free */ + bpage->slot->reserved = false; + bpage->slot = NULL; + } } if (io_type == BUF_IO_WRITE @@ -5778,6 +5790,60 @@ buf_pool_mutex_exit( mutex_exit(&buf_pool->LRU_list_mutex); } +/********************************************************************//** +Reserve unused slot from temporary memory array and allocate necessary +temporary memory if not yet allocated. +@return reserved slot */ +buf_tmp_buffer_t* +buf_pool_reserve_tmp_slot( +/*======================*/ + buf_pool_t* buf_pool, /*!< in: buffer pool where to + reserve */ + bool compressed) /*!< in: is file space compressed */ +{ + buf_tmp_buffer_t *free_slot=NULL; + + /* Array is protected by buf_pool mutex */ + buf_pool_mutex_enter(buf_pool); + + for(ulint i = 0; i < buf_pool->tmp_arr->n_slots; i++) { + buf_tmp_buffer_t *slot = &buf_pool->tmp_arr->slots[i]; + + if(slot->reserved == false) { + free_slot = slot; + break; + } + } + + /* We assume that free slot is found */ + ut_a(free_slot != NULL); + free_slot->reserved = true; + /* Now that we have reserved this slot we can release + buf_pool mutex */ + buf_pool_mutex_exit(buf_pool); + + /* Allocate temporary memory for encryption/decryption */ + if (free_slot->crypt_buf_free == NULL) { + free_slot->crypt_buf_free = static_cast(ut_malloc(UNIV_PAGE_SIZE*2)); + free_slot->crypt_buf = static_cast(ut_align(free_slot->crypt_buf_free, UNIV_PAGE_SIZE)); + memset(free_slot->crypt_buf_free, 0, UNIV_PAGE_SIZE *2); + } + + /* For page compressed tables allocate temporary memory for + compression/decompression */ + if (compressed && free_slot->comp_buf_free == NULL) { + free_slot->comp_buf_free = static_cast(ut_malloc(UNIV_PAGE_SIZE*2)); + free_slot->comp_buf = static_cast(ut_align(free_slot->comp_buf_free, UNIV_PAGE_SIZE)); + memset(free_slot->comp_buf_free, 0, UNIV_PAGE_SIZE *2); +#ifdef HAVE_LZO + free_slot->lzo_mem = static_cast(ut_malloc(LZO1X_1_15_MEM_COMPRESS)); + memset(free_slot->lzo_mem, 0, LZO1X_1_15_MEM_COMPRESS); +#endif + } + + return (free_slot); +} + /********************************************************************//** Encrypts a buffer page right before it's flushed to disk */ @@ -5785,11 +5851,17 @@ byte* buf_page_encrypt_before_write( /*==========================*/ buf_page_t* bpage, /*!< in/out: buffer page to be flushed */ - const byte* src_frame) /*!< in: src frame */ + const byte* src_frame, /*!< in: src frame */ + ulint space_id) /*!< in: space id */ { - bpage->encrypt_later = false; + fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space_id); - if (srv_encrypt_tables == FALSE) { + if (crypt_data != NULL && crypt_data->encryption == FIL_SPACE_ENCRYPTION_OFF) { + /* Encryption is disabled */ + return const_cast(src_frame); + } + + if (!srv_encrypt_tables && (crypt_data == NULL || crypt_data->encryption == FIL_SPACE_ENCRYPTION_DEFAULT)) { /* Encryption is disabled */ return const_cast(src_frame); } @@ -5814,103 +5886,63 @@ buf_page_encrypt_before_write( ulint zip_size = buf_page_get_zip_size(bpage); ulint page_size = (zip_size) ? zip_size : UNIV_PAGE_SIZE; + buf_pool_t* buf_pool = buf_pool_from_bpage(bpage); + bool page_compressed = fil_space_is_page_compressed(bpage->space); - /** - * TODO(jonaso): figure out more clever malloc strategy - * - * This implementation does a malloc/free per iop for encrypted - * tablespaces. Alternative strategies that have been considered are - * - * 1) use buf_block_alloc (i.e alloc from buffer pool) - * this does not work as buf_block_alloc will then be called - * when needing to flush a page, which might be triggered - * due to shortage of memory in buffer pool - * 2) allocate a buffer per fil_node_t - * this would break abstraction layers and has therfore not been - * considered a lot. - */ + /* Find free slot from temporary memory array */ + buf_tmp_buffer_t* slot = buf_pool_reserve_tmp_slot(buf_pool, page_compressed); + bpage->slot = slot; - if (bpage->crypt_buf_free == NULL) { - bpage->crypt_buf_free = (byte*)malloc(page_size*2); - // TODO: Is 4k aligment enough ? - bpage->crypt_buf = (byte *)ut_align(bpage->crypt_buf_free, page_size); - } + byte *dst_frame = slot->crypt_buf; - byte *dst_frame = bpage->crypt_buf; - - if (!fil_space_is_page_compressed(bpage->space)) { - // encrypt page content - fil_space_encrypt(bpage->space, bpage->offset, - bpage->newest_modification, - src_frame, zip_size, dst_frame, 0); + if (!page_compressed) { + /* Encrypt page content */ + fil_space_encrypt(bpage->space, + bpage->offset, + bpage->newest_modification, + src_frame, + zip_size, + dst_frame); unsigned key_version = mach_read_from_4(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); ut_ad(key_version == 0 || key_version >= bpage->key_version); bpage->key_version = key_version; + bpage->real_size = zip_size; } else { - /** Compression and encryption is done later at os0file.cc */ - bpage->encrypt_later = true; - dst_frame = (byte *)src_frame; + /* First we compress the page content */ + ulint out_len = 0; + ulint block_size = fil_space_get_block_size(bpage->space, bpage->offset, page_size); + + byte *tmp = fil_compress_page(bpage->space, + (byte *)src_frame, + slot->comp_buf, + page_size, + fil_space_get_page_compression_level(bpage->space), + block_size, + &out_len, +#ifdef HAVE_LZO + slot->lzo_mem +#else + NULL +#endif + ); + + bpage->real_size = out_len; + + /* And then we encrypt the page content */ + fil_space_encrypt(bpage->space, + bpage->offset, + bpage->newest_modification, + tmp, + zip_size, + dst_frame); } // return dst_frame which will be written return dst_frame; } -/********************************************************************//** -Release memory after encrypted page has been written to disk -*/ -ibool -buf_page_encrypt_after_write( -/*=========================*/ - buf_page_t* bpage) /*!< in/out: buffer page flushed */ -{ - if (bpage->crypt_buf_free != NULL) { - free(bpage->crypt_buf_free); - bpage->crypt_buf_free = NULL; - bpage->crypt_buf = NULL; - } - - if (bpage->comp_buf_free != NULL) { - free(bpage->comp_buf_free); - bpage->comp_buf_free = NULL; - bpage->comp_buf = NULL; - } - - return (TRUE); -} - -/********************************************************************//** -Allocates memory to read in an encrypted page -*/ -byte* -buf_page_decrypt_before_read( -/*=========================*/ - buf_page_t* bpage, /*!< in/out: buffer page to be read */ - ulint zip_size) /*!< in: compressed page size, or 0 */ -{ - ulint size = (zip_size) ? zip_size : UNIV_PAGE_SIZE; - - /* - Here we only need to allocate space for not header pages - in case of file space encryption. Table encryption is handled - later. - */ - if (!srv_encrypt_tables || bpage->offset == 0 || - fil_space_check_encryption_read(bpage->space) == false) - return zip_size ? bpage->zip.data : ((buf_block_t*) bpage)->frame; - - if (bpage->crypt_buf_free == NULL) - { - // allocate buffer to read data into - bpage->crypt_buf_free = (byte*)malloc(size*2); - // TODO: Is 4K aligment enough ? - bpage->crypt_buf = (byte*)ut_align(bpage->crypt_buf_free, size); - } - return bpage->crypt_buf; -} - /********************************************************************//** Decrypt page after it has been read from disk */ @@ -5928,92 +5960,51 @@ buf_page_decrypt_after_read( if (bpage->offset == 0) { /* File header pages are not encrypted */ - ut_a(bpage->crypt_buf == NULL); return (TRUE); } - const byte* src_frame = bpage->crypt_buf != NULL ? - bpage->crypt_buf : dst_frame; - unsigned key_version = - mach_read_from_4(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); + mach_read_from_4(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); bool page_compressed = fil_page_is_compressed(dst_frame); + buf_pool_t* buf_pool = buf_pool_from_bpage(bpage); if (key_version == 0) { /* the page we read is unencrypted */ - if (fil_page_is_compressed(dst_frame)) { - if (bpage->comp_buf_free == NULL) { - bpage->comp_buf_free = (byte *)malloc(UNIV_PAGE_SIZE*2); - // TODO: is 4k aligment enough ? - bpage->comp_buf = (byte*)ut_align(bpage->comp_buf_free, UNIV_PAGE_SIZE); - } - - fil_decompress_page(bpage->comp_buf, dst_frame, size, NULL); - } else { - if (dst_frame != src_frame) { - /* but we had allocated a crypt_buf */ - // TODO: Can this be avoided ? - memcpy(dst_frame, src_frame, size); - } - } - } else { - /* the page we read is encrypted */ - if (dst_frame == src_frame) { - /* but we had NOT allocated a crypt buf - * malloc a buffer, copy page to it - * and then decrypt from that into real page*/ - bpage->crypt_buf_free = (byte *)malloc(UNIV_PAGE_SIZE*2); - // TODO: is 4k aligment enough ? - src_frame = bpage->crypt_buf = (byte*)ut_align(bpage->crypt_buf_free, UNIV_PAGE_SIZE); - memcpy(bpage->crypt_buf, dst_frame, size); - } - /* decrypt from src_frame to dst_frame */ - fil_space_decrypt(bpage->space, - src_frame, size, dst_frame); - - /* decompress from dst_frame to comp_buf and then copy to - buffer pool */ if (page_compressed) { - if (bpage->comp_buf_free == NULL) { - bpage->comp_buf_free = (byte *)malloc(UNIV_PAGE_SIZE*2); - // TODO: is 4k aligment enough ? - bpage->comp_buf = (byte*)ut_align(bpage->comp_buf_free, UNIV_PAGE_SIZE); - } + /* Find free slot from temporary memory array */ + buf_tmp_buffer_t* slot = buf_pool_reserve_tmp_slot(buf_pool, page_compressed); - fil_decompress_page(bpage->comp_buf, dst_frame, size, NULL); - } + fil_decompress_page(slot->comp_buf, + dst_frame, + size, + &bpage->write_size); + + /* Mark this slot as free */ + slot->reserved = false; + } + } else { + /* Find free slot from temporary memory array */ + buf_tmp_buffer_t* slot = buf_pool_reserve_tmp_slot(buf_pool, page_compressed); + memcpy(slot->crypt_buf, dst_frame, size); + + /* decrypt from crypt_buf to dst_frame */ + fil_space_decrypt(bpage->space, + slot->crypt_buf, + size, + dst_frame); + + if (page_compressed) { + fil_decompress_page(slot->comp_buf, + dst_frame, + size, + &bpage->write_size); + } + + /* Mark this slot as free */ + slot->reserved = false; } bpage->key_version = key_version; - if (bpage->crypt_buf_free != NULL) { - // free temp page - free(bpage->crypt_buf_free); - bpage->crypt_buf = NULL; - bpage->crypt_buf_free = NULL; - } - - if (bpage->comp_buf_free != NULL) { - // free temp page - free(bpage->comp_buf_free); - bpage->comp_buf = NULL; - bpage->comp_buf_free = NULL; - } - return (TRUE); } - -/********************************************************************//** -Release memory allocated for decryption -*/ -void -buf_page_decrypt_cleanup( -/*=====================*/ - buf_page_t* bpage) /*!< in/out: buffer page */ -{ - if (bpage->crypt_buf != NULL) { - free(bpage->crypt_buf_free); - bpage->crypt_buf = NULL; - bpage->crypt_buf_free = NULL; - } -} diff --git a/storage/xtradb/buf/buf0dblwr.cc b/storage/xtradb/buf/buf0dblwr.cc index ec733dd5a73..b0fdc58e4d2 100644 --- a/storage/xtradb/buf/buf0dblwr.cc +++ b/storage/xtradb/buf/buf0dblwr.cc @@ -871,7 +871,7 @@ buf_dblwr_write_block_to_datafile( 0, buf_block_get_page_no(block), 0, - UNIV_PAGE_SIZE, + bpage->real_size, frame, (void*) block, (ulint *)&bpage->write_size); @@ -1247,7 +1247,8 @@ retry: TRX_SYS_SPACE, 0, offset, - 0, UNIV_PAGE_SIZE, + 0, + bpage->real_size, frame, NULL, 0); diff --git a/storage/xtradb/buf/buf0flu.cc b/storage/xtradb/buf/buf0flu.cc index d023e2ff227..6443043310b 100644 --- a/storage/xtradb/buf/buf0flu.cc +++ b/storage/xtradb/buf/buf0flu.cc @@ -951,7 +951,7 @@ buf_flush_write_block_low( break; } - frame = buf_page_encrypt_before_write(bpage, frame); + frame = buf_page_encrypt_before_write(bpage, frame, space_id); if (!srv_use_doublewrite_buf || !buf_dblwr) { fil_io(OS_FILE_WRITE | OS_AIO_SIMULATED_WAKE_LATER, @@ -960,7 +960,7 @@ buf_flush_write_block_low( zip_size, buf_page_get_page_no(bpage), 0, - zip_size ? zip_size : UNIV_PAGE_SIZE, + zip_size ? zip_size : bpage->real_size, frame, bpage, &bpage->write_size); @@ -979,7 +979,7 @@ buf_flush_write_block_low( zip_size, buf_page_get_page_no(bpage), 0, - zip_size ? zip_size : UNIV_PAGE_SIZE, + zip_size ? zip_size : bpage->real_size, frame, bpage, &bpage->write_size); diff --git a/storage/xtradb/buf/buf0rea.cc b/storage/xtradb/buf/buf0rea.cc index 164aa3c62c1..6c74c30739e 100644 --- a/storage/xtradb/buf/buf0rea.cc +++ b/storage/xtradb/buf/buf0rea.cc @@ -222,7 +222,7 @@ not_to_recover: ut_ad(buf_page_in_file(bpage)); ut_ad(!mutex_own(&buf_pool_from_bpage(bpage)->LRU_list_mutex)); - byte* frame = buf_page_decrypt_before_read(bpage, zip_size); + byte* frame = zip_size ? bpage->zip.data : ((buf_block_t*) bpage)->frame; if (sync) { thd_wait_begin(NULL, THD_WAIT_DISKIO); @@ -247,7 +247,6 @@ not_to_recover: } if (*err != DB_SUCCESS) { - buf_page_decrypt_cleanup(bpage); if (ignore_nonexistent_pages || *err == DB_TABLESPACE_DELETED) { buf_read_page_handle_error(bpage); return(0); diff --git a/storage/xtradb/fil/fil0crypt.cc b/storage/xtradb/fil/fil0crypt.cc index aab0b1f5568..07636d27879 100644 --- a/storage/xtradb/fil/fil0crypt.cc +++ b/storage/xtradb/fil/fil0crypt.cc @@ -319,6 +319,7 @@ fil_space_create_crypt_data() &crypt_data->mutex, SYNC_NO_ORDER_CHECK); crypt_data->iv_length = iv_length; my_random_bytes(crypt_data->iv, iv_length); + crypt_data->encryption = FIL_SPACE_ENCRYPTION_DEFAULT; return crypt_data; } @@ -421,6 +422,9 @@ fil_space_read_crypt_data( uint min_key_version = mach_read_from_4 (page + offset + MAGIC_SZ + 2 + iv_length); + fil_encryption_t encryption = (fil_encryption_t)mach_read_from_1( + page + offset + MAGIC_SZ + 2 + iv_length + 4); + const uint sz = sizeof(fil_space_crypt_t) + iv_length; fil_space_crypt_t* crypt_data = static_cast( malloc(sz)); @@ -429,6 +433,7 @@ fil_space_read_crypt_data( crypt_data->type = type; crypt_data->min_key_version = min_key_version; crypt_data->page0_offset = offset; + crypt_data->encryption = encryption; mutex_create(fil_crypt_data_mutex_key, &crypt_data->mutex, SYNC_NO_ORDER_CHECK); crypt_data->iv_length = iv_length; @@ -474,8 +479,9 @@ fil_space_write_crypt_data_low( page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); const uint len = crypt_data->iv_length; const uint min_key_version = crypt_data->min_key_version; + const fil_encryption_t encryption = crypt_data->encryption; crypt_data->page0_offset = offset; - ut_a(2 + len + 4 + MAGIC_SZ < maxsize); + ut_a(2 + len + 4 + 1 + MAGIC_SZ < maxsize); /* redo log this as bytewise updates to page 0 @@ -489,8 +495,10 @@ fil_space_write_crypt_data_low( mtr); mlog_write_ulint(page + offset + MAGIC_SZ + 2 + len, min_key_version, MLOG_4BYTES, mtr); + mlog_write_ulint(page + offset + MAGIC_SZ + 2 + len + 4, encryption, + MLOG_1BYTE, mtr); - byte* log_ptr = mlog_open(mtr, 11 + 12 + len); + byte* log_ptr = mlog_open(mtr, 11 + 13 + len); if (log_ptr != NULL) { log_ptr = mlog_write_initial_log_record_fast( @@ -507,6 +515,8 @@ fil_space_write_crypt_data_low( log_ptr += 1; mach_write_to_4(log_ptr, min_key_version); log_ptr += 4; + mach_write_to_1(log_ptr, encryption); + log_ptr += 1; mlog_close(mtr, log_ptr); mlog_catenate_string(mtr, crypt_data->iv, len); @@ -555,7 +565,8 @@ fil_parse_write_crypt_data( 2 + // size of offset 1 + // size of type 1 + // size of iv-len - 4; // size of min_key_version + 4 + // size of min_key_version + 1; // fil_encryption_t if (end_ptr - ptr < entry_size){ return NULL; @@ -582,9 +593,17 @@ fil_parse_write_crypt_data( return NULL; } + fil_encryption_t encryption = (fil_encryption_t)mach_read_from_1(ptr); + ptr +=1; + + if (end_ptr - ptr < len) { + return NULL; + } + fil_space_crypt_t* crypt_data = fil_space_create_crypt_data(); crypt_data->page0_offset = offset; crypt_data->min_key_version = min_key_version; + crypt_data->encryption = encryption; memcpy(crypt_data->iv, ptr, len); ptr += len; @@ -610,7 +629,8 @@ fil_space_clear_crypt_data( 1 + // type 1 + // len len + // iv - 4; // min key version + 4 + // min key version + 1; // fil_encryption_t memset(page + offset, 0, size); } @@ -674,13 +694,11 @@ fil_space_encrypt( const byte* src_frame, /*!< in: Source page to be encrypted */ ulint zip_size, /*!< in: compressed size if row_format compressed */ - byte* dst_frame, /*!< in: outbut buffer */ - ulint encryption_key) /*!< in: encryption key id if page - encrypted */ + byte* dst_frame) /*!< in: outbut buffer */ { fil_space_crypt_t* crypt_data=NULL; ulint page_size = (zip_size) ? zip_size : UNIV_PAGE_SIZE; - uint key_version = (uint)encryption_key; + uint key_version; unsigned char key[MY_AES_MAX_KEY_LENGTH]; uint key_length=MY_AES_MAX_KEY_LENGTH; uint aes_method; @@ -699,6 +717,7 @@ fil_space_encrypt( /* Get crypt data from file space */ crypt_data = fil_space_get_crypt_data(space); + key_version = crypt_data->keys[0].key_id; if (crypt_data == NULL) { //TODO: Is this really needed ? @@ -808,6 +827,10 @@ fil_space_check_encryption_read( return false; } + if (crypt_data->encryption == FIL_SPACE_ENCRYPTION_OFF) { + return false; + } + return true; } @@ -1352,6 +1375,11 @@ fil_crypt_space_needs_rotation( mutex_enter(&crypt_data->mutex); do { + if (crypt_data->encryption == FIL_SPACE_ENCRYPTION_OFF) { + /* This space is unencrypted by user request */ + break; + } + /* prevent threads from starting to rotate space */ if (crypt_data->rotate_state.starting) { /* recheck this space later */ diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index 6b776edd2f9..1084f90c720 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -5579,6 +5579,74 @@ fil_report_invalid_page_access( (ulong) byte_offset, (ulong) len, (ulong) type); } +/********************************************************************//** +Find correct node from file space +@return node */ +static +fil_node_t* +fil_space_get_node( + fil_space_t* space, /*!< in: file spage */ + ulint space_id, /*!< in: space id */ + ulint* block_offset, /*!< in/out: offset in number of blocks */ + ulint byte_offset, /*!< in: remainder of offset in bytes; in + aio this must be divisible by the OS block + size */ + ulint len) /*!< in: how many bytes to read or write; this + must not cross a file boundary; in aio this + must be a block size multiple */ +{ + fil_node_t* node; + ut_ad(mutex_own(&fil_system->mutex)); + + node = UT_LIST_GET_FIRST(space->chain); + + for (;;) { + if (node == NULL) { + return(NULL); + } else if (fil_is_user_tablespace_id(space->id) + && node->size == 0) { + + /* We do not know the size of a single-table tablespace + before we open the file */ + break; + } else if (node->size > *block_offset) { + /* Found! */ + break; + } else { + *block_offset -= node->size; + node = UT_LIST_GET_NEXT(chain, node); + } + } + + return (node); +} +/********************************************************************//** +Return block size of node in file space +@return file block size */ +UNIV_INTERN +ulint +fil_space_get_block_size( +/*=====================*/ + ulint space_id, + ulint block_offset, + ulint len) +{ + ulint block_size = 512; + fil_space_t* space = fil_space_get_space(space_id); + + if (space) { + mutex_enter(&fil_system->mutex); + fil_node_t* node = fil_space_get_node(space, space_id, &block_offset, 0, len); + mutex_exit(&fil_system->mutex); + + if (node) { + block_size = node->file_block_size; + } + } + + return block_size; +} + /********************************************************************//** Reads or writes data. This operation is asynchronous (aio). @return DB_SUCCESS, or DB_TABLESPACE_DELETED if we are trying to do @@ -5626,7 +5694,7 @@ _fil_io( ulint is_log; ulint wake_later; os_offset_t offset; - ibool ignore_nonexistent_pages; + bool ignore_nonexistent_pages; is_log = type & OS_FILE_LOG; type = type & ~OS_FILE_LOG; @@ -5711,34 +5779,18 @@ _fil_io( ut_ad(mode != OS_AIO_IBUF || space->purpose == FIL_TABLESPACE); - node = UT_LIST_GET_FIRST(space->chain); + node = fil_space_get_node(space, space_id, &block_offset, byte_offset, len); - for (;;) { - if (node == NULL) { - if (ignore_nonexistent_pages) { - mutex_exit(&fil_system->mutex); - return(DB_ERROR); - } - - fil_report_invalid_page_access( + if (!node) { + if (ignore_nonexistent_pages) { + mutex_exit(&fil_system->mutex); + return(DB_ERROR); + } + fil_report_invalid_page_access( block_offset, space_id, space->name, byte_offset, len, type); - ut_error; - - } else if (fil_is_user_tablespace_id(space->id) - && node->size == 0) { - - /* We do not know the size of a single-table tablespace - before we open the file */ - break; - } else if (node->size > block_offset) { - /* Found! */ - break; - } else { - block_offset -= node->size; - node = UT_LIST_GET_NEXT(chain, node); - } + ut_error; } /* Open file if closed */ diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index cb6b7d1a263..9f4f52b4f4a 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -631,10 +631,10 @@ ha_create_table_option innodb_table_option_list[]= HA_TOPTION_NUMBER("PAGE_COMPRESSION_LEVEL", page_compression_level, 0, 1, 9, 1), /* With this option user can enable atomic writes feature for this table */ HA_TOPTION_ENUM("ATOMIC_WRITES", atomic_writes, "DEFAULT,ON,OFF", 0), - /* With this option the user can enable page encryption for the table */ - HA_TOPTION_BOOL("PAGE_ENCRYPTION", page_encryption, 0), + /* With this option the user can enable encryption for the table */ + HA_TOPTION_ENUM("ENCRYPTION", encryption, "DEFAULT, ON, OFF", 0), /* With this option the user defines the key identifier using for the encryption */ - HA_TOPTION_NUMBER("PAGE_ENCRYPTION_KEY", page_encryption_key, 0, 1, 255, 1), + HA_TOPTION_NUMBER("ENCRYPTION_KEY", encryption_key, 0, 1, UINT_MAX32, 1), HA_TOPTION_END }; @@ -11791,13 +11791,14 @@ ha_innobase::check_table_options( enum row_type row_format = table->s->row_type; ha_table_option_struct *options= table->s->option_struct; atomic_writes_t awrites = (atomic_writes_t)options->atomic_writes; + fil_encryption_t encrypt = (fil_encryption_t)options->encryption; - if (options->page_encryption) { + if (encrypt == FIL_SPACE_ENCRYPTION_ON) { if (srv_encrypt_tables) { push_warning( thd, Sql_condition::WARN_LEVEL_WARN, HA_WRONG_CREATE_OPTION, - "InnoDB: PAGE_ENCRYPTION not available if innodb_encrypt_tables=ON"); + "InnoDB: ENCRYPTION not available if innodb_encrypt_tables=ON"); return "INNODB_ENCRYPT_TABLES"; } @@ -11805,7 +11806,7 @@ ha_innobase::check_table_options( push_warning( thd, Sql_condition::WARN_LEVEL_WARN, HA_WRONG_CREATE_OPTION, - "InnoDB: PAGE_ENCRYPTION requires" + "InnoDB: ENCRYPTION requires" " innodb_file_per_table."); return "PAGE_ENCRYPTION"; } @@ -11814,14 +11815,6 @@ ha_innobase::check_table_options( /* Check page compression requirements */ if (options->page_compressed) { - if (srv_encrypt_tables) { - push_warning( - thd, Sql_condition::WARN_LEVEL_WARN, - HA_WRONG_CREATE_OPTION, - "InnoDB: PAGE_COMPRESSION not available if innodb_encrypt_tables=ON"); - return "PAGE_COMPRESSED"; - } - if (row_format == ROW_TYPE_COMPRESSED) { push_warning( thd, Sql_condition::WARN_LEVEL_WARN, @@ -11891,29 +11884,29 @@ ha_innobase::check_table_options( } } - if (options->page_encryption_key != 0) { - if (options->page_encryption == false) { + if (options->encryption_key != 0) { + if (options->encryption == FIL_SPACE_ENCRYPTION_OFF) { /* ignore this to allow alter table without changing page_encryption_key ...*/ } - if (options->page_encryption_key < 1 || options->page_encryption_key > 255) { + if (options->encryption_key < 1) { push_warning_printf( thd, Sql_condition::WARN_LEVEL_WARN, HA_WRONG_CREATE_OPTION, - "InnoDB: invalid PAGE_ENCRYPTION_KEY = %lu." - " Valid values are [1..255]", - options->page_encryption_key); - return "PAGE_ENCRYPTION_KEY"; + "InnoDB: invalid ENCRYPTION_KEY = %lu." + " Valid values are [1..UINTMAX32]", + options->encryption_key); + return "ENCRYPTION_KEY"; } - if (!has_encryption_key(options->page_encryption_key)) { + if (!has_encryption_key(options->encryption_key)) { push_warning_printf( thd, Sql_condition::WARN_LEVEL_WARN, HA_WRONG_CREATE_OPTION, - "InnoDB: PAGE_ENCRYPTION_KEY encryption key %lu not available", - options->page_encryption_key + "InnoDB: ENCRYPTION_KEY %lu not available", + options->encryption_key ); - return "PAGE_ENCRYPTION_KEY"; + return "ENCRYPTION_KEY"; } } @@ -11975,6 +11968,11 @@ ha_innobase::create( const char* stmt; size_t stmt_len; + /* Cache table options */ + ha_table_option_struct *options= table->s->option_struct; + fil_encryption_t encrypt = (fil_encryption_t)options->encryption; + ulint key_id = (options->encryption_key == 0) ? srv_default_encryption_key : + options->encryption_key; DBUG_ENTER("ha_innobase::create"); @@ -12220,6 +12218,18 @@ ha_innobase::create( innobase_commit_low(trx); + /* If user has requested that table should be encrypted or table + should remain as unencrypted store crypt data */ + if (encrypt == FIL_SPACE_ENCRYPTION_ON || encrypt == FIL_SPACE_ENCRYPTION_OFF) { + ulint maxsize; + ulint zip_size = fil_space_get_zip_size(innobase_table->space); + fil_space_crypt_t* crypt_data = fil_space_create_crypt_data(); + crypt_data->page0_offset = fsp_header_get_crypt_offset(zip_size, &maxsize); + crypt_data->keys[0].key_id = key_id; + crypt_data->encryption = encrypt; + fil_space_set_crypt_data(innobase_table->space, crypt_data); + } + row_mysql_unlock_data_dictionary(trx); /* Flush the log to reduce probability that the .frm files and @@ -20329,12 +20339,12 @@ static MYSQL_SYSVAR_UINT(encryption_rotation_iops, srv_n_fil_crypt_iops, innodb_encryption_rotation_iops_update, srv_n_fil_crypt_iops, 0, UINT_MAX32, 0); -static MYSQL_SYSVAR_UINT(default_page_encryption_key, srv_default_page_encryption_key, +static MYSQL_SYSVAR_UINT(default_encryption_key, srv_default_encryption_key, PLUGIN_VAR_RQCMDARG, - "Encryption key used for page encryption.", + "Default encryption key used for table encryption.", NULL, NULL, - FIL_DEFAULT_ENCRYPTION_KEY, 1, 255, 0); + FIL_DEFAULT_ENCRYPTION_KEY, 1, UINT_MAX32, 0); static MYSQL_SYSVAR_BOOL(scrub_log, srv_scrub_log, PLUGIN_VAR_OPCMDARG | PLUGIN_VAR_READONLY, @@ -20641,7 +20651,7 @@ static struct st_mysql_sys_var* innobase_system_variables[]= { MYSQL_SYSVAR(scrub_log), MYSQL_SYSVAR(scrub_log_interval), MYSQL_SYSVAR(encrypt_log), - MYSQL_SYSVAR(default_page_encryption_key), + MYSQL_SYSVAR(default_encryption_key), /* Scrubing feature */ MYSQL_SYSVAR(immediate_scrub_data_uncompressed), MYSQL_SYSVAR(background_scrub_data_uncompressed), diff --git a/storage/xtradb/handler/ha_innodb.h b/storage/xtradb/handler/ha_innodb.h index 0acf77da28f..fdc6f9521aa 100644 --- a/storage/xtradb/handler/ha_innodb.h +++ b/storage/xtradb/handler/ha_innodb.h @@ -71,9 +71,8 @@ struct ha_table_option_struct srv_use_atomic_writes=1. Atomic writes are not used if value OFF.*/ - bool page_encryption; /*!< Flag for an encrypted table */ - /* Following can't be unsigned as it's compared with ULINT_UNDEFINED */ - int page_encryption_key; /*!< ID of the encryption key */ + uint encryption; /*!< DEFAULT, ON, OFF */ + int encryption_key; /*!< encryption key id */ }; /** The class defining a handle to an Innodb table */ diff --git a/storage/xtradb/handler/handler0alter.cc b/storage/xtradb/handler/handler0alter.cc index 7e8e6e4598e..9fe3cbc2bd1 100644 --- a/storage/xtradb/handler/handler0alter.cc +++ b/storage/xtradb/handler/handler0alter.cc @@ -282,8 +282,8 @@ ha_innobase::check_if_supported_inplace_alter( DBUG_RETURN(HA_ALTER_INPLACE_NOT_SUPPORTED); } - if (new_options->page_encryption != old_options->page_encryption || - new_options->page_encryption_key != old_options->page_encryption_key) { + if (new_options->encryption != old_options->encryption || + new_options->encryption_key != old_options->encryption_key) { ha_alter_info->unsupported_reason = innobase_get_err_msg( ER_ALTER_OPERATION_NOT_SUPPORTED_REASON); DBUG_RETURN(HA_ALTER_INPLACE_NOT_SUPPORTED); diff --git a/storage/xtradb/include/buf0buf.h b/storage/xtradb/include/buf0buf.h index 7eb58216698..912fbb9194f 100644 --- a/storage/xtradb/include/buf0buf.h +++ b/storage/xtradb/include/buf0buf.h @@ -1469,16 +1469,19 @@ buf_own_zip_mutex_for_page( The hook that is called just before a page is written to disk. The function encrypts the content of the page and returns a pointer to a frame that will be written instead of the real frame. */ +UNIV_INTERN byte* buf_page_encrypt_before_write( /*==========================*/ buf_page_t* page, /*!< in/out: buffer page to be flushed */ - const byte* frame); + const byte* frame, + ulint space_id); /********************************************************************** The hook that is called after page is written to disk. The function releases any resources needed for encryption that was allocated in buf_page_encrypt_before_write */ +UNIV_INTERN ibool buf_page_encrypt_after_write( /*=========================*/ @@ -1488,6 +1491,7 @@ buf_page_encrypt_after_write( The hook that is called just before a page is read from disk. The function allocates memory that is used to temporarily store disk content before getting decrypted */ +UNIV_INTERN byte* buf_page_decrypt_before_read( /*=========================*/ @@ -1498,19 +1502,35 @@ buf_page_decrypt_before_read( The hook that is called just after a page is read from disk. The function decrypt disk content into buf_page_t and releases the temporary buffer that was allocated in buf_page_decrypt_before_read */ +UNIV_INTERN ibool buf_page_decrypt_after_read( /*========================*/ buf_page_t* page); /*!< in/out: buffer page read from disk */ -/********************************************************************//** -Release memory allocated for page decryption. -Only used in scenarios where read fails, e.g due to tablespace being dropped */ -void -buf_page_decrypt_cleanup( -/*=====================*/ - buf_page_t* page); /*!< in/out: buffer page read from disk */ +/** @brief The temporary memory structure. +NOTE! The definition appears here only for other modules of this +directory (buf) to see it. Do not use from outside! */ + +typedef struct { + bool reserved; /*!< true if this slot is reserved + */ +#ifdef HAVE_LZO + byte* lzo_mem; /*!< Temporal memory used by LZO */ +#endif + byte* crypt_buf; /*!< for encryption the data needs to be + copied to a separate buffer before it's + encrypted&written. this as a page can be + read while it's being flushed */ + byte* crypt_buf_free; /*!< for encryption, allocated buffer + that is then alligned */ + byte* comp_buf; /*!< for compression we need + temporal buffer because page + can be read while it's being flushed */ + byte* comp_buf_free; /*!< for compression, allocated + buffer that is then alligned */ +} buf_tmp_buffer_t; /** The common buffer control block structure for compressed and uncompressed frames */ @@ -1587,19 +1607,16 @@ struct buf_page_t{ operation needed. */ unsigned key_version; /*!< key version for this block */ - byte* crypt_buf; /*!< for encryption the data needs to be - copied to a separate buffer before it's - encrypted&written. this as a page can be - read while it's being flushed */ - byte* crypt_buf_free; /*!< for encryption, allocated buffer - that is then alligned */ - byte* comp_buf; /*!< for compression we need - temporal buffer because page - can be read while it's being flushed */ - byte* comp_buf_free; /*!< for compression, allocated - buffer that is then alligned */ - bool encrypt_later; /*!< should we encrypt the page - at os0file.cc ? */ + + ulint real_size; /*!< Real size of the page + Normal pages == UNIV_PAGE_SIZE + page compressed pages, payload + size alligned to sector boundary. + */ + + buf_tmp_buffer_t* slot; /*!< Slot for temporary memory + used for encryption/compression + or NULL */ #ifndef UNIV_HOTBACKUP buf_page_t* hash; /*!< node used in chaining to buf_pool->page_hash or @@ -1918,6 +1935,17 @@ struct buf_buddy_stat_t { ib_uint64_t relocated_usec; }; +/** @brief The temporary memory array structure. + +NOTE! The definition appears here only for other modules of this +directory (buf) to see it. Do not use from outside! */ + +typedef struct { + ulint n_slots; /*!< Total number of slots */ + buf_tmp_buffer_t *slots; /*!< Pointer to the slots in the + array */ +} buf_tmp_array_t; + /** @brief The buffer pool structure. NOTE! The definition appears here only for other modules of this @@ -2091,6 +2119,10 @@ struct buf_pool_t{ /*!< Sentinel records for buffer pool watches. */ + buf_tmp_array_t* tmp_arr; + /*!< Array for temporal memory + used in compression and encryption */ + #if BUF_BUDDY_LOW > UNIV_ZIP_SIZE_MIN # error "BUF_BUDDY_LOW > UNIV_ZIP_SIZE_MIN" #endif diff --git a/storage/xtradb/include/buf0buf.ic b/storage/xtradb/include/buf0buf.ic index b4cbba29f3c..7f6cceade36 100644 --- a/storage/xtradb/include/buf0buf.ic +++ b/storage/xtradb/include/buf0buf.ic @@ -925,28 +925,15 @@ buf_block_free( mutex_exit(&block->mutex); } -/********************************************************************//** -Get crypt buffer. */ -UNIV_INLINE -byte* -buf_page_get_crypt_buffer( -/*=========================*/ - const buf_page_t* bpage) /*!< in: buffer pool page */ -{ - return bpage->crypt_buf; -} - /********************************************************************//** Get buf frame. */ UNIV_INLINE void * buf_page_get_frame( -/*=========================*/ +/*===============*/ const buf_page_t* bpage) /*!< in: buffer pool page */ { - if (bpage->crypt_buf) { - return buf_page_get_crypt_buffer(bpage); - } else if (bpage->zip.data) { + if (bpage->zip.data) { return bpage->zip.data; } else { return ((buf_block_t*) bpage)->frame; diff --git a/storage/xtradb/include/fil0crypt.h b/storage/xtradb/include/fil0crypt.h index 9d02034e4b2..359533dc50a 100644 --- a/storage/xtradb/include/fil0crypt.h +++ b/storage/xtradb/include/fil0crypt.h @@ -37,6 +37,14 @@ Created 04/01/2015 Jan Lindström /* This key will be used if nothing else is given */ #define FIL_DEFAULT_ENCRYPTION_KEY 1 +/** Enum values for encryption table option */ +typedef enum { + FIL_SPACE_ENCRYPTION_DEFAULT = 0, /* Tablespace encrypted if + srv_encrypt_tables = ON */ + FIL_SPACE_ENCRYPTION_ON = 1, /* Tablespace is encrypted always */ + FIL_SPACE_ENCRYPTION_OFF = 2 /* Tablespace is not encrypted */ +} fil_encryption_t; + /** * CRYPT_SCHEME_UNENCRYPTED * @@ -67,6 +75,8 @@ struct key_struct { uint key_version; /*!< Key version used as identifier */ + uint key_id; /*1< Key id used as + identifier */ byte key[MY_AES_MAX_KEY_LENGTH]; /*!< Cached L or key */ uint key_length; /*!< Key length */ }; @@ -95,6 +105,7 @@ struct fil_space_crypt_struct key_struct keys[3]; // cached L = AES_ECB(KEY, IV) uint min_key_version; // min key version for this space ulint page0_offset; // byte offset on page 0 for crypt data + fil_encryption_t encryption; // Encryption setup ib_mutex_t mutex; // mutex protecting following variables bool closing; // is tablespace being closed @@ -227,9 +238,7 @@ fil_space_encrypt( lsn_t lsn, /*!< in: page lsn */ const byte* src_frame,/*!< in: page frame */ ulint size, /*!< in: size of data to encrypt */ - byte* dst_frame, /*!< in: where to encrypt to */ - ulint page_encryption_key); /*!< in: page encryption key id if page - encrypted */ + byte* dst_frame); /*!< in: where to encrypt to */ /********************************************************************* Decrypt buffer page */ diff --git a/storage/xtradb/include/fil0fil.h b/storage/xtradb/include/fil0fil.h index 91653c4bb7d..c7ecaf8b628 100644 --- a/storage/xtradb/include/fil0fil.h +++ b/storage/xtradb/include/fil0fil.h @@ -982,6 +982,16 @@ _fil_io( #define fil_io(type, sync, space_id, zip_size, block_offset, byte_offset, len, buf, message, write_size) \ _fil_io(type, sync, space_id, zip_size, block_offset, byte_offset, len, buf, message, write_size, NULL) +/*******************************************************************//** +Returns the block size of the file space +@return block size */ +UNIV_INTERN +ulint +fil_space_get_block_size( +/*=====================*/ + ulint id, /*!< in: space id */ + ulint offset, /*!< in: page offset */ + ulint len); /*!< in: page len */ /**********************************************************************//** Waits for an aio operation to complete. This function is used to write the diff --git a/storage/xtradb/include/srv0srv.h b/storage/xtradb/include/srv0srv.h index de33f767021..7387f11bd6e 100644 --- a/storage/xtradb/include/srv0srv.h +++ b/storage/xtradb/include/srv0srv.h @@ -705,8 +705,8 @@ that semaphore times out in InnoDB */ #define DEFAULT_SRV_FATAL_SEMAPHORE_TIMEOUT 600 extern ulong srv_fatal_semaphore_wait_threshold; -/** Default encryption key used for page encryption */ -extern uint srv_default_page_encryption_key; +/** Default encryption key used for tablespace encryption */ +extern uint srv_default_encryption_key; /** Enable semaphore request instrumentation */ extern my_bool srv_instrument_semaphores; diff --git a/storage/xtradb/srv/srv0srv.cc b/storage/xtradb/srv/srv0srv.cc index 9396a713cad..df3982b98cd 100644 --- a/storage/xtradb/srv/srv0srv.cc +++ b/storage/xtradb/srv/srv0srv.cc @@ -670,8 +670,8 @@ thread ensures that we flush the log files at least once per second. */ static time_t srv_last_log_flush_time; -/** Default encryption key used for page encryption */ -UNIV_INTERN uint srv_default_page_encryption_key = FIL_DEFAULT_ENCRYPTION_KEY; +/** Default encryption key used for tablespace encryption */ +UNIV_INTERN uint srv_default_encryption_key = FIL_DEFAULT_ENCRYPTION_KEY; /** Enable semaphore request instrumentation */ UNIV_INTERN my_bool srv_instrument_semaphores = FALSE; From 71ec0463af4fb7658d2f37d3a054e50fa49cfb2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Wed, 1 Apr 2015 20:33:50 +0300 Subject: [PATCH 3/6] Encryption cleanup Step 3: -- Make encrytion_algorithm changeable by SUPER -- Remove AES_ECB method from encryption_algorithms -- Support AES method change by storing used method on InnoDB/XtraDB objects -- Store used AES method to crypt_data as different crypt types -- Store used AES method to redo/undo logs and checkpoint -- Store used AES method on every encrypted page after key_version -- Add test --- include/my_aes.h | 4 +- .../encryption_algorithms.combinations | 3 - mysql-test/r/mysqld--help.result | 4 +- .../innodb/r/innodb_encryption_change.result | 89 +++++++ .../innodb/t/innodb_encryption_change.opt | 7 + .../innodb/t/innodb_encryption_change.test | 229 ++++++++++++++++++ .../r/encryption_algorithm_basic.result | 5 +- .../r/sysvars_server_notembedded.result | 4 +- .../t/encryption_algorithm_basic.test | 28 ++- sql/sys_vars.cc | 25 +- storage/innobase/buf/buf0buf.cc | 13 + storage/innobase/fil/fil0crypt.cc | 55 +---- storage/innobase/fil/fil0fil.cc | 21 +- storage/innobase/fil/fil0pagecompress.cc | 2 + storage/innobase/include/fil0crypt.ic | 51 ++++ storage/innobase/include/fil0pagecompress.h | 15 +- storage/innobase/include/log0crypt.h | 2 + storage/innobase/include/log0log.h | 8 +- storage/innobase/log/log0crypt.cc | 65 +++-- storage/innobase/log/log0recv.cc | 3 + storage/xtradb/buf/buf0buf.cc | 13 + storage/xtradb/fil/fil0crypt.cc | 50 ---- storage/xtradb/fil/fil0fil.cc | 21 +- storage/xtradb/fil/fil0pagecompress.cc | 2 + storage/xtradb/include/fil0crypt.ic | 51 ++++ storage/xtradb/include/fil0pagecompress.h | 15 +- storage/xtradb/include/log0crypt.h | 2 + storage/xtradb/include/log0log.h | 8 +- storage/xtradb/log/log0crypt.cc | 64 +++-- storage/xtradb/log/log0recv.cc | 3 + 30 files changed, 676 insertions(+), 186 deletions(-) create mode 100644 mysql-test/suite/innodb/r/innodb_encryption_change.result create mode 100644 mysql-test/suite/innodb/t/innodb_encryption_change.opt create mode 100644 mysql-test/suite/innodb/t/innodb_encryption_change.test diff --git a/include/my_aes.h b/include/my_aes.h index 057fa1e9860..de565960f60 100644 --- a/include/my_aes.h +++ b/include/my_aes.h @@ -115,8 +115,8 @@ extern MYSQL_PLUGIN_IMPORT my_aes_decrypt_dynamic_type my_aes_decrypt_dynamic; enum enum_my_aes_encryption_algorithm { - MY_AES_ALGORITHM_NONE, MY_AES_ALGORITHM_ECB, MY_AES_ALGORITHM_CBC, - MY_AES_ALGORITHM_CTR + MY_AES_ALGORITHM_NONE, MY_AES_ALGORITHM_CTR, + MY_AES_ALGORITHM_CBC, MY_AES_ALGORITHM_ECB }; my_aes_decrypt_dynamic_type get_aes_decrypt_func(enum enum_my_aes_encryption_algorithm method); diff --git a/mysql-test/include/encryption_algorithms.combinations b/mysql-test/include/encryption_algorithms.combinations index 09c1e91e1cd..a97d6b79be8 100644 --- a/mysql-test/include/encryption_algorithms.combinations +++ b/mysql-test/include/encryption_algorithms.combinations @@ -1,8 +1,5 @@ [cbc] encryption-algorithm=aes_cbc -[ecb] -encryption-algorithm=aes_ecb - [ctr] encryption-algorithm=aes_ctr diff --git a/mysql-test/r/mysqld--help.result b/mysql-test/r/mysqld--help.result index 7d59ecd1dc2..edc4fe2a9cd 100644 --- a/mysql-test/r/mysqld--help.result +++ b/mysql-test/r/mysqld--help.result @@ -167,8 +167,8 @@ The following options may be given as the first argument: execution) --encryption-algorithm=name Which encryption algorithm to use for table encryption. - aes_cbc is the recommended one.. One of: none, aes_ecb, - aes_cbc, aes_ctr + aes_cbc is the recommended one.. One of: none, aes_cbc, + aes_ctr --enforce-storage-engine=name Force the use of a storage engine for new tables --event-scheduler[=name] diff --git a/mysql-test/suite/innodb/r/innodb_encryption_change.result b/mysql-test/suite/innodb/r/innodb_encryption_change.result new file mode 100644 index 00000000000..90100852d33 --- /dev/null +++ b/mysql-test/suite/innodb/r/innodb_encryption_change.result @@ -0,0 +1,89 @@ +SET @start_global_value = @@global.innodb_encryption_threads; +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; +SHOW VARIABLES LIKE 'innodb_encrypt%'; +Variable_name Value +innodb_encrypt_log ON +innodb_encrypt_tables OFF +innodb_encryption_rotate_key_age 15 +innodb_encryption_rotation_iops 100 +innodb_encryption_threads 0 +DESCRIBE INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION; +Field Type Null Key Default Extra +SPACE int(11) unsigned NO 0 +NAME varchar(655) YES NULL +ENCRYPTION_SCHEME int(11) unsigned NO 0 +KEYSERVER_REQUESTS int(11) unsigned NO 0 +MIN_KEY_VERSION int(11) unsigned NO 0 +CURRENT_KEY_VERSION int(11) unsigned NO 0 +KEY_ROTATION_PAGE_NUMBER bigint(21) unsigned YES NULL +KEY_ROTATION_MAX_PAGE_NUMBER bigint(21) unsigned YES NULL +create table innodb_normal(c1 bigint not null, b char(200)) engine=innodb; +create table innodb_compact(c1 bigint not null, b char(200)) engine=innodb row_format=compact; +create table innodb_dynamic(c1 bigint not null, b char(200)) engine=innodb row_format=dynamic; +create table innodb_compressed(c1 bigint not null, b char(200)) engine=innodb row_format=compressed; +create table innodb_redundant(c1 bigint not null, b char(200)) engine=innodb row_format=redundant; +create procedure innodb_insert_proc (repeat_count int) +begin +declare current_num int; +set current_num = 0; +while current_num < repeat_count do +insert into innodb_normal values(current_num, substring(MD5(RAND()), -64)); +set current_num = current_num + 1; +end while; +end// +commit; +set autocommit=0; +call innodb_insert_proc(2000); +commit; +set autocommit=1; +insert into innodb_compact select * from innodb_normal; +insert into innodb_dynamic select * from innodb_normal; +insert into innodb_compressed select * from innodb_normal; +insert into innodb_redundant select * from innodb_normal; +" Start encrypt tablespaces +SET GLOBAL innodb_encrypt_tables = on; +SET GLOBAL innodb_encryption_threads = 4; +# Wait max 5 min for key encryption threads to encrypt one space +# Success! +# Wait max 10 min for key encryption threads to encrypt all space +# Success! +# Now turn off encryption and wait for threads to decrypt everything +SET GLOBAL innodb_encrypt_tables = off; +set GLOBAL encryption_algorithm = aes_cbc; +# Wait max 10 min for key encryption threads to decrypt all space +# Success! +# Shutdown innodb_encryption_threads +SET GLOBAL innodb_encryption_threads=0; +# Turn on encryption +# since threads are off tables should remain unencrypted +SET GLOBAL innodb_encrypt_tables = on; +# Wait 15s to check that nothing gets encrypted +# Success! +# Startup innodb_encryption_threads +SET GLOBAL innodb_encryption_threads=4; +# Wait 1 min to check that it start encrypting again +# Success! +# Wait max 10 min for key encryption threads to decrypt all space +# Success! +SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_encrypted'; +variable_value >= 0 +1 +SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_decrypted'; +variable_value >= 0 +1 +SELECT variable_value = 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_encryption_error'; +variable_value = 0 +1 +SELECT variable_value > 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_compressed'; +variable_value > 0 +0 +SELECT variable_value > 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_decompressed'; +variable_value > 0 +0 +drop procedure innodb_insert_proc; +drop table innodb_normal; +drop table innodb_compact; +drop table innodb_dynamic; +drop table innodb_compressed; +drop table innodb_redundant; diff --git a/mysql-test/suite/innodb/t/innodb_encryption_change.opt b/mysql-test/suite/innodb/t/innodb_encryption_change.opt new file mode 100644 index 00000000000..0839ecf7942 --- /dev/null +++ b/mysql-test/suite/innodb/t/innodb_encryption_change.opt @@ -0,0 +1,7 @@ +--aria-encrypt-tables=ON +--encrypt-tmp-disk-tables=ON +--innodb-encryption-rotate-key-age=15 +--innodb-encryption-threads=0 +--innodb-tablespaces-encryption +--innodb-encrypt-log=ON + diff --git a/mysql-test/suite/innodb/t/innodb_encryption_change.test b/mysql-test/suite/innodb/t/innodb_encryption_change.test new file mode 100644 index 00000000000..6c5eab50b84 --- /dev/null +++ b/mysql-test/suite/innodb/t/innodb_encryption_change.test @@ -0,0 +1,229 @@ +# +# +# +-- source include/have_innodb.inc +-- source include/have_example_key_management_plugin.inc + +# embedded does not support restart +-- source include/not_embedded.inc + +--disable_query_log +let $orig_algorithm=`SELECT @@encryption_algorithm`; +let $innodb_file_format_orig = `SELECT @@innodb_file_format`; +let $innodb_file_per_table_orig = `SELECT @@innodb_file_per_table`; +let $encrypt_tables_orig = `SELECT @@innodb_encrypt_tables`; +--enable_query_log + +SET @start_global_value = @@global.innodb_encryption_threads; +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; + +--disable_query_log +EVAL SET GLOBAL encryption_algorithm = $orig_algorithm; +--enable_query_log + +SHOW VARIABLES LIKE 'innodb_encrypt%'; + +DESCRIBE INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION; + +create table innodb_normal(c1 bigint not null, b char(200)) engine=innodb; +create table innodb_compact(c1 bigint not null, b char(200)) engine=innodb row_format=compact; +create table innodb_dynamic(c1 bigint not null, b char(200)) engine=innodb row_format=dynamic; +create table innodb_compressed(c1 bigint not null, b char(200)) engine=innodb row_format=compressed; +create table innodb_redundant(c1 bigint not null, b char(200)) engine=innodb row_format=redundant; + +delimiter //; +create procedure innodb_insert_proc (repeat_count int) +begin + declare current_num int; + set current_num = 0; + while current_num < repeat_count do + insert into innodb_normal values(current_num, substring(MD5(RAND()), -64)); + set current_num = current_num + 1; + end while; +end// +delimiter ;// +commit; + +set autocommit=0; +call innodb_insert_proc(2000); +commit; +set autocommit=1; + +insert into innodb_compact select * from innodb_normal; +insert into innodb_dynamic select * from innodb_normal; +insert into innodb_compressed select * from innodb_normal; +insert into innodb_redundant select * from innodb_normal; + +--echo " Start encrypt tablespaces +SET GLOBAL innodb_encrypt_tables = on; +SET GLOBAL innodb_encryption_threads = 4; + +--echo # Wait max 5 min for key encryption threads to encrypt one space +let $cnt=300; +while ($cnt) +{ + let $success=`SELECT COUNT(*) > 0 FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION > 0`; + if ($success) + { + let $cnt=0; + } + if (!$success) + { + real_sleep 1; + dec $cnt; + } +} +if (!$success) +{ + SELECT * FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION; + SHOW STATUS LIKE 'innodb_encryption%'; + -- die Timeout waiting for encryption threads +} +--echo # Success! + +--echo # Wait max 10 min for key encryption threads to encrypt all space +let $cnt=600; +while ($cnt) +{ + let $success=`SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0`; + if ($success) + { + let $cnt=0; + } + if (!$success) + { + real_sleep 1; + dec $cnt; + } +} +if (!$success) +{ + SELECT * FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION; + SHOW STATUS LIKE 'innodb_encryption%'; + -- die Timeout waiting for encryption threads +} +--echo # Success! + +--echo # Now turn off encryption and wait for threads to decrypt everything +SET GLOBAL innodb_encrypt_tables = off; +set GLOBAL encryption_algorithm = aes_cbc; + +--echo # Wait max 10 min for key encryption threads to decrypt all space +let $cnt=600; +while ($cnt) +{ + let $success=`SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0`; + if ($success) + { + let $cnt=0; + } + if (!$success) + { + real_sleep 1; + dec $cnt; + } +} +if (!$success) +{ + SELECT * FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION; + SHOW STATUS LIKE 'innodb_encryption%'; + -- die Timeout waiting for encryption threads +} +--echo # Success! + +--echo # Shutdown innodb_encryption_threads +SET GLOBAL innodb_encryption_threads=0; + +--echo # Turn on encryption +--echo # since threads are off tables should remain unencrypted +SET GLOBAL innodb_encrypt_tables = on; + +--echo # Wait 15s to check that nothing gets encrypted +let $cnt=15; +while ($cnt) +{ + let $success=`SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0`; + if ($success) + { + real_sleep 1; + dec $cnt; + } + if (!$success) + { + SELECT * FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0; + -- die Failure, tablespace getting encrypted even if innodb_encryption_threads=0 + } +} +--echo # Success! + +--echo # Startup innodb_encryption_threads +SET GLOBAL innodb_encryption_threads=4; + +--echo # Wait 1 min to check that it start encrypting again +let $cnt=60; +while ($cnt) +{ + let $success=`SELECT COUNT(*) > 0 FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0 OR KEY_ROTATION_PAGE_NUMBER IS NOT NULL`; + if ($success) + { + let $cnt=0; + } + if (!$success) + { + real_sleep 1; + dec $cnt; + } +} +if (!$success) +{ + SELECT * FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION; + SHOW STATUS LIKE 'innodb_encryption%'; + -- die Timeout waiting for encryption threads +} +--echo # Success! +--echo # Wait max 10 min for key encryption threads to decrypt all space +let $cnt=600; +while ($cnt) +{ + let $success=`SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0`; + if ($success) + { + let $cnt=0; + } + if (!$success) + { + real_sleep 1; + dec $cnt; + } +} +if (!$success) +{ + SELECT * FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION; + SHOW STATUS LIKE 'innodb_encryption%'; + -- die Timeout waiting for encryption threads +} +--echo # Success! + +SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_encrypted'; +SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_decrypted'; +SELECT variable_value = 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_encryption_error'; +SELECT variable_value > 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_compressed'; +SELECT variable_value > 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_decompressed'; + +drop procedure innodb_insert_proc; +drop table innodb_normal; +drop table innodb_compact; +drop table innodb_dynamic; +drop table innodb_compressed; +drop table innodb_redundant; + +# reset system +--disable_query_log +EVAL SET GLOBAL innodb_encrypt_tables = $encrypt_tables_orig; +SET GLOBAL innodb_encryption_threads=@start_global_value; +EVAL SET GLOBAL encryption_algorithm = $orig_algorithm; +EVAL SET GLOBAL innodb_file_per_table = $innodb_file_per_table_orig; +EVAL SET GLOBAL innodb_file_format = $innodb_file_format_orig; +--enable_query_log + diff --git a/mysql-test/suite/sys_vars/r/encryption_algorithm_basic.result b/mysql-test/suite/sys_vars/r/encryption_algorithm_basic.result index a9101b0f950..02858af1d8d 100644 --- a/mysql-test/suite/sys_vars/r/encryption_algorithm_basic.result +++ b/mysql-test/suite/sys_vars/r/encryption_algorithm_basic.result @@ -3,5 +3,6 @@ select @@global.encryption_algorithm; none select @@session.encryption_algorithm; ERROR HY000: Variable 'encryption_algorithm' is a GLOBAL variable -set global encryption_algorithm="none"; -ERROR HY000: Variable 'encryption_algorithm' is a read only variable +set global encryption_algorithm = aes_cbc; +set global encryption_algorithm = aes_cbc; +ERROR 42000: Access denied; you need (at least one of) the SUPER privilege(s) for this operation diff --git a/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result b/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result index 7908300bf8f..9d4c2f7bbca 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result +++ b/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result @@ -706,8 +706,8 @@ VARIABLE_COMMENT Which encryption algorithm to use for table encryption. aes_cbc NUMERIC_MIN_VALUE NULL NUMERIC_MAX_VALUE NULL NUMERIC_BLOCK_SIZE NULL -ENUM_VALUE_LIST none,aes_ecb,aes_cbc,aes_ctr -READ_ONLY YES +ENUM_VALUE_LIST none,aes_cbc,aes_ctr +READ_ONLY NO COMMAND_LINE_ARGUMENT REQUIRED VARIABLE_NAME ENCRYPT_TMP_DISK_TABLES SESSION_VALUE NULL diff --git a/mysql-test/suite/sys_vars/t/encryption_algorithm_basic.test b/mysql-test/suite/sys_vars/t/encryption_algorithm_basic.test index 065453eba34..8e1e1906f1e 100644 --- a/mysql-test/suite/sys_vars/t/encryption_algorithm_basic.test +++ b/mysql-test/suite/sys_vars/t/encryption_algorithm_basic.test @@ -1,13 +1,33 @@ -# bool global +# enum global +--disable_query_log +let $orig_algorithm=`SELECT @@encryption_algorithm`; +--enable_query_log +# # exists as global only # select @@global.encryption_algorithm; --error ER_INCORRECT_GLOBAL_LOCAL_VAR select @@session.encryption_algorithm; +set global encryption_algorithm = aes_cbc; + # -# show that it's not writable +# Check changeable only by super # ---error 1238 -set global encryption_algorithm="none"; +--source include/add_anonymous_users.inc + +connect (con1,localhost,user_1,,); +connection con1; +--error 1227 +set global encryption_algorithm = aes_cbc; + +connection default; + +--source include/delete_anonymous_users.inc + +# reset system +--disable_query_log +EVAL SET GLOBAL encryption_algorithm = $orig_algorithm; +--enable_query_log + diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index 734b6ed9171..31b0e6fd22a 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -63,7 +63,7 @@ #include "opt_range.h" #include "rpl_parallel.h" #include "encryption_keys.h" - +#include "my_aes.h" /* The rule for this file: everything should be 'static'. When a sys_var variable or a function from this file is - in very rare cases - needed @@ -5185,13 +5185,30 @@ static Sys_var_mybool Sys_encrypt_tmp_disk_tables( GLOBAL_VAR(encrypt_tmp_disk_tables), CMD_LINE(OPT_ARG), DEFAULT(FALSE)); + +static bool fix_encryption_algorithm(sys_var *self, THD *thd, + enum_var_type type) +{ + if (my_aes_init_dynamic_encrypt((enum_my_aes_encryption_algorithm) + encryption_algorithm)) + { + fprintf(stderr, "Can't initialize encryption algorithm to \"%s\".\nCheck that the program is linked with the right library (openssl?)\n", + encryption_algorithm_names[encryption_algorithm]); + return true; + } + + return false; +} + const char *encryption_algorithm_names[]= -{ "none", "aes_ecb", "aes_cbc", "aes_ctr", 0 }; +{ "none", "aes_cbc", "aes_ctr", 0 }; static Sys_var_enum Sys_encryption_algorithm( "encryption_algorithm", "Which encryption algorithm to use for table encryption. aes_cbc is the recommended one.", - READ_ONLY GLOBAL_VAR(encryption_algorithm),CMD_LINE(REQUIRED_ARG), - encryption_algorithm_names, DEFAULT(0)); + GLOBAL_VAR(encryption_algorithm),CMD_LINE(REQUIRED_ARG), + encryption_algorithm_names, DEFAULT(0), + NO_MUTEX_GUARD, NOT_IN_BINLOG, + ON_CHECK(0), ON_UPDATE(fix_encryption_algorithm)); static bool check_pseudo_slave_mode(sys_var *self, THD *thd, set_var *var) { diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index a4825adbbbb..6f01f884a83 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -2454,6 +2454,19 @@ buf_block_align_instance( == 0xffffffff); break; case BUF_BLOCK_FILE_PAGE: + if (block->page.space + != page_get_space_id(page_align(ptr))) { + fprintf(stderr, "Block space id %du page space id %lu page type %s\n", + block->page.space, page_get_space_id(page_align(ptr)), + fil_get_page_type_name(mach_read_from_4(ptr + FIL_PAGE_TYPE))); + } + + if (block->page.offset + != page_get_page_no(page_align(ptr))) { + fprintf(stderr, "Block offset %du page offset %lu page type %s\n", + block->page.offset, page_get_page_no(page_align(ptr)), + fil_get_page_type_name(mach_read_from_4(ptr + FIL_PAGE_TYPE))); + } ut_ad(block->page.space == page_get_space_id(page_align(ptr))); ut_ad(block->page.offset diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index 07636d27879..bc927b86e70 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -23,8 +23,6 @@ Created Jonas Oreland Google Modified Jan Lindström jan.lindstrom@mariadb.com *******************************************************/ -#include "fil0fil.h" -#include "fil0crypt.h" #include "srv0srv.h" #include "srv0start.h" #include "mach0data.h" @@ -37,6 +35,8 @@ Modified Jan Lindström jan.lindstrom@mariadb.com #include "fsp0fsp.h" #include "fil0pagecompress.h" #include "ha_prototypes.h" // IB_LOG_ +#include "fil0fil.h" +#include "fil0crypt.h" #include @@ -119,34 +119,6 @@ static const unsigned char CRYPT_MAGIC[MAGIC_SZ] = { static const unsigned char EMPTY_PATTERN[MAGIC_SZ] = { 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 }; -/****************************************************************** -Map used AES method to crypt scheme -@return used AES crypt scheme */ -UNIV_INTERN -uint -fil_crypt_get_aes_method( -/*=====================*/ - uint aes_method) -{ - switch (aes_method) { - case MY_AES_ALGORITHM_NONE: - return (uint) CRYPT_SCHEME_1_UNENCRYPTED; - break; - case MY_AES_ALGORITHM_CTR: - return (uint) CRYPT_SCHEME_1_CTR; - break; - case MY_AES_ALGORITHM_CBC: - return (uint) CRYPT_SCHEME_1_CBC; - break; - default: - ib_logf(IB_LOG_LEVEL_FATAL, - "Current AES method %d not supported.\n", aes_method); - ut_error; - } - - return (uint) CRYPT_SCHEME_1_UNENCRYPTED; -} - /********************************************************************* Init space crypt */ UNIV_INTERN @@ -659,29 +631,6 @@ fil_space_check_encryption_write( return true; } -/****************************************************************** -Map current aes method -@return AES method */ -UNIV_INTERN -uint -fil_crypt_map_aes_method( -/*=====================*/ - uint aes_method) /*!< in: AES method */ -{ - switch((fil_crypt_method_t)aes_method) { - case CRYPT_SCHEME_1_CTR: - return (uint)MY_AES_ALGORITHM_CTR; - break; - case CRYPT_SCHEME_1_CBC: - return (uint)MY_AES_ALGORITHM_CBC; - break; - default: - ib_logf(IB_LOG_LEVEL_FATAL, - "Current AES method %d not supported.\n", (int)aes_method); - ut_error; - } -} - /****************************************************************** Encrypt a page */ UNIV_INTERN diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 9d0bb03c0e9..35d55af819b 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -6858,7 +6858,7 @@ Get crypt data for a tablespace */ UNIV_INTERN fil_space_crypt_t* fil_space_get_crypt_data( -/*==================*/ +/*=====================*/ ulint id) /*!< in: space id */ { fil_space_t* space; @@ -6879,12 +6879,29 @@ fil_space_get_crypt_data( return(crypt_data); } +/****************************************************************** +Get crypt data for a tablespace */ +UNIV_INTERN +fil_space_crypt_t* +fil_space_crypt_data( +/*=================*/ + fil_space_t* space) /*!< in: filespace */ +{ + fil_space_crypt_t* crypt_data = NULL; + + if (space != NULL) { + crypt_data = space->crypt_data; + } + + return(crypt_data); +} + /****************************************************************** Get crypt data for a tablespace */ UNIV_INTERN void fil_space_set_crypt_data( -/*==================*/ +/*=====================*/ ulint id, /*!< in: space id */ fil_space_crypt_t* crypt_data) /*!< in: crypt data */ { diff --git a/storage/innobase/fil/fil0pagecompress.cc b/storage/innobase/fil/fil0pagecompress.cc index 1c3db26372d..4b5931489d2 100644 --- a/storage/innobase/fil/fil0pagecompress.cc +++ b/storage/innobase/fil/fil0pagecompress.cc @@ -87,6 +87,7 @@ static ulint srv_data_read, srv_data_written; For page compressed pages compress the page before actual write operation. @return compressed page to be written*/ +UNIV_INTERN byte* fil_compress_page( /*==============*/ @@ -379,6 +380,7 @@ fil_compress_page( /****************************************************************//** For page compressed pages decompress the page after actual read operation. */ +UNIV_INTERN void fil_decompress_page( /*================*/ diff --git a/storage/innobase/include/fil0crypt.ic b/storage/innobase/include/fil0crypt.ic index b753ca5cb4f..48862910dc4 100644 --- a/storage/innobase/include/fil0crypt.ic +++ b/storage/innobase/include/fil0crypt.ic @@ -67,3 +67,54 @@ fil_page_encryption_status( } return 0; } + +/****************************************************************** +Map used AES method to crypt scheme +@return used AES crypt scheme */ +UNIV_INLINE +uint +fil_crypt_get_aes_method( +/*=====================*/ + uint aes_method) +{ + switch (aes_method) { + case MY_AES_ALGORITHM_NONE: + return (uint) CRYPT_SCHEME_1_UNENCRYPTED; + break; + case MY_AES_ALGORITHM_CTR: + return (uint) CRYPT_SCHEME_1_CTR; + break; + case MY_AES_ALGORITHM_CBC: + return (uint) CRYPT_SCHEME_1_CBC; + break; + default: + ib_logf(IB_LOG_LEVEL_FATAL, + "Current AES method %d not supported.\n", aes_method); + ut_error; + } + + return (uint) CRYPT_SCHEME_1_UNENCRYPTED; +} + +/****************************************************************** +Map current aes method +@return AES method */ +UNIV_INLINE +uint +fil_crypt_map_aes_method( +/*=====================*/ + uint aes_method) /*!< in: AES method */ +{ + switch((fil_crypt_method_t)aes_method) { + case CRYPT_SCHEME_1_CTR: + return (uint)MY_AES_ALGORITHM_CTR; + break; + case CRYPT_SCHEME_1_CBC: + return (uint)MY_AES_ALGORITHM_CBC; + break; + default: + ib_logf(IB_LOG_LEVEL_FATAL, + "Current AES method %d not supported.\n", (int)aes_method); + ut_error; + } +} diff --git a/storage/innobase/include/fil0pagecompress.h b/storage/innobase/include/fil0pagecompress.h index c797c221efc..1c2c1730b8e 100644 --- a/storage/innobase/include/fil0pagecompress.h +++ b/storage/innobase/include/fil0pagecompress.h @@ -34,6 +34,7 @@ Created 11/12/2013 Jan Lindström jan.lindstrom@skysql.com Returns the page compression level flag of the space, or 0 if the space is not compressed. The tablespace must be cached in the memory cache. @return page compression level if page compressed, ULINT_UNDEFINED if space not found */ +UNIV_INTERN ulint fil_space_get_page_compression_level( /*=================================*/ @@ -42,6 +43,7 @@ fil_space_get_page_compression_level( Returns the page compression flag of the space, or false if the space is not compressed. The tablespace must be cached in the memory cache. @return true if page compressed, false if not or space not found */ +UNIV_INTERN ibool fil_space_is_page_compressed( /*=========================*/ @@ -50,6 +52,7 @@ fil_space_is_page_compressed( Returns the page compression flag of the space, or false if the space is not compressed. The tablespace must be cached in the memory cache. @return true if page compressed, false if not or space not found */ +UNIV_INTERN ibool fil_space_get_page_compressed( /*=========================*/ @@ -58,6 +61,7 @@ fil_space_get_page_compressed( Returns the atomic writes flag of the space, or false if the space is not using atomic writes. The tablespace must be cached in the memory cache. @return atomic write table option value */ +UNIV_INTERN atomic_writes_t fil_space_get_atomic_writes( /*=========================*/ @@ -65,6 +69,7 @@ fil_space_get_atomic_writes( /*******************************************************************//** Find out wheather the page is index page or not @return true if page type index page, false if not */ +UNIV_INTERN ibool fil_page_is_index_page( /*===================*/ @@ -74,6 +79,7 @@ fil_page_is_index_page( Get the name of the compression algorithm used for page compression. @return compression algorithm name or "UNKNOWN" if not known*/ +UNIV_INTERN const char* fil_get_compression_alg_name( /*=========================*/ @@ -83,6 +89,7 @@ fil_get_compression_alg_name( For page compressed pages compress the page before actual write operation. @return compressed page to be written*/ +UNIV_INTERN byte* fil_compress_page( /*==============*/ @@ -102,6 +109,7 @@ fil_compress_page( For page compressed pages decompress the page after actual read operation. @return uncompressed page */ +UNIV_INTERN void fil_decompress_page( /*================*/ @@ -115,6 +123,7 @@ fil_decompress_page( /****************************************************************//** Get space id from fil node @return space id*/ +UNIV_INTERN ulint fil_node_get_space_id( /*==================*/ @@ -123,13 +132,16 @@ fil_node_get_space_id( /****************************************************************//** Get block size from fil node @return block size*/ +UNIV_INTERN ulint fil_node_get_block_size( +/*====================*/ fil_node_t* node); /*!< in: Node where to get block size */ /*******************************************************************//** Find out wheather the page is page compressed -@return true if page is page compressed*/ +@return true if page is page compressed */ +UNIV_INTERN ibool fil_page_is_compressed( /*===================*/ @@ -138,6 +150,7 @@ fil_page_is_compressed( /*******************************************************************//** Find out wheather the page is page compressed with lzo method @return true if page is page compressed with lzo method*/ +UNIV_INTERN ibool fil_page_is_lzo_compressed( /*=======================*/ diff --git a/storage/innobase/include/log0crypt.h b/storage/innobase/include/log0crypt.h index 9c7c0229ba4..cbb430d5209 100644 --- a/storage/innobase/include/log0crypt.h +++ b/storage/innobase/include/log0crypt.h @@ -23,6 +23,8 @@ extern my_bool srv_encrypt_log; extern byte redo_log_crypt_msg[MY_AES_BLOCK_SIZE]; /* IV to concatenate with counter used by AES_CTR for redo log crypto. */ extern byte aes_ctr_nonce[MY_AES_BLOCK_SIZE]; +/* AES method used for redo log encryption/decryption. */ +extern byte redo_aes_method; /*********************************************************************//** Generate a 128-bit random message used to generate redo log crypto key. diff --git a/storage/innobase/include/log0log.h b/storage/innobase/include/log0log.h index 79667097724..f3a60c59fc2 100644 --- a/storage/innobase/include/log0log.h +++ b/storage/innobase/include/log0log.h @@ -678,7 +678,7 @@ extern log_t* log_sys; #define LOG_CHECKPOINT_OFFSET_HIGH32 (16 + LOG_CHECKPOINT_ARRAY_END) #define LOG_CRYPT_VER (20 + LOG_CHECKPOINT_ARRAY_END) /*!< 32-bit key version. Corresponding - key has been used for log records with + key has been used for log records with lsn <= the checkpoint' lsn */ #define LOG_CRYPT_MSG (24 + LOG_CHECKPOINT_ARRAY_END) /*!< a 128-bit value used to @@ -688,8 +688,10 @@ extern log_t* log_sys; 15-byte random number.*/ #define LOG_CRYPT_IV (40 + LOG_CHECKPOINT_ARRAY_END) /*!< a 128-bit random number used as - AES-CTR iv/nonce for redo log */ -#define LOG_CHECKPOINT_SIZE (56 + LOG_CHECKPOINT_ARRAY_END) + AES-* iv/nonce for redo log */ +#define LOG_CRYPT_METHOD (56 + LOG_CHECKPOINT_ARRAY_END) + /*!< AES method used */ +#define LOG_CHECKPOINT_SIZE (57 + LOG_CHECKPOINT_ARRAY_END) /* Offsets of a log file header */ #define LOG_GROUP_ID 0 /* log group number */ diff --git a/storage/innobase/log/log0crypt.cc b/storage/innobase/log/log0crypt.cc index 6f487e36b56..349338b974c 100644 --- a/storage/innobase/log/log0crypt.cc +++ b/storage/innobase/log/log0crypt.cc @@ -30,6 +30,9 @@ Modified Jan Lindström jan.lindstrom@mariadb.com #include "log0log.h" #include "srv0start.h" // for srv_start_lsn #include "log0recv.h" // for recv_sys +#include "ha_prototypes.h" // ib_logf +#include "fil0fil.h" // for page types +#include "fil0crypt.h" // for aes_method #include "mysql/plugin_encryption_key_management.h" // for BAD_ENCRYPTION_KEY_VERSION #include "ha_prototypes.h" // IB_LOG_ @@ -43,15 +46,17 @@ UNIV_INTERN my_bool srv_encrypt_log = FALSE; static const byte redo_log_purpose_byte = 0x02; /* Plain text used by AES_ECB to generate redo log crypt key. */ byte redo_log_crypt_msg[MY_AES_BLOCK_SIZE] = {0}; -/* IV to concatenate with counter used by AES_CTR for redo log +/* IV to concatenate with counter used by AES for redo log * encryption/decryption. */ byte aes_ctr_nonce[MY_AES_BLOCK_SIZE] = {0}; +/* AES method used for redo log encryption/decryption. */ +byte redo_aes_method = 0; /*********************************************************************//** Generate a 128-bit value used to generate crypt key for redo log. It is generated via the concatenation of 1 purpose byte (0x02) and 15-byte random number. -Init AES-CTR iv/nonce with random number. +Init AES iv/nonce with random number. It is called when: - redo logs do not exist when start up, or - transition from without crypto. @@ -69,8 +74,8 @@ log_init_crypt_msg_and_nonce(void) /*==============================*/ { mach_write_to_1(redo_log_crypt_msg, redo_log_purpose_byte); - if (my_random_bytes(redo_log_crypt_msg + 1, PURPOSE_BYTE_LEN) != AES_OK) - { + + if (my_random_bytes(redo_log_crypt_msg + 1, PURPOSE_BYTE_LEN) != AES_OK) { ib_logf(IB_LOG_LEVEL_ERROR, "Redo log crypto: generate " "%u-byte random number as crypto msg failed.", @@ -78,14 +83,16 @@ log_init_crypt_msg_and_nonce(void) abort(); } - if (my_random_bytes(aes_ctr_nonce, MY_AES_BLOCK_SIZE) != AES_OK) - { + if (my_random_bytes(aes_ctr_nonce, MY_AES_BLOCK_SIZE) != AES_OK) { ib_logf(IB_LOG_LEVEL_ERROR, "Redo log crypto: generate " - "%u-byte random number as AES_CTR nonce failed.", + "%u-byte random number failed.", MY_AES_BLOCK_SIZE); abort(); } + + /* Set up current AES method */ + redo_aes_method = (byte)fil_crypt_get_aes_method(current_aes_dynamic_method); } /*********************************************************************//** @@ -98,16 +105,14 @@ log_init_crypt_key( const uint crypt_ver, /*< in: key version */ byte* key) /*< out: crypt key*/ { - if (crypt_ver == UNENCRYPTED_KEY_VER) - { + if (crypt_ver == UNENCRYPTED_KEY_VER) { ib_logf(IB_LOG_LEVEL_INFO, "Redo log crypto: unencrypted key ver."); memset(key, 0, MY_AES_BLOCK_SIZE); return; } - if (crypt_msg[PURPOSE_BYTE_OFFSET] != redo_log_purpose_byte) - { + if (crypt_msg[PURPOSE_BYTE_OFFSET] != redo_log_purpose_byte) { ib_logf(IB_LOG_LEVEL_ERROR, "Redo log crypto: msg type mismatched. " "Expected: %x; Actual: %x.", @@ -116,8 +121,8 @@ log_init_crypt_key( } byte mysqld_key[MY_AES_BLOCK_SIZE] = {0}; - if (get_encryption_key(crypt_ver, mysqld_key, MY_AES_BLOCK_SIZE)) - { + + if (get_encryption_key(crypt_ver, mysqld_key, MY_AES_BLOCK_SIZE)) { ib_logf(IB_LOG_LEVEL_ERROR, "Redo log crypto: getting mysqld crypto key " "from key version failed."); @@ -125,15 +130,16 @@ log_init_crypt_key( } uint32 dst_len; + my_aes_encrypt_dynamic_type func= get_aes_encrypt_func(MY_AES_ALGORITHM_ECB); + int rc= (*func)(crypt_msg, MY_AES_BLOCK_SIZE, //src, srclen key, &dst_len, //dst, &dstlen (unsigned char*)&mysqld_key, sizeof(mysqld_key), NULL, 0, 1); - if (rc != AES_OK || dst_len != MY_AES_BLOCK_SIZE) - { + if (rc != AES_OK || dst_len != MY_AES_BLOCK_SIZE) { ib_logf(IB_LOG_LEVEL_ERROR, "Redo log crypto: getting redo log crypto key " "failed."); @@ -158,7 +164,7 @@ log_block_get_start_lsn( } /*********************************************************************//** -Call AES CTR to encrypt/decrypt log blocks. */ +Call AES to encrypt/decrypt log blocks. */ static Crypt_result log_blocks_crypt( @@ -175,21 +181,22 @@ log_blocks_crypt( ulint log_block_no, log_block_start_lsn; byte *key; ulint lsn; - if (is_encrypt) - { + uint aes_method = (uint)redo_aes_method; + + if (is_encrypt) { ut_a(log_sys && log_sys->redo_log_crypt_ver != UNENCRYPTED_KEY_VER); key = (byte *)(log_sys->redo_log_crypt_key); lsn = log_sys->lsn; - } else { ut_a(recv_sys && recv_sys->recv_log_crypt_ver != UNENCRYPTED_KEY_VER); key = (byte *)(recv_sys->recv_log_crypt_key); lsn = srv_start_lsn; } + ut_a(size % OS_FILE_LOG_BLOCK_SIZE == 0); src_len = OS_FILE_LOG_BLOCK_SIZE - LOG_BLOCK_HDR_SIZE; - for (ulint i = 0; i < size ; i += OS_FILE_LOG_BLOCK_SIZE) - { + + for (ulint i = 0; i < size ; i += OS_FILE_LOG_BLOCK_SIZE) { log_block_no = log_block_get_hdr_no(log_block); log_block_start_lsn = log_block_get_start_lsn(lsn, log_block_no); @@ -205,11 +212,13 @@ log_blocks_crypt( mach_write_to_4(aes_ctr_counter + 11, log_block_no); bzero(aes_ctr_counter + 15, 1); - int rc = (* my_aes_encrypt_dynamic)(log_block + LOG_BLOCK_HDR_SIZE, src_len, - dst_block + LOG_BLOCK_HDR_SIZE, &dst_len, - (unsigned char*)key, 16, - aes_ctr_counter, MY_AES_BLOCK_SIZE, - 1); + my_aes_encrypt_dynamic_type func = get_aes_encrypt_func((enum_my_aes_encryption_algorithm)fil_crypt_map_aes_method(aes_method)); + + int rc = (*func)(log_block + LOG_BLOCK_HDR_SIZE, src_len, + dst_block + LOG_BLOCK_HDR_SIZE, &dst_len, + (unsigned char*)key, 16, + aes_ctr_counter, MY_AES_BLOCK_SIZE, + 1); ut_a(rc == AES_OK); ut_a(dst_len == src_len); @@ -302,13 +311,17 @@ log_crypt_write_checkpoint_buf( { ut_a(log_sys); mach_write_to_4(buf + LOG_CRYPT_VER, log_sys->redo_log_crypt_ver); + if (!srv_encrypt_log || log_sys->redo_log_crypt_ver == UNENCRYPTED_KEY_VER) { memset(buf + LOG_CRYPT_MSG, 0, MY_AES_BLOCK_SIZE); memset(buf + LOG_CRYPT_IV, 0, MY_AES_BLOCK_SIZE); + memset(buf + LOG_CRYPT_METHOD, 0, 1); return; } + ut_a(redo_log_crypt_msg[PURPOSE_BYTE_OFFSET] == redo_log_purpose_byte); memcpy(buf + LOG_CRYPT_MSG, redo_log_crypt_msg, MY_AES_BLOCK_SIZE); memcpy(buf + LOG_CRYPT_IV, aes_ctr_nonce, MY_AES_BLOCK_SIZE); + mach_write_to_1(buf + LOG_CRYPT_METHOD, redo_aes_method); } diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index 3b8c4878e32..c8a65290100 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -3088,13 +3088,16 @@ recv_recovery_from_checkpoint_start_func( #ifdef UNIV_LOG_ARCHIVE archived_lsn = mach_read_from_8(buf + LOG_CHECKPOINT_ARCHIVED_LSN); #endif /* UNIV_LOG_ARCHIVE */ + recv_crypt_ver = mach_read_from_4(buf + LOG_CRYPT_VER); + if (recv_crypt_ver == UNENCRYPTED_KEY_VER) { log_init_crypt_msg_and_nonce(); } else { ut_memcpy(redo_log_crypt_msg, buf + LOG_CRYPT_MSG, MY_AES_BLOCK_SIZE); ut_memcpy(aes_ctr_nonce, buf + LOG_CRYPT_IV, MY_AES_BLOCK_SIZE); + redo_aes_method = (byte)mach_read_from_1(buf + LOG_CRYPT_METHOD); } /* Read the first log file header to print a note if this is diff --git a/storage/xtradb/buf/buf0buf.cc b/storage/xtradb/buf/buf0buf.cc index ab8dbadf0fa..1bccc695223 100644 --- a/storage/xtradb/buf/buf0buf.cc +++ b/storage/xtradb/buf/buf0buf.cc @@ -2450,6 +2450,19 @@ buf_block_align_instance( == 0xffffffff); break; case BUF_BLOCK_FILE_PAGE: + if (block->page.space + != page_get_space_id(page_align(ptr))) { + fprintf(stderr, "Block space id %du page space id %lu page type %s\n", + block->page.space, page_get_space_id(page_align(ptr)), + fil_get_page_type_name(mach_read_from_4(ptr + FIL_PAGE_TYPE))); + } + + if (block->page.offset + != page_get_page_no(page_align(ptr))) { + fprintf(stderr, "Block offset %du page offset %lu page type %s\n", + block->page.offset, page_get_page_no(page_align(ptr)), + fil_get_page_type_name(mach_read_from_4(ptr + FIL_PAGE_TYPE))); + } ut_ad(block->page.space == page_get_space_id(page_align(ptr))); ut_ad(block->page.offset diff --git a/storage/xtradb/fil/fil0crypt.cc b/storage/xtradb/fil/fil0crypt.cc index 07636d27879..fcff53e6823 100644 --- a/storage/xtradb/fil/fil0crypt.cc +++ b/storage/xtradb/fil/fil0crypt.cc @@ -119,33 +119,6 @@ static const unsigned char CRYPT_MAGIC[MAGIC_SZ] = { static const unsigned char EMPTY_PATTERN[MAGIC_SZ] = { 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 }; -/****************************************************************** -Map used AES method to crypt scheme -@return used AES crypt scheme */ -UNIV_INTERN -uint -fil_crypt_get_aes_method( -/*=====================*/ - uint aes_method) -{ - switch (aes_method) { - case MY_AES_ALGORITHM_NONE: - return (uint) CRYPT_SCHEME_1_UNENCRYPTED; - break; - case MY_AES_ALGORITHM_CTR: - return (uint) CRYPT_SCHEME_1_CTR; - break; - case MY_AES_ALGORITHM_CBC: - return (uint) CRYPT_SCHEME_1_CBC; - break; - default: - ib_logf(IB_LOG_LEVEL_FATAL, - "Current AES method %d not supported.\n", aes_method); - ut_error; - } - - return (uint) CRYPT_SCHEME_1_UNENCRYPTED; -} /********************************************************************* Init space crypt */ @@ -659,29 +632,6 @@ fil_space_check_encryption_write( return true; } -/****************************************************************** -Map current aes method -@return AES method */ -UNIV_INTERN -uint -fil_crypt_map_aes_method( -/*=====================*/ - uint aes_method) /*!< in: AES method */ -{ - switch((fil_crypt_method_t)aes_method) { - case CRYPT_SCHEME_1_CTR: - return (uint)MY_AES_ALGORITHM_CTR; - break; - case CRYPT_SCHEME_1_CBC: - return (uint)MY_AES_ALGORITHM_CBC; - break; - default: - ib_logf(IB_LOG_LEVEL_FATAL, - "Current AES method %d not supported.\n", (int)aes_method); - ut_error; - } -} - /****************************************************************** Encrypt a page */ UNIV_INTERN diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index 1084f90c720..f78e281eca6 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -6992,7 +6992,7 @@ Get crypt data for a tablespace */ UNIV_INTERN fil_space_crypt_t* fil_space_get_crypt_data( -/*==================*/ +/*=====================*/ ulint id) /*!< in: space id */ { fil_space_t* space; @@ -7013,12 +7013,29 @@ fil_space_get_crypt_data( return(crypt_data); } +/****************************************************************** +Get crypt data for a tablespace */ +UNIV_INTERN +fil_space_crypt_t* +fil_space_crypt_data( +/*=================*/ + fil_space_t* space) /*!< in: filespace */ +{ + fil_space_crypt_t* crypt_data = NULL; + + if (space != NULL) { + crypt_data = space->crypt_data; + } + + return(crypt_data); +} + /****************************************************************** Get crypt data for a tablespace */ UNIV_INTERN void fil_space_set_crypt_data( -/*==================*/ +/*=====================*/ ulint id, /*!< in: space id */ fil_space_crypt_t* crypt_data) /*!< in: crypt data */ { diff --git a/storage/xtradb/fil/fil0pagecompress.cc b/storage/xtradb/fil/fil0pagecompress.cc index d394ca5215c..c36165bcf9c 100644 --- a/storage/xtradb/fil/fil0pagecompress.cc +++ b/storage/xtradb/fil/fil0pagecompress.cc @@ -87,6 +87,7 @@ static ulint srv_data_read, srv_data_written; For page compressed pages compress the page before actual write operation. @return compressed page to be written*/ +UNIV_INTERN byte* fil_compress_page( /*==============*/ @@ -373,6 +374,7 @@ fil_compress_page( /****************************************************************//** For page compressed pages decompress the page after actual read operation. */ +UNIV_INTERN void fil_decompress_page( /*================*/ diff --git a/storage/xtradb/include/fil0crypt.ic b/storage/xtradb/include/fil0crypt.ic index b753ca5cb4f..48862910dc4 100644 --- a/storage/xtradb/include/fil0crypt.ic +++ b/storage/xtradb/include/fil0crypt.ic @@ -67,3 +67,54 @@ fil_page_encryption_status( } return 0; } + +/****************************************************************** +Map used AES method to crypt scheme +@return used AES crypt scheme */ +UNIV_INLINE +uint +fil_crypt_get_aes_method( +/*=====================*/ + uint aes_method) +{ + switch (aes_method) { + case MY_AES_ALGORITHM_NONE: + return (uint) CRYPT_SCHEME_1_UNENCRYPTED; + break; + case MY_AES_ALGORITHM_CTR: + return (uint) CRYPT_SCHEME_1_CTR; + break; + case MY_AES_ALGORITHM_CBC: + return (uint) CRYPT_SCHEME_1_CBC; + break; + default: + ib_logf(IB_LOG_LEVEL_FATAL, + "Current AES method %d not supported.\n", aes_method); + ut_error; + } + + return (uint) CRYPT_SCHEME_1_UNENCRYPTED; +} + +/****************************************************************** +Map current aes method +@return AES method */ +UNIV_INLINE +uint +fil_crypt_map_aes_method( +/*=====================*/ + uint aes_method) /*!< in: AES method */ +{ + switch((fil_crypt_method_t)aes_method) { + case CRYPT_SCHEME_1_CTR: + return (uint)MY_AES_ALGORITHM_CTR; + break; + case CRYPT_SCHEME_1_CBC: + return (uint)MY_AES_ALGORITHM_CBC; + break; + default: + ib_logf(IB_LOG_LEVEL_FATAL, + "Current AES method %d not supported.\n", (int)aes_method); + ut_error; + } +} diff --git a/storage/xtradb/include/fil0pagecompress.h b/storage/xtradb/include/fil0pagecompress.h index c797c221efc..1c2c1730b8e 100644 --- a/storage/xtradb/include/fil0pagecompress.h +++ b/storage/xtradb/include/fil0pagecompress.h @@ -34,6 +34,7 @@ Created 11/12/2013 Jan Lindström jan.lindstrom@skysql.com Returns the page compression level flag of the space, or 0 if the space is not compressed. The tablespace must be cached in the memory cache. @return page compression level if page compressed, ULINT_UNDEFINED if space not found */ +UNIV_INTERN ulint fil_space_get_page_compression_level( /*=================================*/ @@ -42,6 +43,7 @@ fil_space_get_page_compression_level( Returns the page compression flag of the space, or false if the space is not compressed. The tablespace must be cached in the memory cache. @return true if page compressed, false if not or space not found */ +UNIV_INTERN ibool fil_space_is_page_compressed( /*=========================*/ @@ -50,6 +52,7 @@ fil_space_is_page_compressed( Returns the page compression flag of the space, or false if the space is not compressed. The tablespace must be cached in the memory cache. @return true if page compressed, false if not or space not found */ +UNIV_INTERN ibool fil_space_get_page_compressed( /*=========================*/ @@ -58,6 +61,7 @@ fil_space_get_page_compressed( Returns the atomic writes flag of the space, or false if the space is not using atomic writes. The tablespace must be cached in the memory cache. @return atomic write table option value */ +UNIV_INTERN atomic_writes_t fil_space_get_atomic_writes( /*=========================*/ @@ -65,6 +69,7 @@ fil_space_get_atomic_writes( /*******************************************************************//** Find out wheather the page is index page or not @return true if page type index page, false if not */ +UNIV_INTERN ibool fil_page_is_index_page( /*===================*/ @@ -74,6 +79,7 @@ fil_page_is_index_page( Get the name of the compression algorithm used for page compression. @return compression algorithm name or "UNKNOWN" if not known*/ +UNIV_INTERN const char* fil_get_compression_alg_name( /*=========================*/ @@ -83,6 +89,7 @@ fil_get_compression_alg_name( For page compressed pages compress the page before actual write operation. @return compressed page to be written*/ +UNIV_INTERN byte* fil_compress_page( /*==============*/ @@ -102,6 +109,7 @@ fil_compress_page( For page compressed pages decompress the page after actual read operation. @return uncompressed page */ +UNIV_INTERN void fil_decompress_page( /*================*/ @@ -115,6 +123,7 @@ fil_decompress_page( /****************************************************************//** Get space id from fil node @return space id*/ +UNIV_INTERN ulint fil_node_get_space_id( /*==================*/ @@ -123,13 +132,16 @@ fil_node_get_space_id( /****************************************************************//** Get block size from fil node @return block size*/ +UNIV_INTERN ulint fil_node_get_block_size( +/*====================*/ fil_node_t* node); /*!< in: Node where to get block size */ /*******************************************************************//** Find out wheather the page is page compressed -@return true if page is page compressed*/ +@return true if page is page compressed */ +UNIV_INTERN ibool fil_page_is_compressed( /*===================*/ @@ -138,6 +150,7 @@ fil_page_is_compressed( /*******************************************************************//** Find out wheather the page is page compressed with lzo method @return true if page is page compressed with lzo method*/ +UNIV_INTERN ibool fil_page_is_lzo_compressed( /*=======================*/ diff --git a/storage/xtradb/include/log0crypt.h b/storage/xtradb/include/log0crypt.h index 9c7c0229ba4..cbb430d5209 100644 --- a/storage/xtradb/include/log0crypt.h +++ b/storage/xtradb/include/log0crypt.h @@ -23,6 +23,8 @@ extern my_bool srv_encrypt_log; extern byte redo_log_crypt_msg[MY_AES_BLOCK_SIZE]; /* IV to concatenate with counter used by AES_CTR for redo log crypto. */ extern byte aes_ctr_nonce[MY_AES_BLOCK_SIZE]; +/* AES method used for redo log encryption/decryption. */ +extern byte redo_aes_method; /*********************************************************************//** Generate a 128-bit random message used to generate redo log crypto key. diff --git a/storage/xtradb/include/log0log.h b/storage/xtradb/include/log0log.h index cda4a692aec..b40ebd5d7a9 100644 --- a/storage/xtradb/include/log0log.h +++ b/storage/xtradb/include/log0log.h @@ -748,7 +748,7 @@ extern log_t* log_sys; #define LOG_CHECKPOINT_OFFSET_HIGH32 (16 + LOG_CHECKPOINT_ARRAY_END) #define LOG_CRYPT_VER (20 + LOG_CHECKPOINT_ARRAY_END) /*!< 32-bit key version. Corresponding - key has been used for log records with + key has been used for log records with lsn <= the checkpoint' lsn */ #define LOG_CRYPT_MSG (24 + LOG_CHECKPOINT_ARRAY_END) /*!< a 128-bit value used to @@ -758,8 +758,10 @@ extern log_t* log_sys; 15-byte random number.*/ #define LOG_CRYPT_IV (40 + LOG_CHECKPOINT_ARRAY_END) /*!< a 128-bit random number used as - AES-CTR iv/nonce for redo log */ -#define LOG_CHECKPOINT_SIZE (56 + LOG_CHECKPOINT_ARRAY_END) + AES-* iv/nonce for redo log */ +#define LOG_CRYPT_METHOD (56 + LOG_CHECKPOINT_ARRAY_END) + /*!< AES method used */ +#define LOG_CHECKPOINT_SIZE (57 + LOG_CHECKPOINT_ARRAY_END) /* Offsets of a log file header */ #define LOG_GROUP_ID 0 /* log group number */ diff --git a/storage/xtradb/log/log0crypt.cc b/storage/xtradb/log/log0crypt.cc index abebdd8c61a..97ea84e00ad 100644 --- a/storage/xtradb/log/log0crypt.cc +++ b/storage/xtradb/log/log0crypt.cc @@ -31,6 +31,8 @@ Modified Jan Lindström jan.lindstrom@mariadb.com #include "log0log.h" #include "srv0start.h" // for srv_start_lsn #include "log0recv.h" // for recv_sys +#include "fil0fil.h" // for page types +#include "fil0crypt.h" // for aes_method #include "mysql/plugin_encryption_key_management.h" // for BAD_ENCRYPTION_KEY_VERSION #include "ha_prototypes.h" // IB_LOG_ @@ -44,15 +46,17 @@ UNIV_INTERN my_bool srv_encrypt_log = FALSE; static const byte redo_log_purpose_byte = 0x02; /* Plain text used by AES_ECB to generate redo log crypt key. */ byte redo_log_crypt_msg[MY_AES_BLOCK_SIZE] = {0}; -/* IV to concatenate with counter used by AES_CTR for redo log +/* IV to concatenate with counter used by AES for redo log * encryption/decryption. */ byte aes_ctr_nonce[MY_AES_BLOCK_SIZE] = {0}; +/* AES method used for redo log encryption/decryption. */ +byte redo_aes_method = 0; /*********************************************************************//** Generate a 128-bit value used to generate crypt key for redo log. It is generated via the concatenation of 1 purpose byte (0x02) and 15-byte random number. -Init AES-CTR iv/nonce with random number. +Init AES iv/nonce with random number. It is called when: - redo logs do not exist when start up, or - transition from without crypto. @@ -70,8 +74,8 @@ log_init_crypt_msg_and_nonce(void) /*==============================*/ { mach_write_to_1(redo_log_crypt_msg, redo_log_purpose_byte); - if (my_random_bytes(redo_log_crypt_msg + 1, PURPOSE_BYTE_LEN) != AES_OK) - { + + if (my_random_bytes(redo_log_crypt_msg + 1, PURPOSE_BYTE_LEN) != AES_OK) { ib_logf(IB_LOG_LEVEL_ERROR, "Redo log crypto: generate " "%u-byte random number as crypto msg failed.", @@ -79,14 +83,16 @@ log_init_crypt_msg_and_nonce(void) abort(); } - if (my_random_bytes(aes_ctr_nonce, MY_AES_BLOCK_SIZE) != AES_OK) - { + if (my_random_bytes(aes_ctr_nonce, MY_AES_BLOCK_SIZE) != AES_OK) { ib_logf(IB_LOG_LEVEL_ERROR, "Redo log crypto: generate " - "%u-byte random number as AES_CTR nonce failed.", + "%u-byte random number failed.", MY_AES_BLOCK_SIZE); abort(); } + + /* Set up current AES method */ + redo_aes_method = (byte)fil_crypt_get_aes_method(current_aes_dynamic_method); } /*********************************************************************//** @@ -99,16 +105,14 @@ log_init_crypt_key( const uint crypt_ver, /*< in: key version */ byte* key) /*< out: crypt key*/ { - if (crypt_ver == UNENCRYPTED_KEY_VER) - { + if (crypt_ver == UNENCRYPTED_KEY_VER) { ib_logf(IB_LOG_LEVEL_INFO, "Redo log crypto: unencrypted key ver."); memset(key, 0, MY_AES_BLOCK_SIZE); return; } - if (crypt_msg[PURPOSE_BYTE_OFFSET] != redo_log_purpose_byte) - { + if (crypt_msg[PURPOSE_BYTE_OFFSET] != redo_log_purpose_byte) { ib_logf(IB_LOG_LEVEL_ERROR, "Redo log crypto: msg type mismatched. " "Expected: %x; Actual: %x.", @@ -117,8 +121,8 @@ log_init_crypt_key( } byte mysqld_key[MY_AES_BLOCK_SIZE] = {0}; - if (get_encryption_key(crypt_ver, mysqld_key, MY_AES_BLOCK_SIZE)) - { + + if (get_encryption_key(crypt_ver, mysqld_key, MY_AES_BLOCK_SIZE)) { ib_logf(IB_LOG_LEVEL_ERROR, "Redo log crypto: getting mysqld crypto key " "from key version failed."); @@ -126,15 +130,16 @@ log_init_crypt_key( } uint32 dst_len; + my_aes_encrypt_dynamic_type func= get_aes_encrypt_func(MY_AES_ALGORITHM_ECB); + int rc= (*func)(crypt_msg, MY_AES_BLOCK_SIZE, //src, srclen key, &dst_len, //dst, &dstlen (unsigned char*)&mysqld_key, sizeof(mysqld_key), NULL, 0, 1); - if (rc != AES_OK || dst_len != MY_AES_BLOCK_SIZE) - { + if (rc != AES_OK || dst_len != MY_AES_BLOCK_SIZE) { ib_logf(IB_LOG_LEVEL_ERROR, "Redo log crypto: getting redo log crypto key " "failed."); @@ -159,7 +164,7 @@ log_block_get_start_lsn( } /*********************************************************************//** -Call AES CTR to encrypt/decrypt log blocks. */ +Call AES to encrypt/decrypt log blocks. */ static Crypt_result log_blocks_crypt( @@ -176,21 +181,22 @@ log_blocks_crypt( ulint log_block_no, log_block_start_lsn; byte *key; ulint lsn; - if (is_encrypt) - { + uint aes_method = (uint)redo_aes_method; + + if (is_encrypt) { ut_a(log_sys && log_sys->redo_log_crypt_ver != UNENCRYPTED_KEY_VER); key = (byte *)(log_sys->redo_log_crypt_key); lsn = log_sys->lsn; - } else { ut_a(recv_sys && recv_sys->recv_log_crypt_ver != UNENCRYPTED_KEY_VER); key = (byte *)(recv_sys->recv_log_crypt_key); lsn = srv_start_lsn; } + ut_a(size % OS_FILE_LOG_BLOCK_SIZE == 0); src_len = OS_FILE_LOG_BLOCK_SIZE - LOG_BLOCK_HDR_SIZE; - for (ulint i = 0; i < size ; i += OS_FILE_LOG_BLOCK_SIZE) - { + + for (ulint i = 0; i < size ; i += OS_FILE_LOG_BLOCK_SIZE) { log_block_no = log_block_get_hdr_no(log_block); log_block_start_lsn = log_block_get_start_lsn(lsn, log_block_no); @@ -206,11 +212,13 @@ log_blocks_crypt( mach_write_to_4(aes_ctr_counter + 11, log_block_no); bzero(aes_ctr_counter + 15, 1); - int rc = (* my_aes_encrypt_dynamic)(log_block + LOG_BLOCK_HDR_SIZE, src_len, - dst_block + LOG_BLOCK_HDR_SIZE, &dst_len, - (unsigned char*)key, 16, - aes_ctr_counter, MY_AES_BLOCK_SIZE, - 1); + my_aes_encrypt_dynamic_type func = get_aes_encrypt_func((enum_my_aes_encryption_algorithm)fil_crypt_map_aes_method(aes_method)); + + int rc = (*func)(log_block + LOG_BLOCK_HDR_SIZE, src_len, + dst_block + LOG_BLOCK_HDR_SIZE, &dst_len, + (unsigned char*)key, 16, + aes_ctr_counter, MY_AES_BLOCK_SIZE, + 1); ut_a(rc == AES_OK); ut_a(dst_len == src_len); @@ -303,13 +311,17 @@ log_crypt_write_checkpoint_buf( { ut_a(log_sys); mach_write_to_4(buf + LOG_CRYPT_VER, log_sys->redo_log_crypt_ver); + if (!srv_encrypt_log || log_sys->redo_log_crypt_ver == UNENCRYPTED_KEY_VER) { memset(buf + LOG_CRYPT_MSG, 0, MY_AES_BLOCK_SIZE); memset(buf + LOG_CRYPT_IV, 0, MY_AES_BLOCK_SIZE); + memset(buf + LOG_CRYPT_METHOD, 0, 1); return; } + ut_a(redo_log_crypt_msg[PURPOSE_BYTE_OFFSET] == redo_log_purpose_byte); memcpy(buf + LOG_CRYPT_MSG, redo_log_crypt_msg, MY_AES_BLOCK_SIZE); memcpy(buf + LOG_CRYPT_IV, aes_ctr_nonce, MY_AES_BLOCK_SIZE); + mach_write_to_1(buf + LOG_CRYPT_METHOD, redo_aes_method); } diff --git a/storage/xtradb/log/log0recv.cc b/storage/xtradb/log/log0recv.cc index 46f5954d44f..61386930b22 100644 --- a/storage/xtradb/log/log0recv.cc +++ b/storage/xtradb/log/log0recv.cc @@ -3164,13 +3164,16 @@ recv_recovery_from_checkpoint_start_func( #ifdef UNIV_LOG_ARCHIVE archived_lsn = mach_read_from_8(buf + LOG_CHECKPOINT_ARCHIVED_LSN); #endif /* UNIV_LOG_ARCHIVE */ + recv_crypt_ver = mach_read_from_4(buf + LOG_CRYPT_VER); + if (recv_crypt_ver == UNENCRYPTED_KEY_VER) { log_init_crypt_msg_and_nonce(); } else { ut_memcpy(redo_log_crypt_msg, buf + LOG_CRYPT_MSG, MY_AES_BLOCK_SIZE); ut_memcpy(aes_ctr_nonce, buf + LOG_CRYPT_IV, MY_AES_BLOCK_SIZE); + redo_aes_method = (byte)mach_read_from_1(buf + LOG_CRYPT_METHOD); } /* Read the first log file header to print a note if this is From af768c2f228f90c949c6dae94d34a7296ffd943c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Wed, 1 Apr 2015 22:03:14 +0300 Subject: [PATCH 4/6] InnoDB/XtraDB Encryption code cleanup Step 4: -- Review fixes -- Rename pages_page_* to pages_* -- Remove unnecessary code --- storage/innobase/fil/fil0crypt.cc | 54 ++++++--------- storage/innobase/handler/ha_innodb.cc | 10 ++- storage/innobase/include/srv0mon.h | 5 +- storage/innobase/include/srv0srv.h | 20 +++--- storage/innobase/srv/srv0mon.cc | 28 +++----- storage/innobase/srv/srv0srv.cc | 5 +- storage/xtradb/fil/fil0crypt.cc | 54 ++++++--------- storage/xtradb/fil/fil0fil.cc | 97 +++++++++------------------ storage/xtradb/handler/ha_innodb.cc | 10 ++- storage/xtradb/include/srv0mon.h | 5 +- storage/xtradb/include/srv0srv.h | 20 +++--- storage/xtradb/srv/srv0mon.cc | 28 +++----- storage/xtradb/srv/srv0srv.cc | 5 +- 13 files changed, 128 insertions(+), 213 deletions(-) diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index bc927b86e70..b2137328267 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -180,20 +180,12 @@ fil_crypt_get_key( crypt_data->keys[i] = crypt_data->keys[i - 1]; } - if (has_encryption_key(version)) { - int rc; - *key_length = get_encryption_key_size(version); + *key_length = sizeof(keybuf); + int rc = get_encryption_key(version, (unsigned char*)keybuf, *key_length); - rc = get_encryption_key(version, (unsigned char*)keybuf, *key_length); - - if (rc != CRYPT_KEY_OK) { - ib_logf(IB_LOG_LEVEL_FATAL, - "Key %d can not be found. Reason=%d", version, rc); - ut_error; - } - } else { + if (rc != CRYPT_KEY_OK) { ib_logf(IB_LOG_LEVEL_FATAL, - "Key %d not found", version); + "Key %d can not be found. Reason=%d", version, rc); ut_error; } @@ -207,15 +199,15 @@ fil_crypt_get_key( /* We use AES_ECB to encryp IV */ my_aes_encrypt_dynamic_type func = get_aes_encrypt_func(MY_AES_ALGORITHM_ECB); - int rc = (*func)(src, /* Data to be encrypted = IV */ - srclen, /* data length */ - buf, /* Output buffer */ - &buflen, /* Output buffer */ - keybuf, /* Key */ - *key_length, /* Key length */ - NULL, /* AES_ECB does not use IV */ - 0, /* IV-length */ - 1); /* NoPadding */ + rc = (*func)(src, /* Data to be encrypted = IV */ + srclen, /* data length */ + buf, /* Output buffer */ + &buflen, /* Output buffer */ + keybuf, /* Key */ + *key_length, /* Key length */ + NULL, /* AES_ECB does not use IV */ + 0, /* IV-length */ + 1); /* NoPadding */ if (rc != AES_OK) { ib_logf(IB_LOG_LEVEL_FATAL, @@ -250,14 +242,12 @@ fil_crypt_get_latest_key( fil_space_crypt_t* crypt_data, /*!< in: crypt data */ uint* version) /*!< in: Key version */ { - if (srv_encrypt_tables) { - // used for key rotation - get the next key id from the key provider - int rc = get_latest_encryption_key_version(); + // used for key rotation - get the next key id from the key provider + int rc = get_latest_encryption_key_version(); - // if no new key was created use the last one - if (rc >= 0) { - *version = rc; - } + // if no new key was created use the last one + if (rc >= 0) { + *version = rc; } return fil_crypt_get_key(dst, key_length, crypt_data, *version); @@ -423,10 +413,6 @@ fil_space_destroy_crypt_data( fil_space_crypt_t **crypt_data) /*!< out: crypt data */ { if (crypt_data != NULL && (*crypt_data) != NULL) { - /* lock (and unlock) mutex to make sure no one has it locked - * currently */ - mutex_enter(& (*crypt_data)->mutex); - mutex_exit(& (*crypt_data)->mutex); mutex_free(& (*crypt_data)->mutex); free(*crypt_data); (*crypt_data) = NULL; @@ -755,6 +741,8 @@ fil_space_encrypt( /* Store compression algorithm (for page compresed tables) or 0 */ mach_write_to_2(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 6, compression_algo); + + srv_stats.pages_encrypted.inc(); } /********************************************************************* @@ -934,6 +922,8 @@ fil_space_decrypt( compression_algo); } + srv_stats.pages_decrypted.inc(); + return true; /* page was decrypted */ } diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index bd2d49995a0..bc49b1988f1 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -815,12 +815,10 @@ static SHOW_VAR innodb_status_variables[]= { (char*) &export_vars.innodb_pages_page_decompressed, SHOW_LONGLONG}, {"num_pages_page_compression_error", (char*) &export_vars.innodb_pages_page_compression_error, SHOW_LONGLONG}, - {"num_pages_page_encrypted", - (char*) &export_vars.innodb_pages_page_encrypted, SHOW_LONGLONG}, - {"num_pages_page_decrypted", - (char*) &export_vars.innodb_pages_page_decrypted, SHOW_LONGLONG}, - {"num_pages_page_encryption_error", - (char*) &export_vars.innodb_pages_page_encryption_error, SHOW_LONGLONG}, + {"num_pages_encrypted", + (char*) &export_vars.innodb_pages_encrypted, SHOW_LONGLONG}, + {"num_pages_decrypted", + (char*) &export_vars.innodb_pages_decrypted, SHOW_LONGLONG}, {"have_lz4", (char*) &innodb_have_lz4, SHOW_BOOL}, {"have_lzo", diff --git a/storage/innobase/include/srv0mon.h b/storage/innobase/include/srv0mon.h index d1d902ef57d..79a8a5bf848 100644 --- a/storage/innobase/include/srv0mon.h +++ b/storage/innobase/include/srv0mon.h @@ -328,9 +328,8 @@ enum monitor_id_t { MONITOR_OVLD_PAGES_PAGE_COMPRESSION_ERROR, /* New monitor variables for page encryption */ - MONITOR_OVLD_PAGES_PAGE_ENCRYPTED, - MONITOR_OVLD_PAGES_PAGE_DECRYPTED, - MONITOR_OVLD_PAGES_PAGE_ENCRYPTION_ERROR, + MONITOR_OVLD_PAGES_ENCRYPTED, + MONITOR_OVLD_PAGES_DECRYPTED, /* Index related counters */ MONITOR_MODULE_INDEX, diff --git a/storage/innobase/include/srv0srv.h b/storage/innobase/include/srv0srv.h index 314bce63c7b..bfc65dbc51a 100644 --- a/storage/innobase/include/srv0srv.h +++ b/storage/innobase/include/srv0srv.h @@ -133,12 +133,10 @@ struct srv_stats_t { ulint_ctr_64_t pages_page_decompressed; /* Number of page compression errors */ ulint_ctr_64_t pages_page_compression_error; - /* Number of pages encrypted with page encryption */ - ulint_ctr_64_t pages_page_encrypted; - /* Number of pages decrypted with page encryption */ - ulint_ctr_64_t pages_page_decrypted; - /* Number of page encryption errors */ - ulint_ctr_64_t pages_page_encryption_error; + /* Number of pages encrypted */ + ulint_ctr_64_t pages_encrypted; + /* Number of pages decrypted */ + ulint_ctr_64_t pages_decrypted; /** Number of data read in total (in bytes) */ ulint_ctr_1_t data_read; @@ -1020,12 +1018,10 @@ struct export_var_t{ compression */ ib_int64_t innodb_pages_page_compression_error;/*!< Number of page compression errors */ - ib_int64_t innodb_pages_page_encrypted;/*!< Number of pages - encrypted by page encryption */ - ib_int64_t innodb_pages_page_decrypted;/*!< Number of pages - decrypted by page encryption */ - ib_int64_t innodb_pages_page_encryption_error;/*!< Number of page - encryption errors */ + ib_int64_t innodb_pages_encrypted; /*!< Number of pages + encrypted */ + ib_int64_t innodb_pages_decrypted; /*!< Number of pages + decrypted */ ulint innodb_sec_rec_cluster_reads; /*!< srv_sec_rec_cluster_reads */ ulint innodb_sec_rec_cluster_reads_avoided; diff --git a/storage/innobase/srv/srv0mon.cc b/storage/innobase/srv/srv0mon.cc index 39481a90c00..0bcc876f591 100644 --- a/storage/innobase/srv/srv0mon.cc +++ b/storage/innobase/srv/srv0mon.cc @@ -985,20 +985,15 @@ static monitor_info_t innodb_counter_info[] = MONITOR_NONE, MONITOR_DEFAULT_START, MONITOR_OVLD_PAGES_PAGE_COMPRESSION_ERROR}, - {"compress_pages_page_encrypted", "compression", - "Number of pages encrypted by page encryption", + {"compress_pages_encrypted", "compression", + "Number of pages encrypted", MONITOR_NONE, - MONITOR_DEFAULT_START, MONITOR_OVLD_PAGES_PAGE_ENCRYPTED}, + MONITOR_DEFAULT_START, MONITOR_OVLD_PAGES_ENCRYPTED}, - {"compress_pages_page_decrypted", "compression", - "Number of pages decrypted by page encryption", + {"compress_pages_decrypted", "compression", + "Number of pages decrypted", MONITOR_NONE, - MONITOR_DEFAULT_START, MONITOR_OVLD_PAGES_PAGE_DECRYPTED}, - - {"compress_pages_page_encryption_error", "compression", - "Number of page encryption errors ", - MONITOR_NONE, - MONITOR_DEFAULT_START, MONITOR_OVLD_PAGES_PAGE_ENCRYPTION_ERROR}, + MONITOR_DEFAULT_START, MONITOR_OVLD_PAGES_DECRYPTED}, /* ========== Counters for Index ========== */ {"module_index", "index", "Index Manager", @@ -2014,14 +2009,11 @@ srv_mon_process_existing_counter( case MONITOR_OVLD_PAGES_PAGE_COMPRESSION_ERROR: value = srv_stats.pages_page_compression_error; break; - case MONITOR_OVLD_PAGES_PAGE_ENCRYPTED: - value = srv_stats.pages_page_encrypted; + case MONITOR_OVLD_PAGES_ENCRYPTED: + value = srv_stats.pages_encrypted; break; - case MONITOR_OVLD_PAGES_PAGE_DECRYPTED: - value = srv_stats.pages_page_decrypted; - break; - case MONITOR_OVLD_PAGES_PAGE_ENCRYPTION_ERROR: - value = srv_stats.pages_page_encryption_error; + case MONITOR_OVLD_PAGES_DECRYPTED: + value = srv_stats.pages_decrypted; break; default: diff --git a/storage/innobase/srv/srv0srv.cc b/storage/innobase/srv/srv0srv.cc index d8c921fb612..c19da5dfd91 100644 --- a/storage/innobase/srv/srv0srv.cc +++ b/storage/innobase/srv/srv0srv.cc @@ -1601,9 +1601,8 @@ srv_export_innodb_status(void) export_vars.innodb_page_compressed_trim_op_saved = srv_stats.page_compressed_trim_op_saved; export_vars.innodb_pages_page_decompressed = srv_stats.pages_page_decompressed; export_vars.innodb_pages_page_compression_error = srv_stats.pages_page_compression_error; - export_vars.innodb_pages_page_decrypted = srv_stats.pages_page_decrypted; - export_vars.innodb_pages_page_encrypted = srv_stats.pages_page_encrypted; - export_vars.innodb_pages_page_encryption_error = srv_stats.pages_page_encryption_error; + export_vars.innodb_pages_decrypted = srv_stats.pages_decrypted; + export_vars.innodb_pages_encrypted = srv_stats.pages_encrypted; export_vars.innodb_defragment_compression_failures = btr_defragment_compression_failures; diff --git a/storage/xtradb/fil/fil0crypt.cc b/storage/xtradb/fil/fil0crypt.cc index fcff53e6823..c1ad7c88e68 100644 --- a/storage/xtradb/fil/fil0crypt.cc +++ b/storage/xtradb/fil/fil0crypt.cc @@ -181,20 +181,12 @@ fil_crypt_get_key( crypt_data->keys[i] = crypt_data->keys[i - 1]; } - if (has_encryption_key(version)) { - int rc; - *key_length = get_encryption_key_size(version); + *key_length = sizeof(keybuf); + int rc = get_encryption_key(version, (unsigned char*)keybuf, *key_length); - rc = get_encryption_key(version, (unsigned char*)keybuf, *key_length); - - if (rc != CRYPT_KEY_OK) { - ib_logf(IB_LOG_LEVEL_FATAL, - "Key %d can not be found. Reason=%d", version, rc); - ut_error; - } - } else { + if (rc != CRYPT_KEY_OK) { ib_logf(IB_LOG_LEVEL_FATAL, - "Key %d not found", version); + "Key %d can not be found. Reason=%d", version, rc); ut_error; } @@ -208,15 +200,15 @@ fil_crypt_get_key( /* We use AES_ECB to encryp IV */ my_aes_encrypt_dynamic_type func = get_aes_encrypt_func(MY_AES_ALGORITHM_ECB); - int rc = (*func)(src, /* Data to be encrypted = IV */ - srclen, /* data length */ - buf, /* Output buffer */ - &buflen, /* Output buffer */ - keybuf, /* Key */ - *key_length, /* Key length */ - NULL, /* AES_ECB does not use IV */ - 0, /* IV-length */ - 1); /* NoPadding */ + rc = (*func)(src, /* Data to be encrypted = IV */ + srclen, /* data length */ + buf, /* Output buffer */ + &buflen, /* Output buffer */ + keybuf, /* Key */ + *key_length, /* Key length */ + NULL, /* AES_ECB does not use IV */ + 0, /* IV-length */ + 1); /* NoPadding */ if (rc != AES_OK) { ib_logf(IB_LOG_LEVEL_FATAL, @@ -251,14 +243,12 @@ fil_crypt_get_latest_key( fil_space_crypt_t* crypt_data, /*!< in: crypt data */ uint* version) /*!< in: Key version */ { - if (srv_encrypt_tables) { - // used for key rotation - get the next key id from the key provider - int rc = get_latest_encryption_key_version(); + // used for key rotation - get the next key id from the key provider + int rc = get_latest_encryption_key_version(); - // if no new key was created use the last one - if (rc >= 0) { - *version = rc; - } + // if no new key was created use the last one + if (rc >= 0) { + *version = rc; } return fil_crypt_get_key(dst, key_length, crypt_data, *version); @@ -424,10 +414,6 @@ fil_space_destroy_crypt_data( fil_space_crypt_t **crypt_data) /*!< out: crypt data */ { if (crypt_data != NULL && (*crypt_data) != NULL) { - /* lock (and unlock) mutex to make sure no one has it locked - * currently */ - mutex_enter(& (*crypt_data)->mutex); - mutex_exit(& (*crypt_data)->mutex); mutex_free(& (*crypt_data)->mutex); free(*crypt_data); (*crypt_data) = NULL; @@ -756,6 +742,8 @@ fil_space_encrypt( /* Store compression algorithm (for page compresed tables) or 0 */ mach_write_to_2(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 6, compression_algo); + + srv_stats.pages_encrypted.inc(); } /********************************************************************* @@ -935,6 +923,8 @@ fil_space_decrypt( compression_algo); } + srv_stats.pages_decrypted.inc(); + return true; /* page was decrypted */ } diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index f78e281eca6..e4925182ee7 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -650,20 +650,6 @@ fil_node_open_file( success = os_file_read(node->handle, page, 0, UNIV_PAGE_SIZE); - if (fil_page_is_encrypted(page)) { - /* if page is (still) encrypted, write an error and return. - * Otherwise the server would crash if decrypting is not possible. - * This may be the case, if the key file could not be - * opened on server startup. - */ - ib_logf(IB_LOG_LEVEL_ERROR, - "InnoDB: can not decrypt page, because " - "keys could not be read.\n" - ); - return false; - - } - space_id = fsp_header_get_space_id(page); flags = fsp_header_get_flags(page); @@ -1939,7 +1925,6 @@ fil_check_first_page( { ulint space_id; ulint flags; - ulint page_is_encrypted; if (srv_force_recovery >= SRV_FORCE_IGNORE_CORRUPT) { return(NULL); @@ -1947,20 +1932,14 @@ fil_check_first_page( space_id = mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_ID + page); flags = mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + page); - /* Note: the 1st page is usually not encrypted. If the Key Provider - or the encryption key is not available, the - check for reading the first page should intentionally fail - with "can not decrypt" message. */ - page_is_encrypted = fil_page_encryption_status(page, space_id); - if (!page_is_encrypted) { - if (UNIV_PAGE_SIZE != fsp_flags_get_page_size(flags)) { - fprintf(stderr, - "InnoDB: Error: Current page size %lu != " - " page size on page %lu\n", - UNIV_PAGE_SIZE, fsp_flags_get_page_size(flags)); - return("innodb-page-size mismatch"); - } + if (UNIV_PAGE_SIZE != fsp_flags_get_page_size(flags)) { + fprintf(stderr, + "InnoDB: Error: Current page size %lu != " + " page size on page %lu\n", + UNIV_PAGE_SIZE, fsp_flags_get_page_size(flags)); + + return("innodb-page-size mismatch"); } if (!space_id && !flags) { @@ -1976,17 +1955,9 @@ fil_check_first_page( } } - if (!page_is_encrypted && buf_page_is_corrupted( + if (buf_page_is_corrupted( false, page, fsp_flags_get_zip_size(flags))) { return("checksum mismatch"); - } else { - if (page_is_encrypted) { - /* this error message is interpreted by the calling method, which is - * executed if the server starts in recovery mode. - */ - return(FIL_MSG_CANNOT_DECRYPT); - - } } if (page_get_space_id(page) == space_id @@ -2024,6 +1995,7 @@ fil_read_first_page( byte* page; lsn_t flushed_lsn; const char* check_msg = NULL; + fil_space_crypt_t* cdata; buf = static_cast(ut_malloc(2 * UNIV_PAGE_SIZE)); @@ -2042,13 +2014,6 @@ fil_read_first_page( *space_id = fsp_header_get_space_id(page); } - /* Page is page compressed page, need to decompress, before - continue. */ - if (fil_page_is_compressed(page)) { - ulint write_size=0; - fil_decompress_page(NULL, page, UNIV_PAGE_SIZE, &write_size); - } - if (!one_read_already) { check_msg = fil_check_first_page(page); } @@ -2056,12 +2021,30 @@ fil_read_first_page( flushed_lsn = mach_read_from_8(page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); + + ulint space = fsp_header_get_space_id(page); + ulint offset = fsp_header_get_crypt_offset( + fsp_flags_get_zip_size(*flags), NULL); + cdata = fil_space_read_crypt_data(space, page, offset); + + /* If file space is encrypted we need to have at least some + encryption service available where to get keys */ + if ((cdata && cdata->encryption == FIL_SPACE_ENCRYPTION_ON) || + ( srv_encrypt_tables && + cdata && cdata->encryption == FIL_SPACE_ENCRYPTION_DEFAULT)) { + int rc = get_latest_encryption_key_version(); + + if (rc < 0) { + ib_logf(IB_LOG_LEVEL_FATAL, + "Tablespace id %ld encrypted but encryption service" + " not available. Can't continue opening tablespace.\n", + space); + ut_error; + } + } + if (crypt_data) { - ulint space = fsp_header_get_space_id(page); - ulint offset = - fsp_header_get_crypt_offset( - fsp_flags_get_zip_size(*flags), NULL); - *crypt_data = fil_space_read_crypt_data(space, page, offset); + *crypt_data = cdata; } ut_free(buf); @@ -4216,7 +4199,6 @@ fil_validate_single_table_tablespace( check_first_page: fsp->success = TRUE; - fsp->encryption_error = 0; if (const char* check_msg = fil_read_first_page( fsp->file, FALSE, &fsp->flags, &fsp->id, &fsp->lsn, &fsp->lsn, ULINT_UNDEFINED, &fsp->crypt_data)) { @@ -4224,14 +4206,6 @@ check_first_page: "%s in tablespace %s (table %s)", check_msg, fsp->filepath, tablename); fsp->success = FALSE; - if (strncmp(check_msg, FIL_MSG_CANNOT_DECRYPT, strlen(check_msg))==0) { - /* by returning here, it should be avoided, that the server crashes, - * if started in recovery mode and can not decrypt tables, if - * the key file can not be read. - */ - fsp->encryption_error = 1; - return; - } } if (!fsp->success) { @@ -4386,13 +4360,6 @@ fil_load_single_table_tablespace( if (!def.success && !remote.success) { - if (def.encryption_error || remote.encryption_error) { - fprintf(stderr, - "InnoDB: Error: could not open single-table" - " tablespace file %s. Encryption error!\n", def.filepath); - return; - } - /* The following call prints an error message */ os_file_get_last_error(true); fprintf(stderr, diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index 9f4f52b4f4a..e9ec7c1087b 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -1013,12 +1013,10 @@ static SHOW_VAR innodb_status_variables[]= { (char*) &export_vars.innodb_pages_page_decompressed, SHOW_LONGLONG}, {"num_pages_page_compression_error", (char*) &export_vars.innodb_pages_page_compression_error, SHOW_LONGLONG}, - {"num_pages_page_encrypted", - (char*) &export_vars.innodb_pages_page_encrypted, SHOW_LONGLONG}, - {"num_pages_page_decrypted", - (char*) &export_vars.innodb_pages_page_decrypted, SHOW_LONGLONG}, - {"num_pages_page_encryption_error", - (char*) &export_vars.innodb_pages_page_encryption_error, SHOW_LONGLONG}, + {"num_pages_encrypted", + (char*) &export_vars.innodb_pages_encrypted, SHOW_LONGLONG}, + {"num_pages_decrypted", + (char*) &export_vars.innodb_pages_decrypted, SHOW_LONGLONG}, {"have_lz4", (char*) &innodb_have_lz4, SHOW_BOOL}, {"have_lzo", diff --git a/storage/xtradb/include/srv0mon.h b/storage/xtradb/include/srv0mon.h index 70868463448..094eb0de610 100644 --- a/storage/xtradb/include/srv0mon.h +++ b/storage/xtradb/include/srv0mon.h @@ -329,9 +329,8 @@ enum monitor_id_t { MONITOR_OVLD_PAGES_PAGE_COMPRESSION_ERROR, /* New monitor variables for page encryption */ - MONITOR_OVLD_PAGES_PAGE_ENCRYPTED, - MONITOR_OVLD_PAGES_PAGE_DECRYPTED, - MONITOR_OVLD_PAGES_PAGE_ENCRYPTION_ERROR, + MONITOR_OVLD_PAGES_ENCRYPTED, + MONITOR_OVLD_PAGES_DECRYPTED, /* Index related counters */ MONITOR_MODULE_INDEX, diff --git a/storage/xtradb/include/srv0srv.h b/storage/xtradb/include/srv0srv.h index 7387f11bd6e..3cf312a4b4d 100644 --- a/storage/xtradb/include/srv0srv.h +++ b/storage/xtradb/include/srv0srv.h @@ -133,12 +133,10 @@ struct srv_stats_t { ulint_ctr_64_t pages_page_decompressed; /* Number of page compression errors */ ulint_ctr_64_t pages_page_compression_error; - /* Number of pages encrypted with page encryption */ - ulint_ctr_64_t pages_page_encrypted; - /* Number of pages decrypted with page encryption */ - ulint_ctr_64_t pages_page_decrypted; - /* Number of page encryption errors */ - ulint_ctr_64_t pages_page_encryption_error; + /* Number of pages encrypted */ + ulint_ctr_64_t pages_encrypted; + /* Number of pages decrypted */ + ulint_ctr_64_t pages_decrypted; /** Number of data read in total (in bytes) */ ulint_ctr_1_t data_read; @@ -1238,12 +1236,10 @@ struct export_var_t{ compression */ ib_int64_t innodb_pages_page_compression_error;/*!< Number of page compression errors */ - ib_int64_t innodb_pages_page_encrypted;/*!< Number of pages - encrypted by page encryption */ - ib_int64_t innodb_pages_page_decrypted;/*!< Number of pages - decrypted by page encryption */ - ib_int64_t innodb_pages_page_encryption_error;/*!< Number of page - encryption errors */ + ib_int64_t innodb_pages_encrypted; /*!< Number of pages + encrypted */ + ib_int64_t innodb_pages_decrypted; /*!< Number of pages + decrypted */ ulint innodb_sec_rec_cluster_reads; /*!< srv_sec_rec_cluster_reads */ ulint innodb_sec_rec_cluster_reads_avoided; diff --git a/storage/xtradb/srv/srv0mon.cc b/storage/xtradb/srv/srv0mon.cc index 5bac6214c5b..6595c42ca40 100644 --- a/storage/xtradb/srv/srv0mon.cc +++ b/storage/xtradb/srv/srv0mon.cc @@ -985,20 +985,15 @@ static monitor_info_t innodb_counter_info[] = MONITOR_NONE, MONITOR_DEFAULT_START, MONITOR_OVLD_PAGES_PAGE_COMPRESSION_ERROR}, - {"compress_pages_page_encrypted", "compression", - "Number of pages encrypted by page encryption", + {"compress_pages_encrypted", "compression", + "Number of pages encrypted", MONITOR_NONE, - MONITOR_DEFAULT_START, MONITOR_OVLD_PAGES_PAGE_ENCRYPTED}, + MONITOR_DEFAULT_START, MONITOR_OVLD_PAGES_ENCRYPTED}, - {"compress_pages_page_decrypted", "compression", - "Number of pages decrypted by page encryption", + {"compress_pages_decrypted", "compression", + "Number of pages decrypted", MONITOR_NONE, - MONITOR_DEFAULT_START, MONITOR_OVLD_PAGES_PAGE_DECRYPTED}, - - {"compress_pages_page_encryption_error", "compression", - "Number of page encryption errors ", - MONITOR_NONE, - MONITOR_DEFAULT_START, MONITOR_OVLD_PAGES_PAGE_ENCRYPTION_ERROR}, + MONITOR_DEFAULT_START, MONITOR_OVLD_PAGES_DECRYPTED}, /* ========== Counters for Index ========== */ {"module_index", "index", "Index Manager", @@ -2013,14 +2008,11 @@ srv_mon_process_existing_counter( case MONITOR_OVLD_PAGES_PAGE_COMPRESSION_ERROR: value = srv_stats.pages_page_compression_error; break; - case MONITOR_OVLD_PAGES_PAGE_ENCRYPTED: - value = srv_stats.pages_page_encrypted; + case MONITOR_OVLD_PAGES_ENCRYPTED: + value = srv_stats.pages_encrypted; break; - case MONITOR_OVLD_PAGES_PAGE_DECRYPTED: - value = srv_stats.pages_page_decrypted; - break; - case MONITOR_OVLD_PAGES_PAGE_ENCRYPTION_ERROR: - value = srv_stats.pages_page_encryption_error; + case MONITOR_OVLD_PAGES_DECRYPTED: + value = srv_stats.pages_decrypted; break; default: diff --git a/storage/xtradb/srv/srv0srv.cc b/storage/xtradb/srv/srv0srv.cc index df3982b98cd..0391cd9eecb 100644 --- a/storage/xtradb/srv/srv0srv.cc +++ b/storage/xtradb/srv/srv0srv.cc @@ -1990,9 +1990,8 @@ srv_export_innodb_status(void) export_vars.innodb_page_compressed_trim_op_saved = srv_stats.page_compressed_trim_op_saved; export_vars.innodb_pages_page_decompressed = srv_stats.pages_page_decompressed; export_vars.innodb_pages_page_compression_error = srv_stats.pages_page_compression_error; - export_vars.innodb_pages_page_decrypted = srv_stats.pages_page_decrypted; - export_vars.innodb_pages_page_encrypted = srv_stats.pages_page_encrypted; - export_vars.innodb_pages_page_encryption_error = srv_stats.pages_page_encryption_error; + export_vars.innodb_pages_decrypted = srv_stats.pages_decrypted; + export_vars.innodb_pages_encrypted = srv_stats.pages_encrypted; export_vars.innodb_defragment_compression_failures = btr_defragment_compression_failures; From 4865fd105a56540587d065826ce652fb0850f876 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Thu, 2 Apr 2015 13:49:50 +0300 Subject: [PATCH 5/6] InnoDB/XtraDB Encryption cleanup Step 5: -- Rename encryption_key -> encryption_key_id -- Remove unnecessary code -- Fix few bugs found -- Fix test cases and results files --- .../r/innodb-page_encryption-32k.result | 125 ++++++++++++++++++ .../innodb/r/innodb-page_encryption.result | 53 +++----- .../innodb-page_encryption_compression.result | 55 +++----- ...nodb-page_encryption_log_encryption.result | 45 +++---- .../innodb_encryption-page-compression.result | 6 - .../innodb/r/innodb_encryption_change.result | 7 +- .../innodb/r/innodb_encryption_tables.result | 24 ++-- .../suite/innodb/r/innodb_monitor.result | 5 +- .../innodb/t/innodb-page_encryption-32k.test | 16 ++- .../innodb/t/innodb-page_encryption.test | 39 +++--- .../t/innodb-page_encryption_compression.test | 36 +++-- ...innodb-page_encryption_log_encryption.test | 32 +++-- .../t/innodb_encryption-page-compression.test | 10 -- .../innodb/t/innodb_encryption_change.test | 5 +- .../innodb/t/innodb_encryption_tables.test | 16 +-- mysql-test/suite/sys_vars/r/all_vars.result | 1 + ...b_default_page_encryption_key_basic.result | 100 +++++++------- .../r/innodb_monitor_disable_basic.result | 5 +- .../r/innodb_monitor_enable_basic.result | 5 +- .../r/innodb_monitor_reset_all_basic.result | 5 +- .../r/innodb_monitor_reset_basic.result | 5 +- .../suite/sys_vars/r/sysvars_innodb.result | 6 +- ...odb_default_page_encryption_key_basic.test | 56 ++++---- storage/innobase/buf/buf0buf.cc | 3 +- storage/innobase/fil/fil0fil.cc | 88 +++++------- storage/innobase/handler/ha_innodb.cc | 64 ++++----- storage/innobase/handler/ha_innodb.h | 2 +- storage/innobase/handler/handler0alter.cc | 2 +- storage/innobase/include/fil0crypt.h | 2 +- storage/innobase/include/srv0srv.h | 3 +- storage/xtradb/buf/buf0buf.cc | 3 +- storage/xtradb/handler/ha_innodb.cc | 66 ++++----- storage/xtradb/handler/ha_innodb.h | 2 +- storage/xtradb/handler/handler0alter.cc | 2 +- storage/xtradb/include/fil0crypt.h | 2 +- storage/xtradb/include/fsp0fsp.h | 4 +- storage/xtradb/include/srv0srv.h | 3 +- 37 files changed, 466 insertions(+), 437 deletions(-) create mode 100644 mysql-test/suite/innodb/r/innodb-page_encryption-32k.result diff --git a/mysql-test/suite/innodb/r/innodb-page_encryption-32k.result b/mysql-test/suite/innodb/r/innodb-page_encryption-32k.result new file mode 100644 index 00000000000..f46cf798110 --- /dev/null +++ b/mysql-test/suite/innodb/r/innodb-page_encryption-32k.result @@ -0,0 +1,125 @@ +call mtr.add_suppression("InnoDB: Warning: innodb_page_size has been changed from default value *"); +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; +create table innodb_normal(c1 bigint not null, b char(200)) engine=innodb; +create table innodb_compact(c1 bigint not null, b char(200)) engine=innodb row_format=compact encryption='ON' encryption_key_id=1; +create table innodb_dynamic(c1 bigint not null, b char(200)) engine=innodb row_format=dynamic encryption='ON' encryption_key_id=3; +create table innodb_redundant(c1 bigint not null, b char(200)) engine=innodb row_format=redundant encryption='ON' encryption_key_id=4; +show create table innodb_compact; +Table Create Table +innodb_compact CREATE TABLE `innodb_compact` ( + `c1` bigint(20) NOT NULL, + `b` char(200) DEFAULT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=COMPACT `encryption`='ON' `encryption_key_id`=1 +show create table innodb_dynamic; +Table Create Table +innodb_dynamic CREATE TABLE `innodb_dynamic` ( + `c1` bigint(20) NOT NULL, + `b` char(200) DEFAULT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=DYNAMIC `encryption`='ON' `encryption_key_id`=3 +show create table innodb_redundant; +Table Create Table +innodb_redundant CREATE TABLE `innodb_redundant` ( + `c1` bigint(20) NOT NULL, + `b` char(200) DEFAULT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=REDUNDANT `encryption`='ON' `encryption_key_id`=4 +create procedure innodb_insert_proc (repeat_count int) +begin +declare current_num int; +set current_num = 0; +while current_num < repeat_count do +insert into innodb_normal values(current_num, substring(MD5(RAND()), -150)); +set current_num = current_num + 1; +end while; +end// +commit; +set autocommit=0; +call innodb_insert_proc(5000); +commit; +set autocommit=1; +insert into innodb_compact select * from innodb_normal; +insert into innodb_dynamic select * from innodb_normal; +insert into innodb_redundant select * from innodb_normal; +update innodb_compact set c1 = c1 + 1; +update innodb_dynamic set c1 = c1 + 1; +update innodb_redundant set c1 = c1 + 1; +select count(*) from innodb_compact where c1 < 1500000; +count(*) +5000 +select count(*) from innodb_dynamic where c1 < 1500000; +count(*) +5000 +select count(*) from innodb_redundant where c1 < 1500000; +count(*) +5000 +update innodb_compact set c1 = c1 + 1; +update innodb_dynamic set c1 = c1 + 1; +update innodb_redundant set c1 = c1 + 1; +select count(*) from innodb_compact where c1 < 1500000; +count(*) +5000 +select count(*) from innodb_dynamic where c1 < 1500000; +count(*) +5000 +select count(*) from innodb_redundant where c1 < 1500000; +count(*) +5000 +alter table innodb_compact engine=innodb encryption=DEFAULT encryption_key_id=DEFAULT; +show create table innodb_compact; +Table Create Table +innodb_compact CREATE TABLE `innodb_compact` ( + `c1` bigint(20) NOT NULL, + `b` char(200) DEFAULT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=COMPACT +alter table innodb_dynamic engine=innodb encryption=DEFAULT encryption_key_id=DEFAULT; +Warnings: +Warning 1478 InnoDB: ROW_FORMAT=DYNAMIC requires innodb_file_format > Antelope. +Warning 1478 InnoDB: assuming ROW_FORMAT=COMPACT. +show create table innodb_dynamic; +Table Create Table +innodb_dynamic CREATE TABLE `innodb_dynamic` ( + `c1` bigint(20) NOT NULL, + `b` char(200) DEFAULT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=DYNAMIC +alter table innodb_redundant engine=innodb encryption=DEFAULT encryption_key_id=DEFAULT; +show create table innodb_redundant; +Table Create Table +innodb_redundant CREATE TABLE `innodb_redundant` ( + `c1` bigint(20) NOT NULL, + `b` char(200) DEFAULT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=REDUNDANT +show create table innodb_compact; +Table Create Table +innodb_compact CREATE TABLE `innodb_compact` ( + `c1` bigint(20) NOT NULL, + `b` char(200) DEFAULT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=COMPACT +show create table innodb_dynamic; +Table Create Table +innodb_dynamic CREATE TABLE `innodb_dynamic` ( + `c1` bigint(20) NOT NULL, + `b` char(200) DEFAULT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=DYNAMIC +show create table innodb_redundant; +Table Create Table +innodb_redundant CREATE TABLE `innodb_redundant` ( + `c1` bigint(20) NOT NULL, + `b` char(200) DEFAULT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=REDUNDANT +update innodb_compact set c1 = c1 + 1; +update innodb_dynamic set c1 = c1 + 1; +update innodb_redundant set c1 = c1 + 1; +select count(*) from innodb_compact where c1 < 1500000; +count(*) +5000 +select count(*) from innodb_dynamic where c1 < 1500000; +count(*) +5000 +select count(*) from innodb_redundant where c1 < 1500000; +count(*) +5000 +drop procedure innodb_insert_proc; +drop table innodb_normal; +drop table innodb_compact; +drop table innodb_dynamic; +drop table innodb_redundant; diff --git a/mysql-test/suite/innodb/r/innodb-page_encryption.result b/mysql-test/suite/innodb/r/innodb-page_encryption.result index dfc93508b90..ac45e964d12 100644 --- a/mysql-test/suite/innodb/r/innodb-page_encryption.result +++ b/mysql-test/suite/innodb/r/innodb-page_encryption.result @@ -1,42 +1,42 @@ SET GLOBAL innodb_file_format = `Barracuda`; SET GLOBAL innodb_file_per_table = ON; create table innodb_normal(c1 bigint not null, b char(200)) engine=innodb; -create table innodb_compact(c1 bigint not null, b char(200)) engine=innodb row_format=compact page_encryption=1 page_encryption_key=1; -create table innodb_compressed(c1 bigint not null, b char(200)) engine=innodb row_format=compressed page_encryption=1 page_encryption_key=2; -create table innodb_dynamic(c1 bigint not null, b char(200)) engine=innodb row_format=dynamic page_encryption=1 page_encryption_key=3; -create table innodb_redundant(c1 bigint not null, b char(200)) engine=innodb row_format=redundant page_encryption=1 page_encryption_key=4; -SET GLOBAL innodb_default_page_encryption_key = 5; -create table innodb_defkey(c1 bigint not null, b char(200)) engine=innodb page_encryption=1; +create table innodb_compact(c1 bigint not null, b char(200)) engine=innodb row_format=compact encryption='ON' encryption_key_id=1; +create table innodb_compressed(c1 bigint not null, b char(200)) engine=innodb row_format=compressed encryption='ON' encryption_key_id=2; +create table innodb_dynamic(c1 bigint not null, b char(200)) engine=innodb row_format=dynamic encryption='ON' encryption_key_id=3; +create table innodb_redundant(c1 bigint not null, b char(200)) engine=innodb row_format=redundant encryption='ON' encryption_key_id=4; +SET GLOBAL innodb_default_encryption_key = 5; +create table innodb_defkey(c1 bigint not null, b char(200)) engine=innodb encryption='ON'; show create table innodb_defkey; Table Create Table innodb_defkey CREATE TABLE `innodb_defkey` ( `c1` bigint(20) NOT NULL, `b` char(200) DEFAULT NULL -) ENGINE=InnoDB DEFAULT CHARSET=latin1 `page_encryption`=1 +) ENGINE=InnoDB DEFAULT CHARSET=latin1 `encryption`='ON' show create table innodb_compact; Table Create Table innodb_compact CREATE TABLE `innodb_compact` ( `c1` bigint(20) NOT NULL, `b` char(200) DEFAULT NULL -) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=COMPACT `page_encryption`=1 `page_encryption_key`=1 +) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=COMPACT `encryption`='ON' `encryption_key_id`=1 show create table innodb_compressed; Table Create Table innodb_compressed CREATE TABLE `innodb_compressed` ( `c1` bigint(20) NOT NULL, `b` char(200) DEFAULT NULL -) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=COMPRESSED `page_encryption`=1 `page_encryption_key`=2 +) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=COMPRESSED `encryption`='ON' `encryption_key_id`=2 show create table innodb_dynamic; Table Create Table innodb_dynamic CREATE TABLE `innodb_dynamic` ( `c1` bigint(20) NOT NULL, `b` char(200) DEFAULT NULL -) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=DYNAMIC `page_encryption`=1 `page_encryption_key`=3 +) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=DYNAMIC `encryption`='ON' `encryption_key_id`=3 show create table innodb_redundant; Table Create Table innodb_redundant CREATE TABLE `innodb_redundant` ( `c1` bigint(20) NOT NULL, `b` char(200) DEFAULT NULL -) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=REDUNDANT `page_encryption`=1 `page_encryption_key`=4 +) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=REDUNDANT `encryption`='ON' `encryption_key_id`=4 create procedure innodb_insert_proc (repeat_count int) begin declare current_num int; @@ -97,15 +97,12 @@ select count(*) from innodb_defkey t1, innodb_normal t2 where t1.c1 = t2.c1 and t1.b = t2.b; count(*) 2000 -SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_encrypted'; +SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_encrypted'; variable_value >= 0 1 -SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_decrypted'; +SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_decrypted'; variable_value >= 0 1 -SELECT variable_value = 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_encryption_error'; -variable_value = 0 -1 SET GLOBAL innodb_file_format = `Barracuda`; SET GLOBAL innodb_file_per_table = ON; update innodb_normal set c1 = c1 +1; @@ -149,37 +146,34 @@ select count(*) from innodb_defkey t1, innodb_normal t2 where t1.c1 = t2.c1 and t1.b = t2.b; count(*) 2000 -SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_encrypted'; +SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_encrypted'; variable_value >= 0 1 -SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_decrypted'; +SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_decrypted'; variable_value >= 0 1 -SELECT variable_value = 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_encryption_error'; -variable_value = 0 -1 -alter table innodb_compact engine=innodb page_encryption=DEFAULT page_encryption_key=DEFAULT; +alter table innodb_compact engine=innodb encryption=DEFAULT encryption_key_id=DEFAULT; show create table innodb_compact; Table Create Table innodb_compact CREATE TABLE `innodb_compact` ( `c1` bigint(20) NOT NULL, `b` char(200) DEFAULT NULL ) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=COMPACT -alter table innodb_compressed engine=innodb page_encryption=DEFAULT page_encryption_key=DEFAULT; +alter table innodb_compressed engine=innodb encryption=DEFAULT encryption_key_id=DEFAULT; show create table innodb_compressed; Table Create Table innodb_compressed CREATE TABLE `innodb_compressed` ( `c1` bigint(20) NOT NULL, `b` char(200) DEFAULT NULL ) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=COMPRESSED -alter table innodb_dynamic engine=innodb page_encryption=DEFAULT page_encryption_key=DEFAULT; +alter table innodb_dynamic engine=innodb encryption=DEFAULT encryption_key_id=DEFAULT; show create table innodb_dynamic; Table Create Table innodb_dynamic CREATE TABLE `innodb_dynamic` ( `c1` bigint(20) NOT NULL, `b` char(200) DEFAULT NULL ) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=DYNAMIC -alter table innodb_redundant engine=innodb page_encryption=DEFAULT page_encryption_key=DEFAULT; +alter table innodb_redundant engine=innodb encryption=DEFAULT encryption_key_id=DEFAULT; show create table innodb_redundant; Table Create Table innodb_redundant CREATE TABLE `innodb_redundant` ( @@ -217,7 +211,7 @@ Table Create Table innodb_defkey CREATE TABLE `innodb_defkey` ( `c1` bigint(20) NOT NULL, `b` char(200) DEFAULT NULL -) ENGINE=InnoDB DEFAULT CHARSET=latin1 `page_encryption`=1 +) ENGINE=InnoDB DEFAULT CHARSET=latin1 `encryption`='ON' update innodb_normal set c1 = c1 +1; update innodb_compact set c1 = c1 + 1; update innodb_compressed set c1 = c1 + 1; @@ -251,13 +245,10 @@ select count(*) from innodb_redundant t1, innodb_normal t2 where t1.c1 = t2.c1 and t1.b = t2.b; count(*) 2000 -SELECT variable_value = 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_encrypted'; +SELECT variable_value = 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_encrypted'; variable_value = 0 1 -SELECT variable_value = 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_decrypted'; -variable_value = 0 -1 -SELECT variable_value = 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_encryption_error'; +SELECT variable_value = 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_decrypted'; variable_value = 0 1 drop procedure innodb_insert_proc; diff --git a/mysql-test/suite/innodb/r/innodb-page_encryption_compression.result b/mysql-test/suite/innodb/r/innodb-page_encryption_compression.result index 781a29cb276..20e91e62b2f 100644 --- a/mysql-test/suite/innodb/r/innodb-page_encryption_compression.result +++ b/mysql-test/suite/innodb/r/innodb-page_encryption_compression.result @@ -2,8 +2,8 @@ SET GLOBAL innodb_file_format = `Barracuda`; SET GLOBAL innodb_file_per_table = ON; set global innodb_compression_algorithm = 1; create table innodb_normal(c1 bigint not null, b char(200)) engine=innodb page_compressed=1; -create table innodb_compact(c1 bigint not null, b char(200)) engine=innodb row_format=compact page_encryption=1 page_encryption_key=1 page_compressed=1; -create table innodb_dynamic(c1 bigint not null, b char(200)) engine=innodb row_format=dynamic page_encryption=1 page_encryption_key=2 page_compressed=1; +create table innodb_compact(c1 bigint not null, b char(200)) engine=innodb row_format=compact encryption='ON' encryption_key_id=1 page_compressed=1; +create table innodb_dynamic(c1 bigint not null, b char(200)) engine=innodb row_format=dynamic encryption='ON' encryption_key_id=2 page_compressed=1; show warnings; Level Code Message show create table innodb_normal; @@ -17,13 +17,13 @@ Table Create Table innodb_compact CREATE TABLE `innodb_compact` ( `c1` bigint(20) NOT NULL, `b` char(200) DEFAULT NULL -) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=COMPACT `page_encryption`=1 `page_encryption_key`=1 `page_compressed`=1 +) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=COMPACT `encryption`='ON' `encryption_key_id`=1 `page_compressed`=1 show create table innodb_dynamic; Table Create Table innodb_dynamic CREATE TABLE `innodb_dynamic` ( `c1` bigint(20) NOT NULL, `b` char(200) DEFAULT NULL -) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=DYNAMIC `page_encryption`=1 `page_encryption_key`=2 `page_compressed`=1 +) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=DYNAMIC `encryption`='ON' `encryption_key_id`=2 `page_compressed`=1 create procedure innodb_insert_proc (repeat_count int) begin declare current_num int; @@ -60,21 +60,16 @@ select count(*) from innodb_dynamic t1, innodb_normal t2 where t1.c1 = t2.c1 and t1.b = t2.b; count(*) 5000 -SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_encrypted'; +SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_encrypted'; variable_value >= 0 1 -SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_decrypted'; +SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_decrypted'; variable_value >= 0 1 -SELECT variable_value = 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_encryption_error'; -variable_value = 0 -1 -SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_compressed'; +SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_compressed'; variable_value >= 0 -1 -SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_decompressed'; +SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_decompressed'; variable_value >= 0 -1 SET GLOBAL innodb_file_format = `Barracuda`; SET GLOBAL innodb_file_per_table = ON; set global innodb_compression_algorithm = 1; @@ -98,21 +93,16 @@ select count(*) from innodb_dynamic t1, innodb_normal t2 where t1.c1 = t2.c1 and t1.b = t2.b; count(*) 5000 -SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_encrypted'; +SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_encrypted'; variable_value >= 0 1 -SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_decrypted'; +SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_decrypted'; variable_value >= 0 1 -SELECT variable_value = 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_encryption_error'; -variable_value = 0 -1 -SELECT variable_value > 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_compressed'; -variable_value > 0 -1 -SELECT variable_value > 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_decompressed'; -variable_value > 0 -1 +SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_compressed'; +variable_value >= 0 +SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_decompressed'; +variable_value >= 0 alter table innodb_normal engine=innodb page_compressed=DEFAULT; show create table innodb_normal; Table Create Table @@ -120,14 +110,14 @@ innodb_normal CREATE TABLE `innodb_normal` ( `c1` bigint(20) NOT NULL, `b` char(200) DEFAULT NULL ) ENGINE=InnoDB DEFAULT CHARSET=latin1 -alter table innodb_compact engine=innodb page_encryption=DEFAULT page_encryption_key=DEFAULT page_compressed=DEFAULT; +alter table innodb_compact engine=innodb encryption=DEFAULT encryption_key_id=DEFAULT page_compressed=DEFAULT; show create table innodb_compact; Table Create Table innodb_compact CREATE TABLE `innodb_compact` ( `c1` bigint(20) NOT NULL, `b` char(200) DEFAULT NULL ) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=COMPACT -alter table innodb_dynamic engine=innodb page_encryption=DEFAULT page_encryption_key=DEFAULT page_compressed=DEFAULT; +alter table innodb_dynamic engine=innodb encryption=DEFAULT encryption_key_id=DEFAULT page_compressed=DEFAULT; show create table innodb_dynamic; Table Create Table innodb_dynamic CREATE TABLE `innodb_dynamic` ( @@ -174,21 +164,16 @@ select count(*) from innodb_dynamic t1, innodb_normal t2 where t1.c1 = t2.c1 and t1.b = t2.b; count(*) 5000 -SELECT variable_value = 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_encrypted'; +SELECT variable_value = 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_encrypted'; variable_value = 0 1 -SELECT variable_value = 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_decrypted'; +SELECT variable_value = 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_decrypted'; variable_value = 0 1 -SELECT variable_value = 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_encryption_error'; +SELECT variable_value = 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_compressed'; variable_value = 0 -1 -SELECT variable_value = 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_compressed'; +SELECT variable_value = 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_decompressed'; variable_value = 0 -1 -SELECT variable_value = 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_decompressed'; -variable_value = 0 -1 drop procedure innodb_insert_proc; drop table innodb_normal; drop table innodb_compact; diff --git a/mysql-test/suite/innodb/r/innodb-page_encryption_log_encryption.result b/mysql-test/suite/innodb/r/innodb-page_encryption_log_encryption.result index 3830bfee22b..a74c660da68 100644 --- a/mysql-test/suite/innodb/r/innodb-page_encryption_log_encryption.result +++ b/mysql-test/suite/innodb/r/innodb-page_encryption_log_encryption.result @@ -4,34 +4,34 @@ call mtr.add_suppression("InnoDB: Redo log crypto: Can't initialize to key versi SET GLOBAL innodb_file_format = `Barracuda`; SET GLOBAL innodb_file_per_table = ON; create table innodb_normal(c1 bigint not null, b char(200)) engine=innodb; -create table innodb_compact(c1 bigint not null, b char(200)) engine=innodb row_format=compact page_encryption=1 page_encryption_key=1; -create table innodb_compressed(c1 bigint not null, b char(200)) engine=innodb row_format=compressed page_encryption=1 page_encryption_key=2; -create table innodb_dynamic(c1 bigint not null, b char(200)) engine=innodb row_format=dynamic page_encryption=1 page_encryption_key=3; -create table innodb_redundant(c1 bigint not null, b char(200)) engine=innodb row_format=redundant page_encryption=1 page_encryption_key=4; +create table innodb_compact(c1 bigint not null, b char(200)) engine=innodb row_format=compact encryption='ON' encryption_key_id=1; +create table innodb_compressed(c1 bigint not null, b char(200)) engine=innodb row_format=compressed encryption='ON' encryption_key_id=2; +create table innodb_dynamic(c1 bigint not null, b char(200)) engine=innodb row_format=dynamic encryption='ON' encryption_key_id=3; +create table innodb_redundant(c1 bigint not null, b char(200)) engine=innodb row_format=redundant encryption='ON' encryption_key_id=4; show create table innodb_compact; Table Create Table innodb_compact CREATE TABLE `innodb_compact` ( `c1` bigint(20) NOT NULL, `b` char(200) DEFAULT NULL -) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=COMPACT `page_encryption`=1 `page_encryption_key`=1 +) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=COMPACT `encryption`='ON' `encryption_key_id`=1 show create table innodb_compressed; Table Create Table innodb_compressed CREATE TABLE `innodb_compressed` ( `c1` bigint(20) NOT NULL, `b` char(200) DEFAULT NULL -) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=COMPRESSED `page_encryption`=1 `page_encryption_key`=2 +) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=COMPRESSED `encryption`='ON' `encryption_key_id`=2 show create table innodb_dynamic; Table Create Table innodb_dynamic CREATE TABLE `innodb_dynamic` ( `c1` bigint(20) NOT NULL, `b` char(200) DEFAULT NULL -) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=DYNAMIC `page_encryption`=1 `page_encryption_key`=3 +) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=DYNAMIC `encryption`='ON' `encryption_key_id`=3 show create table innodb_redundant; Table Create Table innodb_redundant CREATE TABLE `innodb_redundant` ( `c1` bigint(20) NOT NULL, `b` char(200) DEFAULT NULL -) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=REDUNDANT `page_encryption`=1 `page_encryption_key`=4 +) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=REDUNDANT `encryption`='ON' `encryption_key_id`=4 create procedure innodb_insert_proc (repeat_count int) begin declare current_num int; @@ -83,15 +83,12 @@ select count(*) from innodb_redundant t1, innodb_normal t2 where t1.c1 = t2.c1 and t1.b = t2.b; count(*) 2000 -SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_encrypted'; +SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_encrypted'; variable_value >= 0 1 -SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_decrypted'; +SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_decrypted'; variable_value >= 0 1 -SELECT variable_value = 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_encryption_error'; -variable_value = 0 -1 SET GLOBAL innodb_file_format = `Barracuda`; SET GLOBAL innodb_file_per_table = ON; update innodb_normal set c1 = c1 +1; @@ -127,37 +124,34 @@ select count(*) from innodb_redundant t1, innodb_normal t2 where t1.c1 = t2.c1 and t1.b = t2.b; count(*) 2000 -SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_encrypted'; +SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_encrypted'; variable_value >= 0 1 -SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_decrypted'; +SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_decrypted'; variable_value >= 0 1 -SELECT variable_value = 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_encryption_error'; -variable_value = 0 -1 -alter table innodb_compact engine=innodb page_encryption=DEFAULT page_encryption_key=DEFAULT; +alter table innodb_compact engine=innodb encryption=DEFAULT encryption_key_id=DEFAULT; show create table innodb_compact; Table Create Table innodb_compact CREATE TABLE `innodb_compact` ( `c1` bigint(20) NOT NULL, `b` char(200) DEFAULT NULL ) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=COMPACT -alter table innodb_compressed engine=innodb page_encryption=DEFAULT page_encryption_key=DEFAULT; +alter table innodb_compressed engine=innodb encryption=DEFAULT encryption_key_id=DEFAULT; show create table innodb_compressed; Table Create Table innodb_compressed CREATE TABLE `innodb_compressed` ( `c1` bigint(20) NOT NULL, `b` char(200) DEFAULT NULL ) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=COMPRESSED -alter table innodb_dynamic engine=innodb page_encryption=DEFAULT page_encryption_key=DEFAULT; +alter table innodb_dynamic engine=innodb encryption=DEFAULT encryption_key_id=DEFAULT; show create table innodb_dynamic; Table Create Table innodb_dynamic CREATE TABLE `innodb_dynamic` ( `c1` bigint(20) NOT NULL, `b` char(200) DEFAULT NULL ) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=DYNAMIC -alter table innodb_redundant engine=innodb page_encryption=DEFAULT page_encryption_key=DEFAULT; +alter table innodb_redundant engine=innodb encryption=DEFAULT encryption_key_id=DEFAULT; show create table innodb_redundant; Table Create Table innodb_redundant CREATE TABLE `innodb_redundant` ( @@ -223,13 +217,10 @@ select count(*) from innodb_redundant t1, innodb_normal t2 where t1.c1 = t2.c1 and t1.b = t2.b; count(*) 2000 -SELECT variable_value = 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_encrypted'; +SELECT variable_value = 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_encrypted'; variable_value = 0 1 -SELECT variable_value = 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_decrypted'; -variable_value = 0 -1 -SELECT variable_value = 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_encryption_error'; +SELECT variable_value = 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_decrypted'; variable_value = 0 1 drop procedure innodb_insert_proc; diff --git a/mysql-test/suite/innodb/r/innodb_encryption-page-compression.result b/mysql-test/suite/innodb/r/innodb_encryption-page-compression.result index d4a049d36d0..c46cfebd7d2 100644 --- a/mysql-test/suite/innodb/r/innodb_encryption-page-compression.result +++ b/mysql-test/suite/innodb/r/innodb_encryption-page-compression.result @@ -1,9 +1,3 @@ SET GLOBAL innodb_file_format = `Barracuda`; SET GLOBAL innodb_file_per_table = ON; set global innodb_compression_algorithm = 1; -create table innodb_normal(c1 bigint not null, b char(200)) engine=innodb page_compressed=1; -ERROR HY000: Can't create table `test`.`innodb_normal` (errno: 140 "Wrong create options") -create table innodb_compact(c1 bigint not null, b char(200)) engine=innodb row_format=compact page_compressed=1; -ERROR HY000: Can't create table `test`.`innodb_compact` (errno: 140 "Wrong create options") -create table innodb_dynamic(c1 bigint not null, b char(200)) engine=innodb row_format=dynamic page_compressed=1; -ERROR HY000: Can't create table `test`.`innodb_dynamic` (errno: 140 "Wrong create options") diff --git a/mysql-test/suite/innodb/r/innodb_encryption_change.result b/mysql-test/suite/innodb/r/innodb_encryption_change.result index 90100852d33..513b2a5902e 100644 --- a/mysql-test/suite/innodb/r/innodb_encryption_change.result +++ b/mysql-test/suite/innodb/r/innodb_encryption_change.result @@ -66,15 +66,12 @@ SET GLOBAL innodb_encryption_threads=4; # Success! # Wait max 10 min for key encryption threads to decrypt all space # Success! -SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_encrypted'; +SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_encrypted'; variable_value >= 0 1 -SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_decrypted'; +SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_decrypted'; variable_value >= 0 1 -SELECT variable_value = 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_encryption_error'; -variable_value = 0 -1 SELECT variable_value > 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_compressed'; variable_value > 0 0 diff --git a/mysql-test/suite/innodb/r/innodb_encryption_tables.result b/mysql-test/suite/innodb/r/innodb_encryption_tables.result index f07aa62837c..ee5be0bfdd3 100644 --- a/mysql-test/suite/innodb/r/innodb_encryption_tables.result +++ b/mysql-test/suite/innodb/r/innodb_encryption_tables.result @@ -91,15 +91,12 @@ select count(*) from innodb_redundant t1, innodb_normal t2 where t1.c1 = t2.c1 and t1.b = t2.b; count(*) 2000 -SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_encrypted'; +SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_encrypted'; variable_value >= 0 1 -SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_decrypted'; +SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_decrypted'; variable_value >= 0 1 -SELECT variable_value = 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_encryption_error'; -variable_value = 0 -1 SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_compressed'; variable_value >= 0 1 @@ -144,21 +141,18 @@ select count(*) from innodb_redundant t1, innodb_normal t2 where t1.c1 = t2.c1 and t1.b = t2.b; count(*) 2000 -SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_encrypted'; +SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_encrypted'; variable_value >= 0 1 -SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_decrypted'; +SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_decrypted'; variable_value >= 0 1 -SELECT variable_value = 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_encryption_error'; -variable_value = 0 +SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_compressed'; +variable_value >= 0 +1 +SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_decompressed'; +variable_value >= 0 1 -SELECT variable_value > 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_compressed'; -variable_value > 0 -0 -SELECT variable_value > 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_decompressed'; -variable_value > 0 -0 drop procedure innodb_insert_proc; drop table innodb_normal; drop table innodb_compact; diff --git a/mysql-test/suite/innodb/r/innodb_monitor.result b/mysql-test/suite/innodb/r/innodb_monitor.result index 02e72ae6a80..0977c8b8594 100644 --- a/mysql-test/suite/innodb/r/innodb_monitor.result +++ b/mysql-test/suite/innodb/r/innodb_monitor.result @@ -180,9 +180,8 @@ compress_page_compressed_trim_op disabled compress_page_compressed_trim_op_saved disabled compress_pages_page_decompressed disabled compress_pages_page_compression_error disabled -compress_pages_page_encrypted disabled -compress_pages_page_decrypted disabled -compress_pages_page_encryption_error disabled +compress_pages_encrypted disabled +compress_pages_decrypted disabled index_page_splits disabled index_page_merge_attempts disabled index_page_merge_successful disabled diff --git a/mysql-test/suite/innodb/t/innodb-page_encryption-32k.test b/mysql-test/suite/innodb/t/innodb-page_encryption-32k.test index e96e352da12..765ba7cb354 100644 --- a/mysql-test/suite/innodb/t/innodb-page_encryption-32k.test +++ b/mysql-test/suite/innodb/t/innodb-page_encryption-32k.test @@ -1,7 +1,9 @@ --source include/no_valgrind_without_big.inc +--source include/not_embedded.inc # Tests for setting innodb-page-size=32k; ---source include/have_xtradb.inc +--source include/have_innodb.inc --source include/have_innodb_32k.inc +--source include/have_file_key_management_plugin.inc call mtr.add_suppression("InnoDB: Warning: innodb_page_size has been changed from default value *"); @@ -15,9 +17,9 @@ SET GLOBAL innodb_file_format = `Barracuda`; SET GLOBAL innodb_file_per_table = ON; create table innodb_normal(c1 bigint not null, b char(200)) engine=innodb; -create table innodb_compact(c1 bigint not null, b char(200)) engine=innodb row_format=compact page_encryption=1 page_encryption_key=1; -create table innodb_dynamic(c1 bigint not null, b char(200)) engine=innodb row_format=dynamic page_encryption=1 page_encryption_key=3; -create table innodb_redundant(c1 bigint not null, b char(200)) engine=innodb row_format=redundant page_encryption=1 page_encryption_key=4; +create table innodb_compact(c1 bigint not null, b char(200)) engine=innodb row_format=compact encryption='ON' encryption_key_id=1; +create table innodb_dynamic(c1 bigint not null, b char(200)) engine=innodb row_format=dynamic encryption='ON' encryption_key_id=3; +create table innodb_redundant(c1 bigint not null, b char(200)) engine=innodb row_format=redundant encryption='ON' encryption_key_id=4; show create table innodb_compact; show create table innodb_dynamic; @@ -61,11 +63,11 @@ select count(*) from innodb_compact where c1 < 1500000; select count(*) from innodb_dynamic where c1 < 1500000; select count(*) from innodb_redundant where c1 < 1500000; -alter table innodb_compact engine=innodb page_encryption=0; +alter table innodb_compact engine=innodb encryption=DEFAULT encryption_key_id=DEFAULT; show create table innodb_compact; -alter table innodb_dynamic engine=innodb page_encryption=0; +alter table innodb_dynamic engine=innodb encryption=DEFAULT encryption_key_id=DEFAULT; show create table innodb_dynamic; -alter table innodb_redundant engine=innodb page_encryption=0; +alter table innodb_redundant engine=innodb encryption=DEFAULT encryption_key_id=DEFAULT; show create table innodb_redundant; --source include/restart_mysqld.inc diff --git a/mysql-test/suite/innodb/t/innodb-page_encryption.test b/mysql-test/suite/innodb/t/innodb-page_encryption.test index 66927d1905f..25f477996db 100644 --- a/mysql-test/suite/innodb/t/innodb-page_encryption.test +++ b/mysql-test/suite/innodb/t/innodb-page_encryption.test @@ -4,20 +4,20 @@ --disable_query_log let $innodb_file_format_orig = `SELECT @@innodb_file_format`; let $innodb_file_per_table_orig = `SELECT @@innodb_file_per_table`; -let $default_page_encryption_key = `SELECT @@innodb_default_page_encryption_key`; +let $default_encryption_key = `SELECT @@innodb_default_encryption_key`; --enable_query_log SET GLOBAL innodb_file_format = `Barracuda`; SET GLOBAL innodb_file_per_table = ON; create table innodb_normal(c1 bigint not null, b char(200)) engine=innodb; -create table innodb_compact(c1 bigint not null, b char(200)) engine=innodb row_format=compact page_encryption=1 page_encryption_key=1; -create table innodb_compressed(c1 bigint not null, b char(200)) engine=innodb row_format=compressed page_encryption=1 page_encryption_key=2; -create table innodb_dynamic(c1 bigint not null, b char(200)) engine=innodb row_format=dynamic page_encryption=1 page_encryption_key=3; -create table innodb_redundant(c1 bigint not null, b char(200)) engine=innodb row_format=redundant page_encryption=1 page_encryption_key=4; +create table innodb_compact(c1 bigint not null, b char(200)) engine=innodb row_format=compact encryption='ON' encryption_key_id=1; +create table innodb_compressed(c1 bigint not null, b char(200)) engine=innodb row_format=compressed encryption='ON' encryption_key_id=2; +create table innodb_dynamic(c1 bigint not null, b char(200)) engine=innodb row_format=dynamic encryption='ON' encryption_key_id=3; +create table innodb_redundant(c1 bigint not null, b char(200)) engine=innodb row_format=redundant encryption='ON' encryption_key_id=4; -SET GLOBAL innodb_default_page_encryption_key = 5; -create table innodb_defkey(c1 bigint not null, b char(200)) engine=innodb page_encryption=1; +SET GLOBAL innodb_default_encryption_key = 5; +create table innodb_defkey(c1 bigint not null, b char(200)) engine=innodb encryption='ON'; show create table innodb_defkey; show create table innodb_compact; @@ -73,9 +73,8 @@ select count(*) from innodb_defkey t1, innodb_normal t2 where t1.c1 = t2.c1 and t1.b = t2.b; # Note there that these variables are updated only when real I/O is done, thus they are not reliable -SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_encrypted'; -SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_decrypted'; -SELECT variable_value = 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_encryption_error'; +SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_encrypted'; +SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_decrypted'; --source include/restart_mysqld.inc @@ -104,17 +103,16 @@ t1.c1 = t2.c1 and t1.b = t2.b; select count(*) from innodb_defkey t1, innodb_normal t2 where t1.c1 = t2.c1 and t1.b = t2.b; -SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_encrypted'; -SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_decrypted'; -SELECT variable_value = 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_encryption_error'; +SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_encrypted'; +SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_decrypted'; -alter table innodb_compact engine=innodb page_encryption=DEFAULT page_encryption_key=DEFAULT; +alter table innodb_compact engine=innodb encryption=DEFAULT encryption_key_id=DEFAULT; show create table innodb_compact; -alter table innodb_compressed engine=innodb page_encryption=DEFAULT page_encryption_key=DEFAULT; +alter table innodb_compressed engine=innodb encryption=DEFAULT encryption_key_id=DEFAULT; show create table innodb_compressed; -alter table innodb_dynamic engine=innodb page_encryption=DEFAULT page_encryption_key=DEFAULT; +alter table innodb_dynamic engine=innodb encryption=DEFAULT encryption_key_id=DEFAULT; show create table innodb_dynamic; -alter table innodb_redundant engine=innodb page_encryption=DEFAULT page_encryption_key=DEFAULT; +alter table innodb_redundant engine=innodb encryption=DEFAULT encryption_key_id=DEFAULT; show create table innodb_redundant; --source include/restart_mysqld.inc @@ -147,9 +145,8 @@ select count(*) from innodb_redundant t1, innodb_normal t2 where t1.c1 = t2.c1 and t1.b = t2.b; # After alter+restart these should be 0 -SELECT variable_value = 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_encrypted'; -SELECT variable_value = 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_decrypted'; -SELECT variable_value = 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_encryption_error'; +SELECT variable_value = 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_encrypted'; +SELECT variable_value = 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_decrypted'; drop procedure innodb_insert_proc; drop table innodb_normal; @@ -163,5 +160,5 @@ drop table innodb_defkey; --disable_query_log EVAL SET GLOBAL innodb_file_per_table = $innodb_file_per_table_orig; EVAL SET GLOBAL innodb_file_format = $innodb_file_format_orig; -EVAL SET GLOBAL innodb_default_page_encryption_key = $default_page_encryption_key; +EVAL SET GLOBAL innodb_default_encryption_key = $default_encryption_key; --enable_query_log diff --git a/mysql-test/suite/innodb/t/innodb-page_encryption_compression.test b/mysql-test/suite/innodb/t/innodb-page_encryption_compression.test index 6b9fb5bca9d..6123745d5c6 100644 --- a/mysql-test/suite/innodb/t/innodb-page_encryption_compression.test +++ b/mysql-test/suite/innodb/t/innodb-page_encryption_compression.test @@ -1,4 +1,5 @@ -- source include/have_innodb.inc +-- source include/not_embedded.inc -- source include/have_file_key_management_plugin.inc --disable_query_log @@ -13,8 +14,8 @@ SET GLOBAL innodb_file_per_table = ON; set global innodb_compression_algorithm = 1; create table innodb_normal(c1 bigint not null, b char(200)) engine=innodb page_compressed=1; -create table innodb_compact(c1 bigint not null, b char(200)) engine=innodb row_format=compact page_encryption=1 page_encryption_key=1 page_compressed=1; -create table innodb_dynamic(c1 bigint not null, b char(200)) engine=innodb row_format=dynamic page_encryption=1 page_encryption_key=2 page_compressed=1; +create table innodb_compact(c1 bigint not null, b char(200)) engine=innodb row_format=compact encryption='ON' encryption_key_id=1 page_compressed=1; +create table innodb_dynamic(c1 bigint not null, b char(200)) engine=innodb row_format=dynamic encryption='ON' encryption_key_id=2 page_compressed=1; show warnings; show create table innodb_normal; @@ -53,11 +54,10 @@ t1.c1 = t2.c1 and t1.b = t2.b; select count(*) from innodb_dynamic t1, innodb_normal t2 where t1.c1 = t2.c1 and t1.b = t2.b; -SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_encrypted'; -SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_decrypted'; -SELECT variable_value = 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_encryption_error'; -SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_compressed'; -SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_decompressed'; +SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_encrypted'; +SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_decrypted'; +SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_compressed'; +SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_decompressed'; --source include/restart_mysqld.inc @@ -77,17 +77,16 @@ t1.c1 = t2.c1 and t1.b = t2.b; select count(*) from innodb_dynamic t1, innodb_normal t2 where t1.c1 = t2.c1 and t1.b = t2.b; -SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_encrypted'; -SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_decrypted'; -SELECT variable_value = 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_encryption_error'; -SELECT variable_value > 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_compressed'; -SELECT variable_value > 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_decompressed'; +SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_encrypted'; +SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_decrypted'; +SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_compressed'; +SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_decompressed'; alter table innodb_normal engine=innodb page_compressed=DEFAULT; show create table innodb_normal; -alter table innodb_compact engine=innodb page_encryption=DEFAULT page_encryption_key=DEFAULT page_compressed=DEFAULT; +alter table innodb_compact engine=innodb encryption=DEFAULT encryption_key_id=DEFAULT page_compressed=DEFAULT; show create table innodb_compact; -alter table innodb_dynamic engine=innodb page_encryption=DEFAULT page_encryption_key=DEFAULT page_compressed=DEFAULT; +alter table innodb_dynamic engine=innodb encryption=DEFAULT encryption_key_id=DEFAULT page_compressed=DEFAULT; show create table innodb_dynamic; --source include/restart_mysqld.inc @@ -110,11 +109,10 @@ t1.c1 = t2.c1 and t1.b = t2.b; select count(*) from innodb_dynamic t1, innodb_normal t2 where t1.c1 = t2.c1 and t1.b = t2.b; -SELECT variable_value = 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_encrypted'; -SELECT variable_value = 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_decrypted'; -SELECT variable_value = 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_encryption_error'; -SELECT variable_value = 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_compressed'; -SELECT variable_value = 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_decompressed'; +SELECT variable_value = 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_encrypted'; +SELECT variable_value = 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_decrypted'; +SELECT variable_value = 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_compressed'; +SELECT variable_value = 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_decompressed'; drop procedure innodb_insert_proc; drop table innodb_normal; diff --git a/mysql-test/suite/innodb/t/innodb-page_encryption_log_encryption.test b/mysql-test/suite/innodb/t/innodb-page_encryption_log_encryption.test index abff5d93083..49deeff8008 100644 --- a/mysql-test/suite/innodb/t/innodb-page_encryption_log_encryption.test +++ b/mysql-test/suite/innodb/t/innodb-page_encryption_log_encryption.test @@ -1,4 +1,5 @@ -- source include/have_innodb.inc +-- source include/not_embedded.inc -- source include/have_file_key_management_plugin.inc --disable_query_log @@ -14,10 +15,10 @@ SET GLOBAL innodb_file_format = `Barracuda`; SET GLOBAL innodb_file_per_table = ON; create table innodb_normal(c1 bigint not null, b char(200)) engine=innodb; -create table innodb_compact(c1 bigint not null, b char(200)) engine=innodb row_format=compact page_encryption=1 page_encryption_key=1; -create table innodb_compressed(c1 bigint not null, b char(200)) engine=innodb row_format=compressed page_encryption=1 page_encryption_key=2; -create table innodb_dynamic(c1 bigint not null, b char(200)) engine=innodb row_format=dynamic page_encryption=1 page_encryption_key=3; -create table innodb_redundant(c1 bigint not null, b char(200)) engine=innodb row_format=redundant page_encryption=1 page_encryption_key=4; +create table innodb_compact(c1 bigint not null, b char(200)) engine=innodb row_format=compact encryption='ON' encryption_key_id=1; +create table innodb_compressed(c1 bigint not null, b char(200)) engine=innodb row_format=compressed encryption='ON' encryption_key_id=2; +create table innodb_dynamic(c1 bigint not null, b char(200)) engine=innodb row_format=dynamic encryption='ON' encryption_key_id=3; +create table innodb_redundant(c1 bigint not null, b char(200)) engine=innodb row_format=redundant encryption='ON' encryption_key_id=4; show create table innodb_compact; show create table innodb_compressed; @@ -66,9 +67,8 @@ select count(*) from innodb_redundant t1, innodb_normal t2 where t1.c1 = t2.c1 and t1.b = t2.b; # Note there that these variables are updated only when real I/O is done, thus they are not reliable -SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_encrypted'; -SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_decrypted'; -SELECT variable_value = 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_encryption_error'; +SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_encrypted'; +SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_decrypted'; --source include/restart_mysqld.inc @@ -93,17 +93,16 @@ t1.c1 = t2.c1 and t1.b = t2.b; select count(*) from innodb_redundant t1, innodb_normal t2 where t1.c1 = t2.c1 and t1.b = t2.b; -SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_encrypted'; -SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_decrypted'; -SELECT variable_value = 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_encryption_error'; +SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_encrypted'; +SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_decrypted'; -alter table innodb_compact engine=innodb page_encryption=DEFAULT page_encryption_key=DEFAULT; +alter table innodb_compact engine=innodb encryption=DEFAULT encryption_key_id=DEFAULT; show create table innodb_compact; -alter table innodb_compressed engine=innodb page_encryption=DEFAULT page_encryption_key=DEFAULT; +alter table innodb_compressed engine=innodb encryption=DEFAULT encryption_key_id=DEFAULT; show create table innodb_compressed; -alter table innodb_dynamic engine=innodb page_encryption=DEFAULT page_encryption_key=DEFAULT; +alter table innodb_dynamic engine=innodb encryption=DEFAULT encryption_key_id=DEFAULT; show create table innodb_dynamic; -alter table innodb_redundant engine=innodb page_encryption=DEFAULT page_encryption_key=DEFAULT; +alter table innodb_redundant engine=innodb encryption=DEFAULT encryption_key_id=DEFAULT; show create table innodb_redundant; --source include/restart_mysqld.inc @@ -135,9 +134,8 @@ select count(*) from innodb_redundant t1, innodb_normal t2 where t1.c1 = t2.c1 and t1.b = t2.b; # After alter+restart these should be 0 -SELECT variable_value = 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_encrypted'; -SELECT variable_value = 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_decrypted'; -SELECT variable_value = 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_encryption_error'; +SELECT variable_value = 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_encrypted'; +SELECT variable_value = 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_decrypted'; drop procedure innodb_insert_proc; drop table innodb_normal; diff --git a/mysql-test/suite/innodb/t/innodb_encryption-page-compression.test b/mysql-test/suite/innodb/t/innodb_encryption-page-compression.test index 48a9a780e0d..34962167ef1 100644 --- a/mysql-test/suite/innodb/t/innodb_encryption-page-compression.test +++ b/mysql-test/suite/innodb/t/innodb_encryption-page-compression.test @@ -15,16 +15,6 @@ SET GLOBAL innodb_file_per_table = ON; # zlib set global innodb_compression_algorithm = 1; ---replace_regex /#sql-[0-9a-f_]*`/#sql-temporary`/ ---error 1005 -create table innodb_normal(c1 bigint not null, b char(200)) engine=innodb page_compressed=1; ---replace_regex /#sql-[0-9a-f_]*`/#sql-temporary`/ ---error 1005 -create table innodb_compact(c1 bigint not null, b char(200)) engine=innodb row_format=compact page_compressed=1; ---replace_regex /#sql-[0-9a-f_]*`/#sql-temporary`/ ---error 1005 -create table innodb_dynamic(c1 bigint not null, b char(200)) engine=innodb row_format=dynamic page_compressed=1; - # reset system --disable_query_log EVAL SET GLOBAL innodb_compression_algorithm = $innodb_compression_algorithm_orig; diff --git a/mysql-test/suite/innodb/t/innodb_encryption_change.test b/mysql-test/suite/innodb/t/innodb_encryption_change.test index 6c5eab50b84..8e9e1603196 100644 --- a/mysql-test/suite/innodb/t/innodb_encryption_change.test +++ b/mysql-test/suite/innodb/t/innodb_encryption_change.test @@ -205,9 +205,8 @@ if (!$success) } --echo # Success! -SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_encrypted'; -SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_decrypted'; -SELECT variable_value = 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_encryption_error'; +SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_encrypted'; +SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_decrypted'; SELECT variable_value > 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_compressed'; SELECT variable_value > 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_decompressed'; diff --git a/mysql-test/suite/innodb/t/innodb_encryption_tables.test b/mysql-test/suite/innodb/t/innodb_encryption_tables.test index d192a16ffd4..6d378270ad9 100644 --- a/mysql-test/suite/innodb/t/innodb_encryption_tables.test +++ b/mysql-test/suite/innodb/t/innodb_encryption_tables.test @@ -1,6 +1,6 @@ -- source include/have_innodb.inc -- source include/have_example_key_management_plugin.inc ---source include/not_embedded.inc +-- source include/not_embedded.inc --disable_query_log let $innodb_file_format_orig = `SELECT @@innodb_file_format`; @@ -66,9 +66,8 @@ t1.c1 = t2.c1 and t1.b = t2.b; select count(*) from innodb_redundant t1, innodb_normal t2 where t1.c1 = t2.c1 and t1.b = t2.b; -SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_encrypted'; -SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_decrypted'; -SELECT variable_value = 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_encryption_error'; +SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_encrypted'; +SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_decrypted'; SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_compressed'; SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_decompressed'; @@ -96,11 +95,10 @@ t1.c1 = t2.c1 and t1.b = t2.b; select count(*) from innodb_redundant t1, innodb_normal t2 where t1.c1 = t2.c1 and t1.b = t2.b; -SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_encrypted'; -SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_decrypted'; -SELECT variable_value = 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_encryption_error'; -SELECT variable_value > 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_compressed'; -SELECT variable_value > 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_decompressed'; +SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_encrypted'; +SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_decrypted'; +SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_compressed'; +SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_page_decompressed'; drop procedure innodb_insert_proc; drop table innodb_normal; diff --git a/mysql-test/suite/sys_vars/r/all_vars.result b/mysql-test/suite/sys_vars/r/all_vars.result index 840da8405c3..8a8d8741236 100644 --- a/mysql-test/suite/sys_vars/r/all_vars.result +++ b/mysql-test/suite/sys_vars/r/all_vars.result @@ -10,6 +10,7 @@ there should be *no* long test name listed below: select distinct variable_name as `there should be *no* variables listed below:` from t2 left join t1 on variable_name=test_name where test_name is null; there should be *no* variables listed below: +innodb_default_encryption_key strict_password_validation drop table t1; drop table t2; diff --git a/mysql-test/suite/sys_vars/r/innodb_default_page_encryption_key_basic.result b/mysql-test/suite/sys_vars/r/innodb_default_page_encryption_key_basic.result index 1e779d7c4f6..b18a9301483 100644 --- a/mysql-test/suite/sys_vars/r/innodb_default_page_encryption_key_basic.result +++ b/mysql-test/suite/sys_vars/r/innodb_default_page_encryption_key_basic.result @@ -1,71 +1,71 @@ -SET @start_global_value = @@global.innodb_default_page_encryption_key; +SET @start_global_value = @@global.innodb_default_encryption_key; SELECT @start_global_value; @start_global_value 1 Valid value 0-9 -select @@global.innodb_default_page_encryption_key <= 9; -@@global.innodb_default_page_encryption_key <= 9 +select @@global.innodb_default_encryption_key <= 9; +@@global.innodb_default_encryption_key <= 9 1 -select @@global.innodb_default_page_encryption_key; -@@global.innodb_default_page_encryption_key +select @@global.innodb_default_encryption_key; +@@global.innodb_default_encryption_key 1 -select @@session.innodb_default_page_encryption_key; -ERROR HY000: Variable 'innodb_default_page_encryption_key' is a GLOBAL variable -show global variables like 'innodb_default_page_encryption_key'; +select @@session.innodb_default_encryption_key; +ERROR HY000: Variable 'innodb_default_encryption_key' is a GLOBAL variable +show global variables like 'innodb_default_encryption_key'; Variable_name Value -innodb_default_page_encryption_key 1 -show session variables like 'innodb_default_page_encryption_key'; +innodb_default_encryption_key 1 +show session variables like 'innodb_default_encryption_key'; Variable_name Value -innodb_default_page_encryption_key 1 -select * from information_schema.global_variables where variable_name='innodb_default_page_encryption_key'; +innodb_default_encryption_key 1 +select * from information_schema.global_variables where variable_name='innodb_default_encryption_key'; VARIABLE_NAME VARIABLE_VALUE -INNODB_DEFAULT_PAGE_ENCRYPTION_KEY 1 -select * from information_schema.session_variables where variable_name='innodb_default_page_encryption_key'; +INNODB_DEFAULT_ENCRYPTION_KEY 1 +select * from information_schema.session_variables where variable_name='innodb_default_encryption_key'; VARIABLE_NAME VARIABLE_VALUE -INNODB_DEFAULT_PAGE_ENCRYPTION_KEY 1 -set global innodb_default_page_encryption_key=2; -select @@global.innodb_default_page_encryption_key; -@@global.innodb_default_page_encryption_key +INNODB_DEFAULT_ENCRYPTION_KEY 1 +set global innodb_default_encryption_key=2; +select @@global.innodb_default_encryption_key; +@@global.innodb_default_encryption_key 2 -select * from information_schema.global_variables where variable_name='innodb_default_page_encryption_key'; +select * from information_schema.global_variables where variable_name='innodb_default_encryption_key'; VARIABLE_NAME VARIABLE_VALUE -INNODB_DEFAULT_PAGE_ENCRYPTION_KEY 2 -select * from information_schema.session_variables where variable_name='innodb_default_page_encryption_key'; +INNODB_DEFAULT_ENCRYPTION_KEY 2 +select * from information_schema.session_variables where variable_name='innodb_default_encryption_key'; VARIABLE_NAME VARIABLE_VALUE -INNODB_DEFAULT_PAGE_ENCRYPTION_KEY 2 -set session innodb_default_page_encryption_key=4; -ERROR HY000: Variable 'innodb_default_page_encryption_key' is a GLOBAL variable and should be set with SET GLOBAL -set global innodb_default_page_encryption_key=1.1; -ERROR 42000: Incorrect argument type to variable 'innodb_default_page_encryption_key' -set global innodb_default_page_encryption_key=1e1; -ERROR 42000: Incorrect argument type to variable 'innodb_default_page_encryption_key' -set global innodb_default_page_encryption_key="foo"; -ERROR 42000: Incorrect argument type to variable 'innodb_default_page_encryption_key' -set global innodb_default_page_encryption_key=10; -select @@global.innodb_default_page_encryption_key; -@@global.innodb_default_page_encryption_key +INNODB_DEFAULT_ENCRYPTION_KEY 2 +set session innodb_default_encryption_key=4; +ERROR HY000: Variable 'innodb_default_encryption_key' is a GLOBAL variable and should be set with SET GLOBAL +set global innodb_default_encryption_key=1.1; +ERROR 42000: Incorrect argument type to variable 'innodb_default_encryption_key' +set global innodb_default_encryption_key=1e1; +ERROR 42000: Incorrect argument type to variable 'innodb_default_encryption_key' +set global innodb_default_encryption_key="foo"; +ERROR 42000: Incorrect argument type to variable 'innodb_default_encryption_key' +set global innodb_default_encryption_key=10; +select @@global.innodb_default_encryption_key; +@@global.innodb_default_encryption_key 10 -select * from information_schema.global_variables where variable_name='innodb_default_page_encryption_key'; +select * from information_schema.global_variables where variable_name='innodb_default_encryption_key'; VARIABLE_NAME VARIABLE_VALUE -INNODB_DEFAULT_PAGE_ENCRYPTION_KEY 10 -set global innodb_default_page_encryption_key=-7; +INNODB_DEFAULT_ENCRYPTION_KEY 10 +set global innodb_default_encryption_key=-7; Warnings: -Warning 1292 Truncated incorrect innodb_default_page_encryption_k value: '-7' -select @@global.innodb_default_page_encryption_key; -@@global.innodb_default_page_encryption_key +Warning 1292 Truncated incorrect innodb_default_encryption_key value: '-7' +select @@global.innodb_default_encryption_key; +@@global.innodb_default_encryption_key 1 -select * from information_schema.global_variables where variable_name='innodb_default_page_encryption_key'; +select * from information_schema.global_variables where variable_name='innodb_default_encryption_key'; VARIABLE_NAME VARIABLE_VALUE -INNODB_DEFAULT_PAGE_ENCRYPTION_KEY 1 -set global innodb_default_page_encryption_key=1; -select @@global.innodb_default_page_encryption_key; -@@global.innodb_default_page_encryption_key +INNODB_DEFAULT_ENCRYPTION_KEY 1 +set global innodb_default_encryption_key=1; +select @@global.innodb_default_encryption_key; +@@global.innodb_default_encryption_key 1 -set global innodb_default_page_encryption_key=255; -select @@global.innodb_default_page_encryption_key; -@@global.innodb_default_page_encryption_key +set global innodb_default_encryption_key=255; +select @@global.innodb_default_encryption_key; +@@global.innodb_default_encryption_key 255 -SET @@global.innodb_default_page_encryption_key = @start_global_value; -SELECT @@global.innodb_default_page_encryption_key; -@@global.innodb_default_page_encryption_key +SET @@global.innodb_default_encryption_key = @start_global_value; +SELECT @@global.innodb_default_encryption_key; +@@global.innodb_default_encryption_key 1 diff --git a/mysql-test/suite/sys_vars/r/innodb_monitor_disable_basic.result b/mysql-test/suite/sys_vars/r/innodb_monitor_disable_basic.result index 85fe22dd816..d992695a7fe 100644 --- a/mysql-test/suite/sys_vars/r/innodb_monitor_disable_basic.result +++ b/mysql-test/suite/sys_vars/r/innodb_monitor_disable_basic.result @@ -180,9 +180,8 @@ compress_page_compressed_trim_op disabled compress_page_compressed_trim_op_saved disabled compress_pages_page_decompressed disabled compress_pages_page_compression_error disabled -compress_pages_page_encrypted disabled -compress_pages_page_decrypted disabled -compress_pages_page_encryption_error disabled +compress_pages_encrypted disabled +compress_pages_decrypted disabled index_page_splits disabled index_page_merge_attempts disabled index_page_merge_successful disabled diff --git a/mysql-test/suite/sys_vars/r/innodb_monitor_enable_basic.result b/mysql-test/suite/sys_vars/r/innodb_monitor_enable_basic.result index 85fe22dd816..d992695a7fe 100644 --- a/mysql-test/suite/sys_vars/r/innodb_monitor_enable_basic.result +++ b/mysql-test/suite/sys_vars/r/innodb_monitor_enable_basic.result @@ -180,9 +180,8 @@ compress_page_compressed_trim_op disabled compress_page_compressed_trim_op_saved disabled compress_pages_page_decompressed disabled compress_pages_page_compression_error disabled -compress_pages_page_encrypted disabled -compress_pages_page_decrypted disabled -compress_pages_page_encryption_error disabled +compress_pages_encrypted disabled +compress_pages_decrypted disabled index_page_splits disabled index_page_merge_attempts disabled index_page_merge_successful disabled diff --git a/mysql-test/suite/sys_vars/r/innodb_monitor_reset_all_basic.result b/mysql-test/suite/sys_vars/r/innodb_monitor_reset_all_basic.result index 85fe22dd816..d992695a7fe 100644 --- a/mysql-test/suite/sys_vars/r/innodb_monitor_reset_all_basic.result +++ b/mysql-test/suite/sys_vars/r/innodb_monitor_reset_all_basic.result @@ -180,9 +180,8 @@ compress_page_compressed_trim_op disabled compress_page_compressed_trim_op_saved disabled compress_pages_page_decompressed disabled compress_pages_page_compression_error disabled -compress_pages_page_encrypted disabled -compress_pages_page_decrypted disabled -compress_pages_page_encryption_error disabled +compress_pages_encrypted disabled +compress_pages_decrypted disabled index_page_splits disabled index_page_merge_attempts disabled index_page_merge_successful disabled diff --git a/mysql-test/suite/sys_vars/r/innodb_monitor_reset_basic.result b/mysql-test/suite/sys_vars/r/innodb_monitor_reset_basic.result index 85fe22dd816..d992695a7fe 100644 --- a/mysql-test/suite/sys_vars/r/innodb_monitor_reset_basic.result +++ b/mysql-test/suite/sys_vars/r/innodb_monitor_reset_basic.result @@ -180,9 +180,8 @@ compress_page_compressed_trim_op disabled compress_page_compressed_trim_op_saved disabled compress_pages_page_decompressed disabled compress_pages_page_compression_error disabled -compress_pages_page_encrypted disabled -compress_pages_page_decrypted disabled -compress_pages_page_encryption_error disabled +compress_pages_encrypted disabled +compress_pages_decrypted disabled index_page_splits disabled index_page_merge_attempts disabled index_page_merge_successful disabled diff --git a/mysql-test/suite/sys_vars/r/sysvars_innodb.result b/mysql-test/suite/sys_vars/r/sysvars_innodb.result index 867fe02d454..fc4f75ba801 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_innodb.result +++ b/mysql-test/suite/sys_vars/r/sysvars_innodb.result @@ -565,16 +565,16 @@ NUMERIC_BLOCK_SIZE NULL ENUM_VALUE_LIST NULL READ_ONLY YES COMMAND_LINE_ARGUMENT REQUIRED -VARIABLE_NAME INNODB_DEFAULT_PAGE_ENCRYPTION_KEY +VARIABLE_NAME INNODB_DEFAULT_ENCRYPTION_KEY SESSION_VALUE NULL GLOBAL_VALUE 1 GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 1 VARIABLE_SCOPE GLOBAL VARIABLE_TYPE INT UNSIGNED -VARIABLE_COMMENT Encryption key used for page encryption. +VARIABLE_COMMENT Default encryption key used for table encryption. NUMERIC_MIN_VALUE 1 -NUMERIC_MAX_VALUE 255 +NUMERIC_MAX_VALUE 4294967295 NUMERIC_BLOCK_SIZE 0 ENUM_VALUE_LIST NULL READ_ONLY NO diff --git a/mysql-test/suite/sys_vars/t/innodb_default_page_encryption_key_basic.test b/mysql-test/suite/sys_vars/t/innodb_default_page_encryption_key_basic.test index c8a02cb8c74..b93ca26ba01 100644 --- a/mysql-test/suite/sys_vars/t/innodb_default_page_encryption_key_basic.test +++ b/mysql-test/suite/sys_vars/t/innodb_default_page_encryption_key_basic.test @@ -1,60 +1,60 @@ --source include/have_innodb.inc -SET @start_global_value = @@global.innodb_default_page_encryption_key; +SET @start_global_value = @@global.innodb_default_encryption_key; SELECT @start_global_value; # # exists as global only # --echo Valid value 0-9 -select @@global.innodb_default_page_encryption_key <= 9; -select @@global.innodb_default_page_encryption_key; +select @@global.innodb_default_encryption_key <= 9; +select @@global.innodb_default_encryption_key; --error ER_INCORRECT_GLOBAL_LOCAL_VAR -select @@session.innodb_default_page_encryption_key; -show global variables like 'innodb_default_page_encryption_key'; -show session variables like 'innodb_default_page_encryption_key'; -select * from information_schema.global_variables where variable_name='innodb_default_page_encryption_key'; -select * from information_schema.session_variables where variable_name='innodb_default_page_encryption_key'; +select @@session.innodb_default_encryption_key; +show global variables like 'innodb_default_encryption_key'; +show session variables like 'innodb_default_encryption_key'; +select * from information_schema.global_variables where variable_name='innodb_default_encryption_key'; +select * from information_schema.session_variables where variable_name='innodb_default_encryption_key'; # # show that it's writable # -set global innodb_default_page_encryption_key=2; -select @@global.innodb_default_page_encryption_key; -select * from information_schema.global_variables where variable_name='innodb_default_page_encryption_key'; -select * from information_schema.session_variables where variable_name='innodb_default_page_encryption_key'; +set global innodb_default_encryption_key=2; +select @@global.innodb_default_encryption_key; +select * from information_schema.global_variables where variable_name='innodb_default_encryption_key'; +select * from information_schema.session_variables where variable_name='innodb_default_encryption_key'; --error ER_GLOBAL_VARIABLE -set session innodb_default_page_encryption_key=4; +set session innodb_default_encryption_key=4; # # incorrect types # --error ER_WRONG_TYPE_FOR_VAR -set global innodb_default_page_encryption_key=1.1; +set global innodb_default_encryption_key=1.1; --error ER_WRONG_TYPE_FOR_VAR -set global innodb_default_page_encryption_key=1e1; +set global innodb_default_encryption_key=1e1; --error ER_WRONG_TYPE_FOR_VAR -set global innodb_default_page_encryption_key="foo"; +set global innodb_default_encryption_key="foo"; -set global innodb_default_page_encryption_key=10; -select @@global.innodb_default_page_encryption_key; -select * from information_schema.global_variables where variable_name='innodb_default_page_encryption_key'; -set global innodb_default_page_encryption_key=-7; -select @@global.innodb_default_page_encryption_key; -select * from information_schema.global_variables where variable_name='innodb_default_page_encryption_key'; +set global innodb_default_encryption_key=10; +select @@global.innodb_default_encryption_key; +select * from information_schema.global_variables where variable_name='innodb_default_encryption_key'; +set global innodb_default_encryption_key=-7; +select @@global.innodb_default_encryption_key; +select * from information_schema.global_variables where variable_name='innodb_default_encryption_key'; # # min/max values # -set global innodb_default_page_encryption_key=1; -select @@global.innodb_default_page_encryption_key; -set global innodb_default_page_encryption_key=255; -select @@global.innodb_default_page_encryption_key; +set global innodb_default_encryption_key=1; +select @@global.innodb_default_encryption_key; +set global innodb_default_encryption_key=255; +select @@global.innodb_default_encryption_key; # # cleanup # -SET @@global.innodb_default_page_encryption_key = @start_global_value; -SELECT @@global.innodb_default_page_encryption_key; +SET @@global.innodb_default_encryption_key = @start_global_value; +SELECT @@global.innodb_default_encryption_key; diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 6f01f884a83..de6cf97230c 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -5710,6 +5710,7 @@ buf_page_encrypt_before_write( ulint space_id) /*!< in: space id */ { fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space_id); + bpage->real_size = UNIV_PAGE_SIZE; if (crypt_data != NULL && crypt_data->encryption == FIL_SPACE_ENCRYPTION_OFF) { /* Encryption is disabled */ @@ -5763,7 +5764,7 @@ buf_page_encrypt_before_write( mach_read_from_4(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); ut_ad(key_version == 0 || key_version >= bpage->key_version); bpage->key_version = key_version; - bpage->real_size = zip_size; + bpage->real_size = page_size; } else { /* First we compress the page content */ ulint out_len = 0; diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 35d55af819b..d26c33c2925 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -647,20 +647,6 @@ fil_node_open_file( success = os_file_read(node->handle, page, 0, UNIV_PAGE_SIZE); - if (fil_page_is_encrypted(page)) { - /* if page is (still) encrypted, write an error and return. - * Otherwise the server would crash if decrypting is not possible. - * This may be the case, if the key file could not be - * opened on server startup. - */ - ib_logf(IB_LOG_LEVEL_ERROR, - "InnoDB: can not decrypt page, because " - "keys could not be read.\n" - ); - return false; - - } - space_id = fsp_header_get_space_id(page); flags = fsp_header_get_flags(page); @@ -1896,7 +1882,6 @@ fil_check_first_page( { ulint space_id; ulint flags; - ulint page_is_encrypted; if (srv_force_recovery >= SRV_FORCE_IGNORE_CORRUPT) { return(NULL); @@ -1904,20 +1889,14 @@ fil_check_first_page( space_id = mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_ID + page); flags = mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + page); - /* Note: the 1st page is usually not encrypted. If the Key Provider - or the encryption key is not available, the - check for reading the first page should intentionally fail - with "can not decrypt" message. */ - page_is_encrypted = fil_page_encryption_status(page, space_id); - if (!page_is_encrypted) { - if (UNIV_PAGE_SIZE != fsp_flags_get_page_size(flags)) { - fprintf(stderr, - "InnoDB: Error: Current page size %lu != " - " page size on page %lu\n", - UNIV_PAGE_SIZE, fsp_flags_get_page_size(flags)); - return("innodb-page-size mismatch"); - } + if (UNIV_PAGE_SIZE != fsp_flags_get_page_size(flags)) { + fprintf(stderr, + "InnoDB: Error: Current page size %lu != " + " page size on page %lu\n", + UNIV_PAGE_SIZE, fsp_flags_get_page_size(flags)); + + return("innodb-page-size mismatch"); } if (!space_id && !flags) { @@ -1933,17 +1912,9 @@ fil_check_first_page( } } - if (!page_is_encrypted && buf_page_is_corrupted( + if (buf_page_is_corrupted( false, page, fsp_flags_get_zip_size(flags))) { return("checksum mismatch"); - } else { - if (page_is_encrypted) { - /* this error message is interpreted by the calling method, which is - * executed if the server starts in recovery mode. - */ - return(FIL_MSG_CANNOT_DECRYPT); - - } } if (page_get_space_id(page) == space_id @@ -1987,6 +1958,7 @@ fil_read_first_page( byte* page; lsn_t flushed_lsn; const char* check_msg = NULL; + fil_space_crypt_t* cdata; buf = static_cast(ut_malloc(2 * UNIV_PAGE_SIZE)); @@ -2005,13 +1977,6 @@ fil_read_first_page( *space_id = fsp_header_get_space_id(page); } - /* Page is page compressed page, need to decompress, before - continue. */ - if (fil_page_is_compressed(page)) { - ulint write_size=0; - fil_decompress_page(NULL, page, UNIV_PAGE_SIZE, &write_size); - } - if (!one_read_already) { check_msg = fil_check_first_page(page); } @@ -2019,12 +1984,29 @@ fil_read_first_page( flushed_lsn = mach_read_from_8(page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); + ulint space = fsp_header_get_space_id(page); + ulint offset = fsp_header_get_crypt_offset( + fsp_flags_get_zip_size(*flags), NULL); + cdata = fil_space_read_crypt_data(space, page, offset); + + /* If file space is encrypted we need to have at least some + encryption service available where to get keys */ + if ((cdata && cdata->encryption == FIL_SPACE_ENCRYPTION_ON) || + ( srv_encrypt_tables && + cdata && cdata->encryption == FIL_SPACE_ENCRYPTION_DEFAULT)) { + int rc = get_latest_encryption_key_version(); + + if (rc < 0) { + ib_logf(IB_LOG_LEVEL_FATAL, + "Tablespace id %ld encrypted but encryption service" + " not available. Can't continue opening tablespace.\n", + space); + ut_error; + } + } + if (crypt_data) { - ulint space = fsp_header_get_space_id(page); - ulint offset = - fsp_header_get_crypt_offset( - fsp_flags_get_zip_size(*flags), NULL); - *crypt_data = fil_space_read_crypt_data(space, page, offset); + *crypt_data = cdata; } ut_free(buf); @@ -4203,14 +4185,6 @@ check_first_page: "%s in tablespace %s (table %s)", check_msg, fsp->filepath, tablename); fsp->success = FALSE; - if (strncmp(check_msg, FIL_MSG_CANNOT_DECRYPT, strlen(check_msg))==0) { - /* by returning here, it should be avoided, that the server crashes, - * if started in recovery mode and can not decrypt tables, if - * the key file can not be read. - */ - fsp->encryption_error = 1; - return; - } } if (!fsp->success) { diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index bc49b1988f1..fdc4d7f1ba4 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -567,9 +567,9 @@ ha_create_table_option innodb_table_option_list[]= /* With this option user can enable atomic writes feature for this table */ HA_TOPTION_ENUM("ATOMIC_WRITES", atomic_writes, "DEFAULT,ON,OFF", 0), /* With this option the user can enable encryption for the table */ - HA_TOPTION_ENUM("ENCRYPTION", encryption, "DEFAULT, ON, OFF", 0), + HA_TOPTION_ENUM("ENCRYPTION", encryption, "DEFAULT,ON,OFF", 0), /* With this option the user defines the key identifier using for the encryption */ - HA_TOPTION_NUMBER("ENCRYPTION_KEY", encryption_key, 0, 1, UINT_MAX32, 1), + HA_TOPTION_NUMBER("ENCRYPTION_KEY_ID", encryption_key_id, 0, 1, UINT_MAX32, 1), HA_TOPTION_END }; @@ -11375,29 +11375,19 @@ ha_innobase::check_table_options( } } - if (options->encryption_key != 0) { + if (options->encryption_key_id != 0) { if (options->encryption == FIL_SPACE_ENCRYPTION_OFF) { /* ignore this to allow alter table without changing page_encryption_key ...*/ } - if (options->encryption_key < 1) { + if (!has_encryption_key(options->encryption_key_id)) { push_warning_printf( thd, Sql_condition::WARN_LEVEL_WARN, HA_WRONG_CREATE_OPTION, - "InnoDB: invalid ENCRYPTION_KEY = %lu." - " Valid values are [1..INT32_MAX]", - options->encryption_key); - return "ENCRYPTION_KEY"; - } - - if (!has_encryption_key(options->encryption_key)) { - push_warning_printf( - thd, Sql_condition::WARN_LEVEL_WARN, - HA_WRONG_CREATE_OPTION, - "InnoDB: ENCRYPTION_KEY encryption key %lu not available", - options->encryption_key + "InnoDB: ENCRYPTION_KEY_ID key %lu not available", + options->encryption_key_id ); - return "ENCRYPTION_KEY"; + return "ENCRYPTION_KEY_ID"; } } @@ -11459,10 +11449,10 @@ ha_innobase::create( const char* stmt; size_t stmt_len; /* Cache table options */ - ha_table_option_struct *options= table->s->option_struct; + ha_table_option_struct *options= form->s->option_struct; fil_encryption_t encrypt = (fil_encryption_t)options->encryption; - ulint key_id = (options->encryption_key == 0) ? srv_default_encryption_key : - options->encryption_key; + ulint key_id = (options->encryption_key_id == 0) ? srv_default_encryption_key : + options->encryption_key_id; DBUG_ENTER("ha_innobase::create"); @@ -11702,18 +11692,6 @@ ha_innobase::create( innobase_commit_low(trx); - /* If user has requested that table should be encrypted or table - should remain as unencrypted store crypt data */ - if (encrypt == FIL_SPACE_ENCRYPTION_ON || encrypt == FIL_SPACE_ENCRYPTION_OFF) { - ulint maxsize; - ulint zip_size = fil_space_get_zip_size(innobase_table->space); - fil_space_crypt_t* crypt_data = fil_space_create_crypt_data(); - crypt_data->page0_offset = fsp_header_get_crypt_offset(zip_size, &maxsize); - crypt_data->keys[0].key_id = key_id; - crypt_data->encryption = encrypt; - fil_space_set_crypt_data(innobase_table->space, crypt_data); - } - row_mysql_unlock_data_dictionary(trx); /* Flush the log to reduce probability that the .frm files and @@ -11727,6 +11705,28 @@ ha_innobase::create( DBUG_ASSERT(innobase_table != 0); + /* If user has requested that table should be encrypted or table + should remain as unencrypted store crypt data */ + if (encrypt == FIL_SPACE_ENCRYPTION_ON || encrypt == FIL_SPACE_ENCRYPTION_OFF) { + ulint maxsize; + ulint zip_size = fil_space_get_zip_size(innobase_table->space); + fil_space_crypt_t* old_crypt_data = fil_space_get_crypt_data(innobase_table->space); + fil_space_crypt_t* crypt_data; + + crypt_data = fil_space_create_crypt_data(); + crypt_data->page0_offset = fsp_header_get_crypt_offset(zip_size, &maxsize); + crypt_data->keys[0].key_id = key_id; + crypt_data->encryption = encrypt; + + /* If there is old crypt data, copy IV */ + if (old_crypt_data) { + memcpy(crypt_data->iv, old_crypt_data->iv, old_crypt_data->iv_length); + crypt_data->iv_length = old_crypt_data->iv_length; + } + + fil_space_set_crypt_data(innobase_table->space, crypt_data); + } + innobase_copy_frm_flags_from_create_info(innobase_table, create_info); dict_stats_update(innobase_table, DICT_STATS_EMPTY_TABLE); diff --git a/storage/innobase/handler/ha_innodb.h b/storage/innobase/handler/ha_innodb.h index 2b9f1506339..b613053a992 100644 --- a/storage/innobase/handler/ha_innodb.h +++ b/storage/innobase/handler/ha_innodb.h @@ -71,7 +71,7 @@ struct ha_table_option_struct Atomic writes are not used if value OFF.*/ uint encryption; /*!< DEFAULT, ON, OFF */ - int encryption_key; /*!< encryption key id*/ + int encryption_key_id; /*!< encryption key id*/ }; diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index 327e4e92d1b..dc5c5d85b8d 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -279,7 +279,7 @@ ha_innobase::check_if_supported_inplace_alter( } if (new_options->encryption != old_options->encryption || - new_options->encryption_key != old_options->encryption_key) { + new_options->encryption_key_id != old_options->encryption_key_id) { ha_alter_info->unsupported_reason = innobase_get_err_msg( ER_ALTER_OPERATION_NOT_SUPPORTED_REASON); DBUG_RETURN(HA_ALTER_INPLACE_NOT_SUPPORTED); diff --git a/storage/innobase/include/fil0crypt.h b/storage/innobase/include/fil0crypt.h index 359533dc50a..da32d29e9e1 100644 --- a/storage/innobase/include/fil0crypt.h +++ b/storage/innobase/include/fil0crypt.h @@ -159,7 +159,7 @@ void fil_space_set_crypt_data( /*======================*/ ulint space, /*!< in: tablespace id */ - fil_space_crypt_t* crypt_data); /*!< in: crypt data */ + fil_space_crypt_t* crypt_data); /*!< in: crypt data to set */ /********************************************************************* Compare crypt data*/ diff --git a/storage/innobase/include/srv0srv.h b/storage/innobase/include/srv0srv.h index bfc65dbc51a..11e648efc85 100644 --- a/storage/innobase/include/srv0srv.h +++ b/storage/innobase/include/srv0srv.h @@ -1024,8 +1024,7 @@ struct export_var_t{ decrypted */ ulint innodb_sec_rec_cluster_reads; /*!< srv_sec_rec_cluster_reads */ - ulint innodb_sec_rec_cluster_reads_avoided; - /*!< srv_sec_rec_cluster_reads_avoided */ + ulint innodb_sec_rec_cluster_reads_avoided;/*!< srv_sec_rec_cluster_reads_avoided */ ulint innodb_encryption_rotation_pages_read_from_cache; ulint innodb_encryption_rotation_pages_read_from_disk; diff --git a/storage/xtradb/buf/buf0buf.cc b/storage/xtradb/buf/buf0buf.cc index 1bccc695223..bb8e9c6cce2 100644 --- a/storage/xtradb/buf/buf0buf.cc +++ b/storage/xtradb/buf/buf0buf.cc @@ -5868,6 +5868,7 @@ buf_page_encrypt_before_write( ulint space_id) /*!< in: space id */ { fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space_id); + bpage->real_size = UNIV_PAGE_SIZE; if (crypt_data != NULL && crypt_data->encryption == FIL_SPACE_ENCRYPTION_OFF) { /* Encryption is disabled */ @@ -5921,7 +5922,7 @@ buf_page_encrypt_before_write( mach_read_from_4(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); ut_ad(key_version == 0 || key_version >= bpage->key_version); bpage->key_version = key_version; - bpage->real_size = zip_size; + bpage->real_size = page_size; } else { /* First we compress the page content */ ulint out_len = 0; diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index e9ec7c1087b..ca61605c1f9 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -632,9 +632,9 @@ ha_create_table_option innodb_table_option_list[]= /* With this option user can enable atomic writes feature for this table */ HA_TOPTION_ENUM("ATOMIC_WRITES", atomic_writes, "DEFAULT,ON,OFF", 0), /* With this option the user can enable encryption for the table */ - HA_TOPTION_ENUM("ENCRYPTION", encryption, "DEFAULT, ON, OFF", 0), + HA_TOPTION_ENUM("ENCRYPTION", encryption, "DEFAULT,ON,OFF", 0), /* With this option the user defines the key identifier using for the encryption */ - HA_TOPTION_NUMBER("ENCRYPTION_KEY", encryption_key, 0, 1, UINT_MAX32, 1), + HA_TOPTION_NUMBER("ENCRYPTION_KEY_ID", encryption_key_id, 0, 1, UINT_MAX32, 1), HA_TOPTION_END }; @@ -11882,29 +11882,19 @@ ha_innobase::check_table_options( } } - if (options->encryption_key != 0) { + if (options->encryption_key_id != 0) { if (options->encryption == FIL_SPACE_ENCRYPTION_OFF) { /* ignore this to allow alter table without changing page_encryption_key ...*/ } - if (options->encryption_key < 1) { + if (!has_encryption_key(options->encryption_key_id)) { push_warning_printf( thd, Sql_condition::WARN_LEVEL_WARN, HA_WRONG_CREATE_OPTION, - "InnoDB: invalid ENCRYPTION_KEY = %lu." - " Valid values are [1..UINTMAX32]", - options->encryption_key); - return "ENCRYPTION_KEY"; - } - - if (!has_encryption_key(options->encryption_key)) { - push_warning_printf( - thd, Sql_condition::WARN_LEVEL_WARN, - HA_WRONG_CREATE_OPTION, - "InnoDB: ENCRYPTION_KEY %lu not available", - options->encryption_key + "InnoDB: ENCRYPTION_KEY_ID %lu not available", + options->encryption_key_id ); - return "ENCRYPTION_KEY"; + return "ENCRYPTION_KEY_ID"; } } @@ -11967,10 +11957,10 @@ ha_innobase::create( const char* stmt; size_t stmt_len; /* Cache table options */ - ha_table_option_struct *options= table->s->option_struct; + ha_table_option_struct *options= form->s->option_struct; fil_encryption_t encrypt = (fil_encryption_t)options->encryption; - ulint key_id = (options->encryption_key == 0) ? srv_default_encryption_key : - options->encryption_key; + ulint key_id = (options->encryption_key_id == 0) ? srv_default_encryption_key : + options->encryption_key_id; DBUG_ENTER("ha_innobase::create"); @@ -12216,18 +12206,6 @@ ha_innobase::create( innobase_commit_low(trx); - /* If user has requested that table should be encrypted or table - should remain as unencrypted store crypt data */ - if (encrypt == FIL_SPACE_ENCRYPTION_ON || encrypt == FIL_SPACE_ENCRYPTION_OFF) { - ulint maxsize; - ulint zip_size = fil_space_get_zip_size(innobase_table->space); - fil_space_crypt_t* crypt_data = fil_space_create_crypt_data(); - crypt_data->page0_offset = fsp_header_get_crypt_offset(zip_size, &maxsize); - crypt_data->keys[0].key_id = key_id; - crypt_data->encryption = encrypt; - fil_space_set_crypt_data(innobase_table->space, crypt_data); - } - row_mysql_unlock_data_dictionary(trx); /* Flush the log to reduce probability that the .frm files and @@ -12241,6 +12219,28 @@ ha_innobase::create( DBUG_ASSERT(innobase_table != 0); + /* If user has requested that table should be encrypted or table + should remain as unencrypted store crypt data */ + if (encrypt == FIL_SPACE_ENCRYPTION_ON || encrypt == FIL_SPACE_ENCRYPTION_OFF) { + ulint maxsize; + ulint zip_size = fil_space_get_zip_size(innobase_table->space); + fil_space_crypt_t* old_crypt_data = fil_space_get_crypt_data(innobase_table->space); + fil_space_crypt_t* crypt_data; + + crypt_data = fil_space_create_crypt_data(); + crypt_data->page0_offset = fsp_header_get_crypt_offset(zip_size, &maxsize); + crypt_data->keys[0].key_id = key_id; + crypt_data->encryption = encrypt; + + /* If there is old crypt data, copy IV */ + if (old_crypt_data) { + memcpy(crypt_data->iv, old_crypt_data->iv, old_crypt_data->iv_length); + crypt_data->iv_length = old_crypt_data->iv_length; + } + + fil_space_set_crypt_data(innobase_table->space, crypt_data); + } + innobase_copy_frm_flags_from_create_info(innobase_table, create_info); dict_stats_update(innobase_table, DICT_STATS_EMPTY_TABLE); @@ -20339,7 +20339,7 @@ static MYSQL_SYSVAR_UINT(encryption_rotation_iops, srv_n_fil_crypt_iops, static MYSQL_SYSVAR_UINT(default_encryption_key, srv_default_encryption_key, PLUGIN_VAR_RQCMDARG, - "Default encryption key used for table encryption.", + "Default encryption key id used for table encryption.", NULL, NULL, FIL_DEFAULT_ENCRYPTION_KEY, 1, UINT_MAX32, 0); diff --git a/storage/xtradb/handler/ha_innodb.h b/storage/xtradb/handler/ha_innodb.h index fdc6f9521aa..aadd2fc5bb8 100644 --- a/storage/xtradb/handler/ha_innodb.h +++ b/storage/xtradb/handler/ha_innodb.h @@ -72,7 +72,7 @@ struct ha_table_option_struct Atomic writes are not used if value OFF.*/ uint encryption; /*!< DEFAULT, ON, OFF */ - int encryption_key; /*!< encryption key id */ + int encryption_key_id; /*!< encryption key id */ }; /** The class defining a handle to an Innodb table */ diff --git a/storage/xtradb/handler/handler0alter.cc b/storage/xtradb/handler/handler0alter.cc index 9fe3cbc2bd1..8b1bd89d1c5 100644 --- a/storage/xtradb/handler/handler0alter.cc +++ b/storage/xtradb/handler/handler0alter.cc @@ -283,7 +283,7 @@ ha_innobase::check_if_supported_inplace_alter( } if (new_options->encryption != old_options->encryption || - new_options->encryption_key != old_options->encryption_key) { + new_options->encryption_key_id != old_options->encryption_key_id) { ha_alter_info->unsupported_reason = innobase_get_err_msg( ER_ALTER_OPERATION_NOT_SUPPORTED_REASON); DBUG_RETURN(HA_ALTER_INPLACE_NOT_SUPPORTED); diff --git a/storage/xtradb/include/fil0crypt.h b/storage/xtradb/include/fil0crypt.h index 359533dc50a..da32d29e9e1 100644 --- a/storage/xtradb/include/fil0crypt.h +++ b/storage/xtradb/include/fil0crypt.h @@ -159,7 +159,7 @@ void fil_space_set_crypt_data( /*======================*/ ulint space, /*!< in: tablespace id */ - fil_space_crypt_t* crypt_data); /*!< in: crypt data */ + fil_space_crypt_t* crypt_data); /*!< in: crypt data to set */ /********************************************************************* Compare crypt data*/ diff --git a/storage/xtradb/include/fsp0fsp.h b/storage/xtradb/include/fsp0fsp.h index 3d3d482a6c0..8fdacc51277 100644 --- a/storage/xtradb/include/fsp0fsp.h +++ b/storage/xtradb/include/fsp0fsp.h @@ -94,8 +94,8 @@ dictionary */ #define FSP_FLAGS_POS_ATOMIC_WRITES (FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL \ + FSP_FLAGS_WIDTH_PAGE_COMPRESSION_LEVEL) /** Zero relative shift position of the PAGE_SSIZE field */ -#define FSP_FLAGS_POS_PAGE_SSIZE (FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL \ - + FSP_FLAGS_WIDTH_PAGE_COMPRESSION_LEVEL) +#define FSP_FLAGS_POS_PAGE_SSIZE (FSP_FLAGS_POS_ATOMIC_WRITES \ + + FSP_FLAGS_WIDTH_ATOMIC_WRITES) /** Zero relative shift position of the start of the DATA DIR bits */ #define FSP_FLAGS_POS_DATA_DIR (FSP_FLAGS_POS_PAGE_SSIZE \ + FSP_FLAGS_WIDTH_PAGE_SSIZE) diff --git a/storage/xtradb/include/srv0srv.h b/storage/xtradb/include/srv0srv.h index 3cf312a4b4d..0135529ffe3 100644 --- a/storage/xtradb/include/srv0srv.h +++ b/storage/xtradb/include/srv0srv.h @@ -1242,8 +1242,7 @@ struct export_var_t{ decrypted */ ulint innodb_sec_rec_cluster_reads; /*!< srv_sec_rec_cluster_reads */ - ulint innodb_sec_rec_cluster_reads_avoided; - /*!< srv_sec_rec_cluster_reads_avoided */ + ulint innodb_sec_rec_cluster_reads_avoided; /*!< srv_sec_rec_cluster_reads_avoided */ ulint innodb_encryption_rotation_pages_read_from_cache; ulint innodb_encryption_rotation_pages_read_from_disk; From d2b87801b68e81ae4225f90bbcab03c7e35aea01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Thu, 2 Apr 2015 16:52:30 +0300 Subject: [PATCH 6/6] Fix test failure on these scrub tests. --- .../innodb/t/innodb_scrub_background.opt | 19 +++++----------- .../innodb/t/innodb_scrub_compressed.opt | 22 ++++++------------- 2 files changed, 13 insertions(+), 28 deletions(-) diff --git a/mysql-test/suite/innodb/t/innodb_scrub_background.opt b/mysql-test/suite/innodb/t/innodb_scrub_background.opt index bb1f4ce4e86..cd5f74570bb 100644 --- a/mysql-test/suite/innodb/t/innodb_scrub_background.opt +++ b/mysql-test/suite/innodb/t/innodb_scrub_background.opt @@ -1,14 +1,7 @@ ---innodb-file-per-table=1 ---innodb-file-format=Barracuda ---innodb-encrypt-tables=0 ---innodb-encryption-threads=0 ---innodb-immediate-scrub-data-uncompressed=OFF ---innodb-background-scrub-data-uncompressed=ON ---innodb-background-scrub-data-compressed=ON +--innodb_file_per_table=1 +--innodb_file_format=Barracuda +--innodb-immediate-scrub-data-uncompressed=OFF +--innodb-background-scrub-data-uncompressed=ON +--innodb-background-scrub-data-compressed=ON --loose-innodb-scrub-force-testing=ON ---encrypt-tmp-disk-tables=ON ---aria-encrypt-tables=ON ---innodb-encryption-threads=4 ---innodb-encryption-rotate-key-age=15 ---innodb-encrypt-tables=ON ---innodb-tablespaces-scrubbing +--innodb-tablespaces_scrubbing diff --git a/mysql-test/suite/innodb/t/innodb_scrub_compressed.opt b/mysql-test/suite/innodb/t/innodb_scrub_compressed.opt index ac7ef8a1675..c94e9b34300 100644 --- a/mysql-test/suite/innodb/t/innodb_scrub_compressed.opt +++ b/mysql-test/suite/innodb/t/innodb_scrub_compressed.opt @@ -1,15 +1,7 @@ ---innodb-file-per-table=1 ---innodb-file-format=Barracuda ---innodb-encrypt-tables=off ---innodb-immediate-scrub-data-uncompressed=ON ---innodb-background-scrub-data-uncompressed=ON ---innodb-background-scrub-data-compressed=ON ---loose-innodb-scrub-force-testing=ON ---innodb-encryption-threads=0 ---encrypt-tmp-disk-tables=ON ---aria-encrypt-tables=ON ---innodb-encryption-threads=4 ---innodb-encryption-rotate-key-age=15 ---innodb-encrypt-tables=ON ---innodb-tablespaces-scrubbing - +--innodb_file_per_table=1 +--innodb_file_format=Barracuda +--innodb-immediate-scrub-data-uncompressed=ON +--innodb-background-scrub-data-uncompressed=ON +--innodb-background-scrub-data-compressed=ON +--loose-innodb-scrub-force-testing=ON +--innodb-tablespaces_scrubbing