diff --git a/extra/CMakeLists.txt b/extra/CMakeLists.txt index 5b650413bf3..c8e15dd4fb4 100644 --- a/extra/CMakeLists.txt +++ b/extra/CMakeLists.txt @@ -77,7 +77,9 @@ IF(WITH_INNOBASE_STORAGE_ENGINE OR WITH_XTRADB_STORAGE_ENGINE) ../storage/innobase/buf/buf0checksum.cc ../storage/innobase/ut/ut0crc32.cc ../storage/innobase/ut/ut0ut.cc + ../storage/innobase/buf/buf0buf.cc ../storage/innobase/page/page0zip.cc + ../storage/innobase/fil/fil0crypt.cc ) IF(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64le") diff --git a/extra/innochecksum.cc b/extra/innochecksum.cc index 28191da2c48..e258a8c1d08 100644 --- a/extra/innochecksum.cc +++ b/extra/innochecksum.cc @@ -1,6 +1,6 @@ /* - Copyright (c) 2005, 2012, Oracle and/or its affiliates. - Copyright (c) 2014, 2015, MariaDB Corporation. + Copyright (c) 2005, 2016, Oracle and/or its affiliates. All rights reserved. + Copyright (c) 2014, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -42,6 +42,8 @@ /* Only parts of these files are included from the InnoDB codebase. The parts not included are excluded by #ifndef UNIV_INNOCHECKSUM. */ +typedef void fil_space_t; + #include "univ.i" /* include all of this */ #define FLST_BASE_NODE_SIZE (4 + 2 * FIL_ADDR_SIZE) @@ -55,6 +57,7 @@ The parts not included are excluded by #ifndef UNIV_INNOCHECKSUM. */ #include "fsp0types.h" #include "rem0rec.h" #include "buf0checksum.h" /* buf_calc_page_*() */ +#include "buf0buf.h" /* buf_page_is_corrupted */ #include "fil0fil.h" /* FIL_* */ #include "page0page.h" /* PAGE_* */ #include "page0zip.h" /* page_zip_*() */ @@ -63,6 +66,9 @@ The parts not included are excluded by #ifndef UNIV_INNOCHECKSUM. */ fsp_flags_get_zip_size() */ #include "ut0crc32.h" /* ut_crc32_init() */ #include "fsp0pagecompress.h" /* fil_get_compression_alg_name */ +#include "fil0crypt.h" /* fil_space_verify_crypt_checksum */ + +#include #ifdef UNIV_NONINL # include "fsp0fsp.ic" @@ -70,50 +76,104 @@ The parts not included are excluded by #ifndef UNIV_INNOCHECKSUM. */ # include "ut0rnd.ic" #endif +#ifndef PRIuMAX +#define PRIuMAX "llu" +#endif + /* Global variables */ -static my_bool verbose; -static my_bool debug; -static my_bool skip_corrupt; -static my_bool just_count; -static ulong start_page; -static ulong end_page; -static ulong do_page; -static my_bool use_end_page; -static my_bool do_one_page; -static my_bool per_page_details; +static bool verbose; +static bool just_count; +static ulint start_page; +static ulint end_page; +static ulint do_page; +static bool use_end_page; +static bool do_one_page; static my_bool do_leaf; +static my_bool per_page_details; static ulong n_merge; -ulong srv_page_size; /* replaces declaration in srv0srv.c */ +extern ulong srv_checksum_algorithm; static ulong physical_page_size; /* Page size in bytes on disk. */ static ulong logical_page_size; /* Page size when uncompressed. */ -static bool compressed= false; /* Is tablespace compressed */ +ulong srv_page_size; +/* Current page number (0 based). */ +ulint cur_page_num; +/* Skip the checksum verification. */ +static bool no_check; +/* Enabled for strict checksum verification. */ +bool strict_verify = 0; +/* Enabled for rewrite checksum. */ +static bool do_write; +/* Mismatches count allowed (0 by default). */ +static ulint allow_mismatches; +static bool page_type_summary; +static bool page_type_dump; +/* Store filename for page-type-dump option. */ +char* page_dump_filename = 0; +/* skip the checksum verification & rewrite if page is doublewrite buffer. */ +static bool skip_page = 0; +const char *dbug_setting = "FALSE"; +char* log_filename = NULL; +/* User defined filename for logging. */ +FILE* log_file = NULL; +/* Enabled for log write option. */ +static bool is_log_enabled = false; -int n_undo_state_active; -int n_undo_state_cached; -int n_undo_state_to_free; -int n_undo_state_to_purge; -int n_undo_state_prepared; -int n_undo_state_other; -int n_undo_insert, n_undo_update, n_undo_other; -int n_bad_checksum; -int n_fil_page_index; -int n_fil_page_undo_log; -int n_fil_page_inode; -int n_fil_page_ibuf_free_list; -int n_fil_page_allocated; -int n_fil_page_ibuf_bitmap; -int n_fil_page_type_sys; -int n_fil_page_type_trx_sys; -int n_fil_page_type_fsp_hdr; -int n_fil_page_type_allocated; -int n_fil_page_type_xdes; -int n_fil_page_type_blob; -int n_fil_page_type_zblob; -int n_fil_page_type_other; -int n_fil_page_type_page_compressed; -int n_fil_page_type_page_compressed_encrypted; +#ifndef _WIN32 +/* advisory lock for non-window system. */ +struct flock lk; +#endif /* _WIN32 */ -int n_fil_page_max_index_id; +/* Strict check algorithm name. */ +static ulong strict_check; +/* Rewrite checksum algorithm name. */ +static ulong write_check; + +/* Innodb page type. */ +struct innodb_page_type { + int n_undo_state_active; + int n_undo_state_cached; + int n_undo_state_to_free; + int n_undo_state_to_purge; + int n_undo_state_prepared; + int n_undo_state_other; + int n_undo_insert; + int n_undo_update; + int n_undo_other; + int n_fil_page_index; + int n_fil_page_undo_log; + int n_fil_page_inode; + int n_fil_page_ibuf_free_list; + int n_fil_page_ibuf_bitmap; + int n_fil_page_type_sys; + int n_fil_page_type_trx_sys; + int n_fil_page_type_fsp_hdr; + int n_fil_page_type_allocated; + int n_fil_page_type_xdes; + int n_fil_page_type_blob; + int n_fil_page_type_zblob; + int n_fil_page_type_other; + int n_fil_page_type_zblob2; + int n_fil_page_type_page_compressed; + int n_fil_page_type_page_compressed_encrypted; +} page_type; + +/* Possible values for "--strict-check" for strictly verify checksum +and "--write" for rewrite checksum. */ +static const char *innochecksum_algorithms[] = { + "crc32", + "crc32", + "innodb", + "innodb", + "none", + "none", + NullS +}; + +/* Used to define an enumerate type of the "innochecksum algorithm". */ +static TYPELIB innochecksum_algorithms_typelib = { + array_elements(innochecksum_algorithms)-1,"", + innochecksum_algorithms, NULL +}; #define SIZE_RANGES_FOR_PAGE 10 #define NUM_RETRIES 3 @@ -143,7 +203,7 @@ struct per_index_stats { last element for pages with more than logical_page_size */ unsigned long long pages_in_size_range[SIZE_RANGES_FOR_PAGE+2]; - std::map leaves; + std::map leaves; per_index_stats():pages(0), leaf_pages(0), first_leaf_page(0), count(0), free_pages(0), max_data_size(0), total_n_recs(0), @@ -155,64 +215,940 @@ struct per_index_stats { std::map index_ids; -bool encrypted = false; +void print_index_leaf_stats( + unsigned long long id, + const per_index_stats& index, + FILE* fil_out) -/* Get the page size of the filespace from the filespace header. */ -static -my_bool -get_page_size( -/*==========*/ - FILE* f, /*!< in: file pointer, must be open - and set to start of file */ - byte* buf, /*!< in: buffer used to read the page */ - ulong* logical_page_size, /*!< out: Logical/Uncompressed page size */ - ulong* physical_page_size) /*!< out: Physical/Commpressed page size */ { - ulong flags; - - int bytes= fread(buf, 1, UNIV_PAGE_SIZE_MIN, f); - - if (ferror(f)) - { - perror("Error reading file header"); - return FALSE; - } - - if (bytes != UNIV_PAGE_SIZE_MIN) - { - fprintf(stderr, "Error; Was not able to read the minimum page size "); - fprintf(stderr, "of %d bytes. Bytes read was %d\n", UNIV_PAGE_SIZE_MIN, bytes); - return FALSE; - } - - rewind(f); - - flags = mach_read_from_4(buf + FIL_PAGE_DATA + FSP_SPACE_FLAGS); - - /* srv_page_size is used by InnoDB code as UNIV_PAGE_SIZE */ - srv_page_size = *logical_page_size = fsp_flags_get_page_size(flags); - - /* fsp_flags_get_zip_size() will return zero if not compressed. */ - *physical_page_size = fsp_flags_get_zip_size(flags); - if (*physical_page_size == 0) - { - *physical_page_size= *logical_page_size; - } - else - { - compressed= true; - } - - - return TRUE; + ulint page_no = index.first_leaf_page; + std::map::const_iterator it_page = index.leaves.find(page_no); + fprintf(fil_out, "\nindex: %llu leaf page stats: n_pages = %llu\n", + id, index.leaf_pages); + fprintf(fil_out, "page_no\tdata_size\tn_recs\n"); + while (it_page != index.leaves.end()) { + const per_page_stats& stat = it_page->second; + fprintf(fil_out, "%llu\t%lu\t%lu\n", it_page->first, stat.data_size, stat.n_recs); + page_no = stat.right_page_no; + it_page = index.leaves.find(page_no); + } } - -/* command line argument to do page checks (that's it) */ -/* another argument to specify page ranges... seek to right spot and go from there */ - -static struct my_option innochecksum_options[] = +void defrag_analysis( + unsigned long long id, + const per_index_stats& index, + FILE* fil_out) { + // TODO: make it work for compressed pages too + std::map::const_iterator it = index.leaves.find(index.first_leaf_page); + ulint n_pages = 0; + ulint n_leaf_pages = 0; + while (it != index.leaves.end()) { + ulint data_size_total = 0; + for (ulong i = 0; i < n_merge; i++) { + const per_page_stats& stat = it->second; + n_leaf_pages ++; + data_size_total += stat.data_size; + it = index.leaves.find(stat.right_page_no); + if (it == index.leaves.end()) { + break; + } + } + + if (index.max_data_size) { + n_pages += data_size_total / index.max_data_size; + if (data_size_total % index.max_data_size != 0) { + n_pages += 1; + } + } + } + + if (index.leaf_pages) { + fprintf(fil_out, "count = %lu free = %lu\n", index.count, index.free_pages); + } + + if (!n_leaf_pages) { + n_leaf_pages = 1; + } + + fprintf(fil_out, "%llu\t\t%llu\t\t%lu\t\t%lu\t\t%lu\t\t%.2f\t%lu\n", + id, index.leaf_pages, n_leaf_pages, n_merge, n_pages, + 1.0 - (double)n_pages / (double)n_leaf_pages, index.max_data_size); +} + +void print_leaf_stats( + FILE* fil_out) +{ + fprintf(fil_out, "\n**************************************************\n"); + fprintf(fil_out, "index_id\t#leaf_pages\t#actual_leaf_pages\tn_merge\t" + "#leaf_after_merge\tdefrag\n"); + for (std::map::const_iterator it = index_ids.begin(); + it != index_ids.end(); it++) { + const per_index_stats& index = it->second; + + if (verbose) { + print_index_leaf_stats(it->first, index, fil_out); + } + + if (n_merge) { + defrag_analysis(it->first, index, fil_out); + } + } +} + +#ifdef _WIN32 +/***********************************************//* + @param [in] error error no. from the getLastError(). + + @retval error message corresponding to error no. +*/ +static +char* +error_message( + int error) +{ + static char err_msg[1024] = {'\0'}; + FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM, + NULL, error, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + (LPTSTR)err_msg, sizeof(err_msg), NULL ); + + return (err_msg); +} +#endif /* _WIN32 */ + +/***********************************************//* + @param>>_______[in] name>_____name of file. + @retval file pointer; file pointer is NULL when error occured. +*/ + +FILE* +open_file( + const char* name) +{ + int fd; /* file descriptor. */ + FILE* fil_in; +#ifdef _WIN32 + HANDLE hFile; /* handle to open file. */ + DWORD access; /* define access control */ + int flags = 0; /* define the mode for file + descriptor */ + + if (do_write) { + access = GENERIC_READ | GENERIC_WRITE; + flags = _O_RDWR | _O_BINARY; + } else { + access = GENERIC_READ; + flags = _O_RDONLY | _O_BINARY; + } + + /* CreateFile() also provide advisory lock with the usage of + access and share mode of the file.*/ + hFile = CreateFile( + (LPCTSTR) name, access, 0L, NULL, + OPEN_EXISTING, NULL, NULL); + + if (hFile == INVALID_HANDLE_VALUE) { + /* print the error message. */ + fprintf(stderr, "Filename::%s %s\n", name, + error_message(GetLastError())); + + return (NULL); + } + + /* get the file descriptor. */ + fd= _open_osfhandle((intptr_t)hFile, flags); +#else /* _WIN32 */ + + int create_flag; + /* define the advisory lock and open file mode. */ + if (do_write) { + create_flag = O_RDWR; + lk.l_type = F_WRLCK; + } else { + create_flag = O_RDONLY; + lk.l_type = F_RDLCK; + } + + fd = open(name, create_flag); + + lk.l_whence = SEEK_SET; + lk.l_start = lk.l_len = 0; + + if (fcntl(fd, F_SETLK, &lk) == -1) { + fprintf(stderr, "Error: Unable to lock file::" + " %s\n", name); + perror("fcntl"); + return (NULL); + } +#endif /* _WIN32 */ + + if (do_write) { + fil_in = fdopen(fd, "rb+"); + } else { + fil_in = fdopen(fd, "rb"); + } + + return (fil_in); +} + +/************************************************************//* + Read the content of file + + @param [in,out] buf read the file in buffer + @param [in] partial_page_read enable when to read the + remaining buffer for first page. + @param [in] physical_page_size Physical/Commpressed page size. + @param [in,out] fil_in file pointer created for the + tablespace. + @retval no. of bytes read. +*/ +ulong read_file( + byte* buf, + bool partial_page_read, + ulong physical_page_size, + FILE* fil_in) +{ + ulong bytes = 0; + + DBUG_ASSERT(physical_page_size >= UNIV_ZIP_SIZE_MIN); + + if (partial_page_read) { + buf += UNIV_ZIP_SIZE_MIN; + physical_page_size -= UNIV_ZIP_SIZE_MIN; + bytes = UNIV_ZIP_SIZE_MIN; + } + + bytes += ulong(fread(buf, 1, physical_page_size, fil_in)); + + return bytes; +} + +/** Check if page is corrupted or not. +@param[in] buf page frame +@param[in] page_size page size +@param[in] zip_size != if page row compressed +@param[in] is_encrypted true if page0 contained cryp_data + with crypt_scheme encrypted +@param[in] is_compressed true if page0 fsp_flags contained + page compression flag +@retval true if page is corrupted otherwise false. */ +static +bool +is_page_corrupted( + byte* buf, + ulint page_size, + ulint zip_size, + bool is_encrypted, + bool is_compressed) +{ + + /* enable if page is corrupted. */ + bool is_corrupted; + /* use to store LSN values. */ + ulint logseq; + ulint logseqfield; + ulint page_type = mach_read_from_2(buf+FIL_PAGE_TYPE); + ulint key_version = mach_read_from_4(buf+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); + ulint space_id = mach_read_from_4( + buf + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); + + /* We can't trust only a page type, thus we take account + also fsp_flags or crypt_data on page 0 */ + if ((page_type == FIL_PAGE_PAGE_COMPRESSED && is_compressed) || + (page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED && + is_compressed && is_encrypted)) { + /* Page compressed tables do not contain post compression + checksum. */ + return (false); + } + + if (!zip_size) { + /* check the stored log sequence numbers + for uncompressed tablespace. */ + logseq = mach_read_from_4(buf + FIL_PAGE_LSN + 4); + logseqfield = mach_read_from_4( + buf + page_size - + FIL_PAGE_END_LSN_OLD_CHKSUM + 4); + + if (is_log_enabled) { + fprintf(log_file, + "space::%" PRIuMAX " page::%" PRIuMAX + "; log sequence number:first = " ULINTPF + "; second = " ULINTPF "\n", + space_id, cur_page_num, logseq, logseqfield); + if (logseq != logseqfield) { + fprintf(log_file, + "Fail; space %" PRIuMAX " page %" PRIuMAX + " invalid (fails log " + "sequence number check)\n", + space_id, cur_page_num); + } + } + } + + /* Again we can't trust only FIL_PAGE_FILE_FLUSH_LSN field + now repurposed as FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, + we need to check also crypt_data contents. */ + if (is_encrypted && key_version != 0) { + is_corrupted = !fil_space_verify_crypt_checksum(buf, + zip_size, NULL, cur_page_num); + } else { + is_corrupted = buf_page_is_corrupted( + true, buf, zip_size, NULL); + } + + return(is_corrupted); +} + +/********************************************//* + Check if page is doublewrite buffer or not. + @param [in] page buffer page + + @retval true if page is doublewrite buffer otherwise false. +*/ +static +bool +is_page_doublewritebuffer( + const byte* page) +{ + if ((cur_page_num >= FSP_EXTENT_SIZE) + && (cur_page_num < FSP_EXTENT_SIZE * 3)) { + /* page is doublewrite buffer. */ + return (true); + } + + return (false); +} + +/*******************************************************//* +Check if page is empty or not. + @param [in] page page to checked for empty. + @param [in] len size of page. + + @retval true if page is empty. + @retval false if page is not empty. +*/ +static +bool +is_page_empty( + const byte* page, + size_t len) +{ + while (len--) { + if (*page++) { + return (false); + } + } + return (true); +} + +/********************************************************************//** +Rewrite the checksum for the page. +@param [in/out] page page buffer +@param [in] physical_page_size page size in bytes on disk. +@param [in] iscompressed Is compressed/Uncompressed Page. + +@retval true : do rewrite +@retval false : skip the rewrite as checksum stored match with + calculated or page is doublwrite buffer. +*/ + +bool +update_checksum( + byte* page, + ulong physical_page_size, + bool iscompressed) +{ + ib_uint32_t checksum = 0; + byte stored1[4]; /* get FIL_PAGE_SPACE_OR_CHKSUM field checksum */ + byte stored2[4]; /* get FIL_PAGE_END_LSN_OLD_CHKSUM field checksum */ + + ut_ad(page); + /* If page is doublewrite buffer, skip the rewrite of checksum. */ + if (skip_page) { + return (false); + } + + memcpy(stored1, page + FIL_PAGE_SPACE_OR_CHKSUM, 4); + memcpy(stored2, page + physical_page_size - + FIL_PAGE_END_LSN_OLD_CHKSUM, 4); + + /* Check if page is empty, exclude the checksum field */ + if (is_page_empty(page + 4, physical_page_size - 12) + && is_page_empty(page + physical_page_size - 4, 4)) { + + memset(page + FIL_PAGE_SPACE_OR_CHKSUM, 0, 4); + memset(page + physical_page_size - + FIL_PAGE_END_LSN_OLD_CHKSUM, 0, 4); + + goto func_exit; + } + + if (iscompressed) { + /* page is compressed */ + checksum = page_zip_calc_checksum( + page, physical_page_size, + static_cast(write_check)); + + mach_write_to_4(page + FIL_PAGE_SPACE_OR_CHKSUM, checksum); + if (is_log_enabled) { + fprintf(log_file, "page::%" PRIuMAX "; Updated checksum =" + " %u\n", cur_page_num, checksum); + } + + } else { + /* page is uncompressed. */ + + /* Store the new formula checksum */ + switch ((srv_checksum_algorithm_t) write_check) { + + case SRV_CHECKSUM_ALGORITHM_CRC32: + case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32: + checksum = buf_calc_page_crc32(page); + break; + + case SRV_CHECKSUM_ALGORITHM_INNODB: + case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB: + checksum = (ib_uint32_t) + buf_calc_page_new_checksum(page); + break; + + case SRV_CHECKSUM_ALGORITHM_NONE: + case SRV_CHECKSUM_ALGORITHM_STRICT_NONE: + checksum = BUF_NO_CHECKSUM_MAGIC; + break; + /* no default so the compiler will emit a warning if new + enum is added and not handled here */ + } + + mach_write_to_4(page + FIL_PAGE_SPACE_OR_CHKSUM, checksum); + if (is_log_enabled) { + fprintf(log_file, "page::%" PRIuMAX "; Updated checksum field1" + " = %u\n", cur_page_num, checksum); + } + + if (write_check == SRV_CHECKSUM_ALGORITHM_STRICT_INNODB + || write_check == SRV_CHECKSUM_ALGORITHM_INNODB) { + checksum = (ib_uint32_t) + buf_calc_page_old_checksum(page); + } + + mach_write_to_4(page + physical_page_size - + FIL_PAGE_END_LSN_OLD_CHKSUM,checksum); + + if (is_log_enabled) { + fprintf(log_file, "page::%" PRIuMAX "; Updated checksum " + "field2 = %u\n", cur_page_num, checksum); + } + + } + + func_exit: + /* The following code is to check the stored checksum with the + calculated checksum. If it matches, then return FALSE to skip + the rewrite of checksum, otherwise return TRUE. */ + if (iscompressed) { + if (!memcmp(stored1, page + FIL_PAGE_SPACE_OR_CHKSUM, 4)) { + return (false); + } + return (true); + } + + if (!memcmp(stored1, page + FIL_PAGE_SPACE_OR_CHKSUM, 4) + && !memcmp(stored2, page + physical_page_size - + FIL_PAGE_END_LSN_OLD_CHKSUM, 4)) { + return (false); + + } + + return (true); +} + +/** + Write the content to the file +@param[in] filename name of the file. +@param[in,out] file file pointer where content + have to be written +@param[in] buf file buffer read +@param[in] compressed Enabled if tablespace is + compressed. +@param[in,out] pos current file position. +@param[in] page_size page size in bytes on disk. + +@retval true if successfully written +@retval false if a non-recoverable error occurred +*/ +static +bool +write_file( + const char* filename, + FILE* file, + byte* buf, + bool compressed, + fpos_t* pos, + ulong page_size) +{ + bool do_update; + + do_update = update_checksum(buf, page_size, compressed); + + if (file != stdin) { + if (do_update) { + /* Set the previous file pointer position + saved in pos to current file position. */ + if (0 != fsetpos(file, pos)) { + perror("fsetpos"); + return(false); + } + } else { + /* Store the current file position in pos */ + if (0 != fgetpos(file, pos)) { + perror("fgetpos"); + return(false); + } + return(true); + } + } + + if (page_size + != fwrite(buf, 1, page_size, file == stdin ? stdout : file)) { + fprintf(stderr, "Failed to write page %" PRIuMAX " to %s: %s\n", + cur_page_num, filename, strerror(errno)); + + return(false); + } + if (file != stdin) { + fflush(file); + /* Store the current file position in pos */ + if (0 != fgetpos(file, pos)) { + perror("fgetpos"); + return(false); + } + } + + return(true); +} + +/* +Parse the page and collect/dump the information about page type +@param [in] page buffer page +@param [out] xdes extend descriptor page +@param [in] file file for diagnosis. +*/ +void +parse_page( + const byte* page, + byte* xdes, + FILE* file) +{ + unsigned long long id; + ulint undo_page_type; + char str[20]={'\0'}; + ulint n_recs; + ulint page_no; + ulint left_page_no; + ulint right_page_no; + ulint data_bytes; + int is_leaf; + int size_range_id; + + /* Check whether page is doublewrite buffer. */ + if(skip_page) { + strcpy(str, "Double_write_buffer"); + } else { + strcpy(str, "-"); + } + + switch (mach_read_from_2(page + FIL_PAGE_TYPE)) { + + case FIL_PAGE_INDEX: + page_type.n_fil_page_index++; + id = mach_read_from_8(page + PAGE_HEADER + PAGE_INDEX_ID); + n_recs = page_get_n_recs(page); + page_no = page_get_page_no(page); + left_page_no = mach_read_from_4(page + FIL_PAGE_PREV); + right_page_no = mach_read_from_4(page + FIL_PAGE_NEXT); + data_bytes = page_get_data_size(page); + is_leaf = page_is_leaf(page); + size_range_id = (data_bytes * SIZE_RANGES_FOR_PAGE + + logical_page_size - 1) / + logical_page_size; + if (size_range_id > SIZE_RANGES_FOR_PAGE + 1) { + /* data_bytes is bigger than logical_page_size */ + size_range_id = SIZE_RANGES_FOR_PAGE + 1; + } + if (per_page_details) { + printf("index %llu page %lu leaf %u n_recs %lu data_bytes %lu" + "\n", id, page_no, is_leaf, n_recs, data_bytes); + } + /* update per-index statistics */ + { + if (index_ids.count(id) == 0) { + index_ids[id] = per_index_stats(); + } + std::map::iterator it; + it = index_ids.find(id); + per_index_stats &index = (it->second); + const byte* des = xdes + XDES_ARR_OFFSET + + XDES_SIZE * ((page_no & (physical_page_size - 1)) + / FSP_EXTENT_SIZE); + if (xdes_get_bit(des, XDES_FREE_BIT, + page_no % FSP_EXTENT_SIZE)) { + index.free_pages++; + return; + } + index.pages++; + if (is_leaf) { + index.leaf_pages++; + if (data_bytes > index.max_data_size) { + index.max_data_size = data_bytes; + } + struct per_page_stats pp(n_recs, data_bytes, + left_page_no, right_page_no); + + index.leaves[page_no] = pp; + + if (left_page_no == ULINT32_UNDEFINED) { + index.first_leaf_page = page_no; + index.count++; + } + } + index.total_n_recs += n_recs; + index.total_data_bytes += data_bytes; + index.pages_in_size_range[size_range_id] ++; + } + + if (page_type_dump) { + fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tIndex page\t\t\t|" + "\tindex id=%llu,", cur_page_num, id); + + fprintf(file, + " page level=" ULINTPF + ", No. of records=" ULINTPF + ", garbage=" ULINTPF ", %s\n", + page_header_get_field(page, PAGE_LEVEL), + page_header_get_field(page, PAGE_N_RECS), + page_header_get_field(page, PAGE_GARBAGE), str); + } + break; + + case FIL_PAGE_UNDO_LOG: + page_type.n_fil_page_undo_log++; + undo_page_type = mach_read_from_2(page + + TRX_UNDO_PAGE_HDR + TRX_UNDO_PAGE_TYPE); + if (page_type_dump) { + fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tUndo log page\t\t\t|", + cur_page_num); + } + if (undo_page_type == TRX_UNDO_INSERT) { + page_type.n_undo_insert++; + if (page_type_dump) { + fprintf(file, "\t%s", + "Insert Undo log page"); + } + + } else if (undo_page_type == TRX_UNDO_UPDATE) { + page_type.n_undo_update++; + if (page_type_dump) { + fprintf(file, "\t%s", + "Update undo log page"); + } + } + + undo_page_type = mach_read_from_2(page + TRX_UNDO_SEG_HDR + + TRX_UNDO_STATE); + switch (undo_page_type) { + case TRX_UNDO_ACTIVE: + page_type.n_undo_state_active++; + if (page_type_dump) { + fprintf(file, ", %s", "Undo log of " + "an active transaction"); + } + break; + + case TRX_UNDO_CACHED: + page_type.n_undo_state_cached++; + if (page_type_dump) { + fprintf(file, ", %s", "Page is " + "cached for quick reuse"); + } + break; + + case TRX_UNDO_TO_FREE: + page_type.n_undo_state_to_free++; + if (page_type_dump) { + fprintf(file, ", %s", "Insert undo " + "segment that can be freed"); + } + break; + + case TRX_UNDO_TO_PURGE: + page_type.n_undo_state_to_purge++; + if (page_type_dump) { + fprintf(file, ", %s", "Will be " + "freed in purge when all undo" + "data in it is removed"); + } + break; + + case TRX_UNDO_PREPARED: + page_type.n_undo_state_prepared++; + if (page_type_dump) { + fprintf(file, ", %s", "Undo log of " + "an prepared transaction"); + } + break; + + default: + page_type.n_undo_state_other++; + break; + } + if(page_type_dump) { + fprintf(file, ", %s\n", str); + } + break; + + case FIL_PAGE_INODE: + page_type.n_fil_page_inode++; + if (page_type_dump) { + fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tInode page\t\t\t|" + "\t%s\n",cur_page_num, str); + } + break; + + case FIL_PAGE_IBUF_FREE_LIST: + page_type.n_fil_page_ibuf_free_list++; + if (page_type_dump) { + fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tInsert buffer free list" + " page\t|\t%s\n", cur_page_num, str); + } + break; + + case FIL_PAGE_TYPE_ALLOCATED: + page_type.n_fil_page_type_allocated++; + if (page_type_dump) { + fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tFreshly allocated " + "page\t\t|\t%s\n", cur_page_num, str); + } + break; + + case FIL_PAGE_IBUF_BITMAP: + page_type.n_fil_page_ibuf_bitmap++; + if (page_type_dump) { + fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tInsert Buffer " + "Bitmap\t\t|\t%s\n", cur_page_num, str); + } + break; + + case FIL_PAGE_TYPE_SYS: + page_type.n_fil_page_type_sys++; + if (page_type_dump) { + fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tSystem page\t\t\t|" + "\t%s\n",cur_page_num, str); + } + break; + + case FIL_PAGE_TYPE_TRX_SYS: + page_type.n_fil_page_type_trx_sys++; + if (page_type_dump) { + fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tTransaction system " + "page\t\t|\t%s\n", cur_page_num, str); + } + break; + + case FIL_PAGE_TYPE_FSP_HDR: + page_type.n_fil_page_type_fsp_hdr++; + memcpy(xdes, page, physical_page_size); + if (page_type_dump) { + fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tFile Space " + "Header\t\t|\t%s\n", cur_page_num, str); + } + break; + + case FIL_PAGE_TYPE_XDES: + page_type.n_fil_page_type_xdes++; + memcpy(xdes, page, physical_page_size); + if (page_type_dump) { + fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tExtent descriptor " + "page\t\t|\t%s\n", cur_page_num, str); + } + break; + + case FIL_PAGE_TYPE_BLOB: + page_type.n_fil_page_type_blob++; + if (page_type_dump) { + fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tBLOB page\t\t\t|\t%s\n", + cur_page_num, str); + } + break; + + case FIL_PAGE_TYPE_ZBLOB: + page_type.n_fil_page_type_zblob++; + if (page_type_dump) { + fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tCompressed BLOB " + "page\t\t|\t%s\n", cur_page_num, str); + } + break; + + case FIL_PAGE_TYPE_ZBLOB2: + page_type.n_fil_page_type_zblob2++; + if (page_type_dump) { + fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tSubsequent Compressed " + "BLOB page\t|\t%s\n", cur_page_num, str); + } + break; + + case FIL_PAGE_PAGE_COMPRESSED: + page_type.n_fil_page_type_page_compressed++; + if (page_type_dump) { + fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tPage compressed " + "page\t|\t%s\n", cur_page_num, str); + } + break; + + case FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED: + page_type.n_fil_page_type_page_compressed_encrypted++; + if (page_type_dump) { + fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tPage compressed encrypted " + "page\t|\t%s\n", cur_page_num, str); + } + break; + default: + page_type.n_fil_page_type_other++; + break; + } +} +/** +@param [in/out] file_name name of the filename + +@retval FILE pointer if successfully created else NULL when error occured. +*/ +FILE* +create_file( + char* file_name) +{ + FILE* file = NULL; + +#ifndef _WIN32 + file = fopen(file_name, "wb"); + if (file == NULL) { + fprintf(stderr, "Failed to create file: %s: %s\n", + file_name, strerror(errno)); + return(NULL); + } +#else + HANDLE hFile; /* handle to open file. */ + int fd = 0; + hFile = CreateFile((LPCTSTR) file_name, + GENERIC_READ | GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_DELETE, + NULL, CREATE_NEW, NULL, NULL); + + if (hFile == INVALID_HANDLE_VALUE) { + /* print the error message. */ + fprintf(stderr, "Filename::%s %s\n", + file_name, + error_message(GetLastError())); + + return(NULL); + } + + /* get the file descriptor. */ + fd= _open_osfhandle((intptr_t)hFile, _O_RDWR | _O_BINARY); + file = fdopen(fd, "wb"); +#endif /* _WIN32 */ + + return(file); +} + +/* + Print the page type count of a tablespace. + @param [in] fil_out stream where the output goes. +*/ +void +print_summary( + FILE* fil_out) +{ + fprintf(fil_out, "\n================PAGE TYPE SUMMARY==============\n"); + fprintf(fil_out, "#PAGE_COUNT\tPAGE_TYPE"); + fprintf(fil_out, "\n===============================================\n"); + fprintf(fil_out, "%8d\tIndex page\n", + page_type.n_fil_page_index); + fprintf(fil_out, "%8d\tUndo log page\n", + page_type.n_fil_page_undo_log); + fprintf(fil_out, "%8d\tInode page\n", + page_type.n_fil_page_inode); + fprintf(fil_out, "%8d\tInsert buffer free list page\n", + page_type.n_fil_page_ibuf_free_list); + fprintf(fil_out, "%8d\tFreshly allocated page\n", + page_type.n_fil_page_type_allocated); + fprintf(fil_out, "%8d\tInsert buffer bitmap\n", + page_type.n_fil_page_ibuf_bitmap); + fprintf(fil_out, "%8d\tSystem page\n", + page_type.n_fil_page_type_sys); + fprintf(fil_out, "%8d\tTransaction system page\n", + page_type.n_fil_page_type_trx_sys); + fprintf(fil_out, "%8d\tFile Space Header\n", + page_type.n_fil_page_type_fsp_hdr); + fprintf(fil_out, "%8d\tExtent descriptor page\n", + page_type.n_fil_page_type_xdes); + fprintf(fil_out, "%8d\tBLOB page\n", + page_type.n_fil_page_type_blob); + fprintf(fil_out, "%8d\tCompressed BLOB page\n", + page_type.n_fil_page_type_zblob); + fprintf(fil_out, "%8d\tOther type of page\n", + page_type.n_fil_page_type_other); + fprintf(fil_out, "%8d\tPage compressed page\n", + page_type.n_fil_page_type_page_compressed); + fprintf(fil_out, "%8d\tPage compressed encrypted page\n", + page_type.n_fil_page_type_page_compressed_encrypted); + fprintf(fil_out, "\n===============================================\n"); + fprintf(fil_out, "Additional information:\n"); + fprintf(fil_out, "Undo page type: %d insert, %d update, %d other\n", + page_type.n_undo_insert, + page_type.n_undo_update, + page_type.n_undo_other); + fprintf(fil_out, "Undo page state: %d active, %d cached, %d to_free, %d" + " to_purge, %d prepared, %d other\n", + page_type.n_undo_state_active, + page_type.n_undo_state_cached, + page_type.n_undo_state_to_free, + page_type.n_undo_state_to_purge, + page_type.n_undo_state_prepared, + page_type.n_undo_state_other); + + fprintf(fil_out, "index_id\t#pages\t\t#leaf_pages\t#recs_per_page" + "\t#bytes_per_page\n"); + + for (std::map::const_iterator it = index_ids.begin(); + it != index_ids.end(); it++) { + const per_index_stats& index = it->second; + fprintf(fil_out, "%lld\t\t%lld\t\t%lld\t\t%lld\t\t%lld\n", + it->first, index.pages, index.leaf_pages, + index.total_n_recs / index.pages, + index.total_data_bytes / index.pages); + } + + fprintf(fil_out, "\n"); + fprintf(fil_out, "index_id\tpage_data_bytes_histgram(empty,...,oversized)\n"); + + for (std::map::const_iterator it = index_ids.begin(); + it != index_ids.end(); it++) { + fprintf(fil_out, "%lld\t", it->first); + const per_index_stats& index = it->second; + for (ulint i = 0; i < SIZE_RANGES_FOR_PAGE+2; i++) { + fprintf(fil_out, "\t%lld", index.pages_in_size_range[i]); + } + fprintf(fil_out, "\n"); + } + + if (do_leaf) { + print_leaf_stats(fil_out); + } +} + +/* command line argument for innochecksum tool. */ +static struct my_option innochecksum_options[] = { {"help", '?', "Displays this help and exits.", 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, {"info", 'I', "Synonym for --help.", @@ -221,864 +1157,664 @@ static struct my_option innochecksum_options[] = 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, {"verbose", 'v', "Verbose (prints progress every 5 seconds).", &verbose, &verbose, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, - {"debug", 'd', "Debug mode (prints checksums for each page, implies verbose).", - &debug, &debug, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, - {"skip_corrupt", 'u', "Skip corrupt pages.", - &skip_corrupt, &skip_corrupt, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, - {"count", 'c', "Print the count of pages in the file.", +#ifndef DBUG_OFF + {"debug", '#', "Output debug log. See " REFMAN "dbug-package.html", + &dbug_setting, &dbug_setting, 0, GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0}, +#endif /* !DBUG_OFF */ + {"count", 'c', "Print the count of pages in the file and exits.", &just_count, &just_count, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, {"start_page", 's', "Start on this page number (0 based).", - &start_page, &start_page, 0, GET_ULONG, REQUIRED_ARG, - 0, 0, (longlong) 2L*1024L*1024L*1024L, 0, 1, 0}, + &start_page, &start_page, 0, GET_ULL, REQUIRED_ARG, + 0, 0, ULLONG_MAX, 0, 1, 0}, {"end_page", 'e', "End at this page number (0 based).", - &end_page, &end_page, 0, GET_ULONG, REQUIRED_ARG, - 0, 0, (longlong) 2L*1024L*1024L*1024L, 0, 1, 0}, + &end_page, &end_page, 0, GET_ULL, REQUIRED_ARG, + 0, 0, ULLONG_MAX, 0, 1, 0}, {"page", 'p', "Check only this page (0 based).", - &do_page, &do_page, 0, GET_ULONG, REQUIRED_ARG, - 0, 0, (longlong) 2L*1024L*1024L*1024L, 0, 1, 0}, - {"per_page_details", 'i', "Print out per-page detail information.", - &per_page_details, &per_page_details, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0} - , - {"leaf", 'l', "Examine leaf index pages", + &do_page, &do_page, 0, GET_ULL, REQUIRED_ARG, + 0, 0, ULLONG_MAX, 0, 1, 0}, + {"strict-check", 'C', "Specify the strict checksum algorithm by the user.", + &strict_check, &strict_check, &innochecksum_algorithms_typelib, + GET_ENUM, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"no-check", 'n', "Ignore the checksum verification.", + &no_check, &no_check, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"allow-mismatches", 'a', "Maximum checksum mismatch allowed.", + &allow_mismatches, &allow_mismatches, 0, + GET_ULL, REQUIRED_ARG, 0, 0, ULLONG_MAX, 0, 1, 0}, + {"write", 'w', "Rewrite the checksum algorithm by the user.", + &write_check, &write_check, &innochecksum_algorithms_typelib, + GET_ENUM, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"page-type-summary", 'S', "Display a count of each page type " + "in a tablespace.", &page_type_summary, &page_type_summary, 0, + GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"page-type-dump", 'D', "Dump the page type info for each page in a " + "tablespace.", &page_dump_filename, &page_dump_filename, 0, + GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"per-page-details", 'i', "Print out per-page detail information.", + &per_page_details, &per_page_details, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"log", 'l', "log output.", + &log_filename, &log_filename, 0, + GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"leaf", 'f', "Examine leaf index pages", &do_leaf, &do_leaf, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, {"merge", 'm', "leaf page count if merge given number of consecutive pages", - &n_merge, &n_merge, 0, GET_ULONG, REQUIRED_ARG, - 0, 0, (longlong)10L, 0, 1, 0}, + &n_merge, &n_merge, 0, GET_ULONG, REQUIRED_ARG, 0, 0, (longlong)10L, 0, 1, 0}, + {0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0} }; +/* Print out the Innodb version and machine information. */ static void print_version(void) { - printf("%s Ver %s, for %s (%s)\n", - my_progname, INNODB_VERSION_STR, - SYSTEM_TYPE, MACHINE_TYPE); +#ifdef DBUG_OFF + printf("%s Ver %s, for %s (%s)\n", + my_progname, INNODB_VERSION_STR, + SYSTEM_TYPE, MACHINE_TYPE); +#else + printf("%s-debug Ver %s, for %s (%s)\n", + my_progname, INNODB_VERSION_STR, + SYSTEM_TYPE, MACHINE_TYPE); +#endif /* DBUG_OFF */ } static void usage(void) { - print_version(); - puts(ORACLE_WELCOME_COPYRIGHT_NOTICE("2000")); - printf("InnoDB offline file checksum utility.\n"); - printf("Usage: %s [-c] [-s ] [-e ] [-p ] [-v] [-d] \n", my_progname); - my_print_help(innochecksum_options); - my_print_variables(innochecksum_options); + print_version(); + puts(ORACLE_WELCOME_COPYRIGHT_NOTICE("2000")); + printf("InnoDB offline file checksum utility.\n"); + printf("Usage: %s [-c] [-s ] [-e ] " + "[-p ] [-i] [-v] [-a ] [-n] " + "[-C ] [-w ] [-S] [-D ] " + "[-l ] [-l] [-m ] \n", my_progname); + printf("See " REFMAN "innochecksum.html for usage hints.\n"); + my_print_help(innochecksum_options); + my_print_variables(innochecksum_options); } extern "C" my_bool innochecksum_get_one_option( -/*========================*/ - int optid, - const struct my_option *opt __attribute__((unused)), - char *argument __attribute__((unused))) + int optid, + const struct my_option *opt MY_ATTRIBUTE((unused)), + char *argument MY_ATTRIBUTE((unused))) { - switch (optid) { - case 'd': - verbose=1; /* debug implies verbose... */ - break; - case 'e': - use_end_page= 1; - break; - case 'p': - end_page= start_page= do_page; - use_end_page= 1; - do_one_page= 1; - break; - case 'V': - print_version(); - exit(0); - break; - case 'I': - case '?': - usage(); - exit(0); - break; - } - return 0; -} + switch (optid) { +#ifndef DBUG_OFF + case '#': + dbug_setting = argument + ? argument + : IF_WIN("d:O,innochecksum.trace", + "d:o,/tmp/innochecksum.trace"); + DBUG_PUSH(dbug_setting); + break; +#endif /* !DBUG_OFF */ + case 'e': + use_end_page = true; + break; + case 'p': + end_page = start_page = do_page; + use_end_page = true; + do_one_page = true; + break; + case 'V': + print_version(); + exit(EXIT_SUCCESS); + break; + case 'C': + strict_verify = true; + switch ((srv_checksum_algorithm_t) strict_check) { -static int get_options( -/*===================*/ - int *argc, - char ***argv) -{ - int ho_error; + case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32: + case SRV_CHECKSUM_ALGORITHM_CRC32: + srv_checksum_algorithm = + SRV_CHECKSUM_ALGORITHM_STRICT_CRC32; + break; - if ((ho_error=handle_options(argc, argv, innochecksum_options, innochecksum_get_one_option))) - exit(ho_error); + case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB: + case SRV_CHECKSUM_ALGORITHM_INNODB: + srv_checksum_algorithm = + SRV_CHECKSUM_ALGORITHM_STRICT_INNODB; + break; - /* The next arg must be the filename */ - if (!*argc) - { - usage(); - return 1; - } - return 0; -} /* get_options */ - -/*********************************************************************//** -Gets the file page type. -@return type; NOTE that if the type has not been written to page, the -return value not defined */ -ulint -fil_page_get_type( -/*==============*/ - uchar* page) /*!< in: file page */ -{ - return(mach_read_from_2(page + FIL_PAGE_TYPE)); -} - -/**************************************************************//** -Gets the index id field of a page. -@return index id */ -ib_uint64_t -btr_page_get_index_id( -/*==================*/ - uchar* page) /*!< in: index page */ -{ - return(mach_read_from_8(page + PAGE_HEADER + PAGE_INDEX_ID)); -} - -/********************************************************//** -Gets the next index page number. -@return next page number */ -ulint -btr_page_get_next( -/*==============*/ - const page_t* page) /*!< in: index page */ -{ - return(mach_read_from_4(page + FIL_PAGE_NEXT)); -} - -/********************************************************//** -Gets the previous index page number. -@return prev page number */ -ulint -btr_page_get_prev( -/*==============*/ - const page_t* page) /*!< in: index page */ -{ - return(mach_read_from_4(page + FIL_PAGE_PREV)); -} - -void -parse_page( -/*=======*/ - uchar* page, /* in: buffer page */ - uchar* xdes) /* in: extend descriptor page */ -{ - ib_uint64_t id; - ulint x; - ulint n_recs; - ulint page_no; - ulint left_page_no; - ulint right_page_no; - ulint data_bytes; - int is_leaf; - int size_range_id; - - switch (fil_page_get_type(page)) { - case FIL_PAGE_INDEX: - n_fil_page_index++; - id = btr_page_get_index_id(page); - n_recs = page_get_n_recs(page); - page_no = page_get_page_no(page); - left_page_no = btr_page_get_prev(page); - right_page_no = btr_page_get_next(page); - data_bytes = page_get_data_size(page); - is_leaf = page_is_leaf(page); - size_range_id = (data_bytes * SIZE_RANGES_FOR_PAGE - + logical_page_size - 1) / - logical_page_size; - if (size_range_id > SIZE_RANGES_FOR_PAGE + 1) { - /* data_bytes is bigger than logical_page_size */ - size_range_id = SIZE_RANGES_FOR_PAGE + 1; - } - if (per_page_details) { - printf("index " IB_ID_FMT " page " ULINTPF - " leaf %d n_recs " ULINTPF " data_bytes " ULINTPF - "\n", id, page_no, is_leaf, n_recs, data_bytes); - } - /* update per-index statistics */ - { - if (index_ids.count(id) == 0) { - index_ids[id] = per_index_stats(); - } - std::map::iterator it; - it = index_ids.find(id); - per_index_stats &index = (it->second); - uchar* des = xdes + XDES_ARR_OFFSET - + XDES_SIZE * ((page_no & (physical_page_size - 1)) - / FSP_EXTENT_SIZE); - if (xdes_get_bit(des, XDES_FREE_BIT, - page_no % FSP_EXTENT_SIZE)) { - index.free_pages++; - return; - } - index.pages++; - if (is_leaf) { - index.leaf_pages++; - if (data_bytes > index.max_data_size) { - index.max_data_size = data_bytes; - } - struct per_page_stats pp(n_recs, data_bytes, - left_page_no, right_page_no); - - index.leaves[page_no] = pp; - - if (left_page_no == ULINT32_UNDEFINED) { - index.first_leaf_page = page_no; - index.count++; - } - } - index.total_n_recs += n_recs; - index.total_data_bytes += data_bytes; - index.pages_in_size_range[size_range_id] ++; - } - - break; - case FIL_PAGE_UNDO_LOG: - if (per_page_details) { - printf("FIL_PAGE_UNDO_LOG\n"); - } - n_fil_page_undo_log++; - x = mach_read_from_2(page + TRX_UNDO_PAGE_HDR + - TRX_UNDO_PAGE_TYPE); - if (x == TRX_UNDO_INSERT) - n_undo_insert++; - else if (x == TRX_UNDO_UPDATE) - n_undo_update++; - else - n_undo_other++; - - x = mach_read_from_2(page + TRX_UNDO_SEG_HDR + TRX_UNDO_STATE); - switch (x) { - case TRX_UNDO_ACTIVE: n_undo_state_active++; break; - case TRX_UNDO_CACHED: n_undo_state_cached++; break; - case TRX_UNDO_TO_FREE: n_undo_state_to_free++; break; - case TRX_UNDO_TO_PURGE: n_undo_state_to_purge++; break; - case TRX_UNDO_PREPARED: n_undo_state_prepared++; break; - default: n_undo_state_other++; break; - } - break; - case FIL_PAGE_INODE: - if (per_page_details) { - printf("FIL_PAGE_INODE\n"); - } - n_fil_page_inode++; - break; - case FIL_PAGE_IBUF_FREE_LIST: - if (per_page_details) { - printf("FIL_PAGE_IBUF_FREE_LIST\n"); - } - n_fil_page_ibuf_free_list++; - break; - case FIL_PAGE_TYPE_ALLOCATED: - if (per_page_details) { - printf("FIL_PAGE_TYPE_ALLOCATED\n"); - } - n_fil_page_type_allocated++; - break; - case FIL_PAGE_IBUF_BITMAP: - if (per_page_details) { - printf("FIL_PAGE_IBUF_BITMAP\n"); - } - n_fil_page_ibuf_bitmap++; - break; - case FIL_PAGE_TYPE_SYS: - if (per_page_details) { - printf("FIL_PAGE_TYPE_SYS\n"); - } - n_fil_page_type_sys++; - break; - case FIL_PAGE_TYPE_TRX_SYS: - if (per_page_details) { - printf("FIL_PAGE_TYPE_TRX_SYS\n"); - } - n_fil_page_type_trx_sys++; - break; - case FIL_PAGE_TYPE_FSP_HDR: - if (per_page_details) { - printf("FIL_PAGE_TYPE_FSP_HDR\n"); - } - memcpy(xdes, page, physical_page_size); - n_fil_page_type_fsp_hdr++; - break; - case FIL_PAGE_TYPE_XDES: - if (per_page_details) { - printf("FIL_PAGE_TYPE_XDES\n"); - } - memcpy(xdes, page, physical_page_size); - n_fil_page_type_xdes++; - break; - case FIL_PAGE_TYPE_BLOB: - if (per_page_details) { - printf("FIL_PAGE_TYPE_BLOB\n"); - } - n_fil_page_type_blob++; - break; - case FIL_PAGE_TYPE_ZBLOB: - case FIL_PAGE_TYPE_ZBLOB2: - if (per_page_details) { - printf("FIL_PAGE_TYPE_ZBLOB/2\n"); - } - n_fil_page_type_zblob++; - break; - case FIL_PAGE_PAGE_COMPRESSED: - if (per_page_details) { - printf("FIL_PAGE_PAGE_COMPRESSED\n"); - } - n_fil_page_type_page_compressed++; - break; - case FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED: - if (per_page_details) { - printf("FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED\n"); - } - n_fil_page_type_page_compressed_encrypted++; - break; - default: - if (per_page_details) { - printf("FIL_PAGE_TYPE_OTHER\n"); - } - n_fil_page_type_other++; - } -} - -void print_index_leaf_stats(unsigned long long id, const per_index_stats& index) -{ - ulint page_no = index.first_leaf_page; - std::map::const_iterator it_page = index.leaves.find(page_no); - printf("\nindex: %llu leaf page stats: n_pages = %llu\n", - id, index.leaf_pages); - printf("page_no\tdata_size\tn_recs\n"); - while (it_page != index.leaves.end()) { - const per_page_stats& stat = it_page->second; - printf(ULINTPF "\t" ULINTPF "\t" ULINTPF "\n", - it_page->first, stat.data_size, stat.n_recs); - page_no = stat.right_page_no; - it_page = index.leaves.find(page_no); - } -} - -void defrag_analysis(unsigned long long id, const per_index_stats& index) -{ - // TODO: make it work for compressed pages too - std::map::const_iterator it = index.leaves.find(index.first_leaf_page); - ulint n_pages = 0; - ulint n_leaf_pages = 0; - while (it != index.leaves.end()) { - ulint data_size_total = 0; - for (ulong i = 0; i < n_merge; i++) { - const per_page_stats& stat = it->second; - n_leaf_pages ++; - data_size_total += stat.data_size; - it = index.leaves.find(stat.right_page_no); - if (it == index.leaves.end()) { - break; - } - } - if (index.max_data_size) { - n_pages += data_size_total / index.max_data_size; - if (data_size_total % index.max_data_size != 0) { - n_pages += 1; - } - } - } - if (!n_leaf_pages) { - n_leaf_pages=1; - } - printf("count = " ULINTPF " free = " ULINTPF "\n", - index.count, index.free_pages); - if (n_leaf_pages) { - printf("%llu\t\t%llu\t\t" - ULINTPF "\t\t%lu\t\t" ULINTPF "\t\t%.2f\t" ULINTPF "\n", - id, index.leaf_pages, n_leaf_pages, n_merge, n_pages, - 1.0 - (double)n_pages / (double)n_leaf_pages, index.max_data_size); - } -} - -void print_leaf_stats() -{ - printf("\n**************************************************\n"); - printf("index_id\t#leaf_pages\t#actual_leaf_pages\tn_merge\t" - "#leaf_after_merge\tdefrag\n"); - for (std::map::const_iterator it = index_ids.begin(); it != index_ids.end(); it++) { - const per_index_stats& index = it->second; - if (verbose) { - print_index_leaf_stats(it->first, index); - } - if (n_merge) { - defrag_analysis(it->first, index); - } - } -} - -void -print_stats() -/*========*/ -{ - unsigned long long i; - - printf("%d\tbad checksum\n", n_bad_checksum); - printf("%d\tFIL_PAGE_INDEX\n", n_fil_page_index); - printf("%d\tFIL_PAGE_UNDO_LOG\n", n_fil_page_undo_log); - printf("%d\tFIL_PAGE_INODE\n", n_fil_page_inode); - printf("%d\tFIL_PAGE_IBUF_FREE_LIST\n", n_fil_page_ibuf_free_list); - printf("%d\tFIL_PAGE_TYPE_ALLOCATED\n", n_fil_page_type_allocated); - printf("%d\tFIL_PAGE_IBUF_BITMAP\n", n_fil_page_ibuf_bitmap); - printf("%d\tFIL_PAGE_TYPE_SYS\n", n_fil_page_type_sys); - printf("%d\tFIL_PAGE_TYPE_TRX_SYS\n", n_fil_page_type_trx_sys); - printf("%d\tFIL_PAGE_TYPE_FSP_HDR\n", n_fil_page_type_fsp_hdr); - printf("%d\tFIL_PAGE_TYPE_XDES\n", n_fil_page_type_xdes); - printf("%d\tFIL_PAGE_TYPE_BLOB\n", n_fil_page_type_blob); - printf("%d\tFIL_PAGE_TYPE_ZBLOB\n", n_fil_page_type_zblob); - printf("%d\tFIL_PAGE_PAGE_COMPRESSED\n", n_fil_page_type_page_compressed); - printf("%d\tFIL_PAGE_PAGE_COMPRESSED_ENCRYPTED\n", n_fil_page_type_page_compressed_encrypted); - printf("%d\tother\n", n_fil_page_type_other); - printf("%d\tmax index_id\n", n_fil_page_max_index_id); - printf("undo type: %d insert, %d update, %d other\n", - n_undo_insert, n_undo_update, n_undo_other); - printf("undo state: %d active, %d cached, %d to_free, %d to_purge," - " %d prepared, %d other\n", n_undo_state_active, - n_undo_state_cached, n_undo_state_to_free, - n_undo_state_to_purge, n_undo_state_prepared, - n_undo_state_other); - - printf("index_id\t#pages\t\t#leaf_pages\t#recs_per_page" - "\t#bytes_per_page\n"); - for (std::map::const_iterator it = index_ids.begin(); it != index_ids.end(); it++) { - const per_index_stats& index = it->second; - ulonglong recs_per_page = index.total_n_recs; - ulonglong bytes_per_page = index.total_data_bytes; - if (index.total_n_recs && index.pages) { - recs_per_page = index.total_n_recs / index.pages; - } - if (index.total_data_bytes && index.pages) { - bytes_per_page = index.total_data_bytes / index.pages; - } - printf("%llu\t\t%llu\t\t%llu\t\t%llu\t\t%llu\n", - it->first, index.pages, index.leaf_pages, - recs_per_page, - bytes_per_page); - } - printf("\n"); - printf("index_id\tpage_data_bytes_histgram(empty,...,oversized)\n"); - for (std::map::const_iterator it = index_ids.begin(); it != index_ids.end(); it++) { - printf("%llu\t", it->first); - const per_index_stats& index = it->second; - for (i = 0; i < SIZE_RANGES_FOR_PAGE+2; i++) { - printf("\t%llu", index.pages_in_size_range[i]); - } - printf("\n"); - } - if (do_leaf) { - print_leaf_stats(); - } -} - -int main(int argc, char **argv) -{ - FILE* f; /* our input file */ - char* filename; /* our input filename. */ - unsigned char *big_buf= 0, *buf; - unsigned char *big_xdes= 0, *xdes; - ulong bytes; /* bytes read count */ - ulint ct; /* current page number (0 based) */ - time_t now; /* current time */ - time_t lastt; /* last time */ - ulint oldcsum, oldcsumfield, csum, csumfield, crc32, logseq, logseqfield; - /* ulints for checksum storage */ - unsigned long long int size; /* size of file (has to be 64 bits) */ - ulint pages; /* number of pages in file */ - long long offset= 0; - int fd; - - printf("InnoDB offline file checksum utility.\n"); - - ut_crc32_init(); - - MY_INIT(argv[0]); - - if (get_options(&argc,&argv)) - exit(1); - - if (verbose) - my_print_variables(innochecksum_options); - - /* The file name is not optional */ - filename = *argv; - if (*filename == '\0') - { - fprintf(stderr, "Error; File name missing\n"); - goto error_out; - } - -#ifdef _WIN32 - /* Switch off OS file buffering for the file. */ - - HANDLE h = CreateFile(filename, GENERIC_READ, - FILE_SHARE_READ|FILE_SHARE_WRITE, 0, - OPEN_EXISTING, FILE_FLAG_NO_BUFFERING, 0); - - if (!h) - { - fprintf(stderr, "Error; cant open file\n"); - goto error; - } - - if (!GetFileSizeEx(h, (LARGE_INTEGER *)&size)) - { - fprintf(stderr, "Error; GetFileSize() failed\n"); - goto error; - } - - fd = _open_osfhandle ((intptr_t) h, _O_RDONLY); - if (fd < 0) - { - fprintf(stderr, "Error; _open_osfhandle() failed\n"); - goto error; - } - - f = _fdopen(fd, "rb"); - if (!f) - { - fprintf(stderr, "Error; fdopen() failed\n"); - goto error; - } - - /* - Disable stdio buffering (FILE_FLAG_NO_BUFFERING requires properly IO buffers - which stdio does not guarantee. - */ - setvbuf(f, NULL, _IONBF, 0); - -#else - struct stat st; - /* stat the file to get size and page count */ - if (stat(filename, &st)) - { - fprintf(stderr, "Error; %s cannot be found\n", filename); - goto error_out; - } - size= st.st_size; - - /* Open the file for reading */ - f= fopen(filename, "rb"); -#endif - - if (f == NULL) - { - fprintf(stderr, "Error; %s cannot be opened", filename); - perror(" "); - goto error_out; - } - - big_buf = (unsigned char *)malloc(2 * UNIV_PAGE_SIZE_MAX); - if (big_buf == NULL) - { - fprintf(stderr, "Error; failed to allocate memory\n"); - perror(""); - goto error_f; - } - - /* Make sure the page is aligned */ - buf = (unsigned char*)ut_align_down(big_buf - + UNIV_PAGE_SIZE_MAX, UNIV_PAGE_SIZE_MAX); - - big_xdes = (unsigned char *)malloc(2 * UNIV_PAGE_SIZE_MAX); - if (big_xdes == NULL) - { - fprintf(stderr, "Error; failed to allocate memory\n"); - perror(""); - goto error_big_buf; - } - - /* Make sure the page is aligned */ - xdes = (unsigned char*)ut_align_down(big_xdes - + UNIV_PAGE_SIZE_MAX, UNIV_PAGE_SIZE_MAX); - - - if (!get_page_size(f, buf, &logical_page_size, &physical_page_size)) - goto error; - - if (compressed) - { - printf("Table is compressed\n"); - printf("Key block size is %lu\n", physical_page_size); - } - else - { - printf("Table is uncompressed\n"); - printf("Page size is %lu\n", physical_page_size); - } - - pages= (ulint) (size / physical_page_size); - - if (just_count) - { - if (verbose) - printf("Number of pages: "); - printf(ULINTPF "\n", pages); - goto ok; - } - else if (verbose) - { - printf("file %s = %llu bytes (" ULINTPF " pages)...\n", - filename, size, pages); - if (do_one_page) - printf("InnoChecksum; checking page %lu\n", do_page); - else - printf("InnoChecksum; checking pages in range %lu to %lu\n", start_page, use_end_page ? end_page : (pages - 1)); - } - -#ifdef UNIV_LINUX - if (posix_fadvise(fileno(f), 0, 0, POSIX_FADV_SEQUENTIAL) || - posix_fadvise(fileno(f), 0, 0, POSIX_FADV_NOREUSE)) - { - perror("posix_fadvise failed"); - } -#endif - - /* seek to the necessary position */ - if (start_page) - { - fd= fileno(f); - if (!fd) - { - perror("Error; Unable to obtain file descriptor number"); - goto error; - } - - offset= (longlong)start_page * (longlong)physical_page_size; -#ifdef _WIN32 - if (_lseeki64(fd, offset, SEEK_SET) != offset) -#else - if (lseek(fd, offset, SEEK_SET) != offset) -#endif - { - perror("Error; Unable to seek to necessary offset"); - goto error; - } - } - - /* main checksumming loop */ - ct= start_page; - lastt= 0; - while (!feof(f)) - { - int page_ok = 1; - - bytes= fread(buf, 1, physical_page_size, f); - - if (!bytes && feof(f)) - goto ok; - - if (ferror(f)) - { - fprintf(stderr, "Error reading %lu bytes", physical_page_size); - perror(" "); - goto error; - } - - ulint page_type = mach_read_from_2(buf+FIL_PAGE_TYPE); - ulint key_version = mach_read_from_4(buf + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); - - if (key_version && page_type != FIL_PAGE_PAGE_COMPRESSED) { - encrypted = true; - } else { - encrypted = false; - } - - ulint comp_method = 0; - - if (encrypted) { - comp_method = mach_read_from_2(buf+FIL_PAGE_DATA+FIL_PAGE_COMPRESSED_SIZE); - } else { - comp_method = mach_read_from_8(buf+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); - } - - ulint comp_size = mach_read_from_2(buf+FIL_PAGE_DATA); - ib_uint32_t encryption_checksum = mach_read_from_4(buf+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4); - - - if (page_type == FIL_PAGE_PAGE_COMPRESSED) { - /* Page compressed tables do not have any checksum */ - if (debug) - fprintf(stderr, "Page " ULINTPF - " page compressed with method %s real_size " ULINTPF "\n", ct, - fil_get_compression_alg_name(comp_method), comp_size); - page_ok = 1; - } else if (compressed) { - /* compressed pages */ - ulint crccsum = page_zip_calc_checksum(buf, physical_page_size, SRV_CHECKSUM_ALGORITHM_CRC32); - ulint icsum = page_zip_calc_checksum(buf, physical_page_size, SRV_CHECKSUM_ALGORITHM_INNODB); - - if (debug) { - if (key_version != 0) { - fprintf(stderr, - "Page " ULINTPF - " encrypted key_version " ULINTPF - " calculated = " ULINTPF "; crc32 = " ULINTPF - "; recorded = %u\n", - ct, key_version, icsum, crccsum, encryption_checksum); - } - } - - if (encrypted) { - if (encryption_checksum != 0 && crccsum != encryption_checksum && icsum != encryption_checksum) { - if (debug) - fprintf(stderr, "page " ULINTPF - ": compressed: calculated = " ULINTPF - "; crc32 = " ULINTPF "; recorded = %u\n", - ct, icsum, crccsum, encryption_checksum); - fprintf(stderr, "Fail; page " ULINTPF - " invalid (fails compressed page checksum).\n", ct); - } - } else { - if (!page_zip_verify_checksum(buf, physical_page_size)) { - fprintf(stderr, "Fail; page " ULINTPF - " invalid (fails compressed page checksum).\n", ct); - if (!skip_corrupt) - goto error; - page_ok = 0; - } + case SRV_CHECKSUM_ALGORITHM_STRICT_NONE: + case SRV_CHECKSUM_ALGORITHM_NONE: + srv_checksum_algorithm = + SRV_CHECKSUM_ALGORITHM_STRICT_NONE; + break; + default: + return(true); + } + break; + case 'n': + no_check = true; + break; + case 'a': + case 'S': + break; + case 'w': + do_write = true; + break; + case 'D': + page_type_dump = true; + break; + case 'l': + is_log_enabled = true; + break; + case 'I': + case '?': + usage(); + my_end(0); + exit(EXIT_SUCCESS); + break; } - } else { - if (key_version != 0) { - /* Encrypted page */ - if (debug) { - if (page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { - fprintf(stderr, - "Page " ULINTPF - " page compressed with method %s real_size " ULINTPF - " and encrypted key_version " ULINTPF " checksum %u\n", - ct, fil_get_compression_alg_name(comp_method), comp_size, key_version, encryption_checksum); - } else { - fprintf(stderr, - "Page " ULINTPF - " encrypted key_version " ULINTPF " checksum %u\n", - ct, key_version, encryption_checksum); - } - } - } - /* Page compressed tables do not contain FIL tailer */ - if (page_type != FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED && page_type != FIL_PAGE_PAGE_COMPRESSED) { - /* check the "stored log sequence numbers" */ - logseq= mach_read_from_4(buf + FIL_PAGE_LSN + 4); - logseqfield= mach_read_from_4(buf + logical_page_size - FIL_PAGE_END_LSN_OLD_CHKSUM + 4); - if (debug) - printf("page " ULINTPF - ": log sequence number: first = " ULINTPF - "; second = " ULINTPF "\n", - ct, logseq, logseqfield); - if (logseq != logseqfield) - { - fprintf(stderr, "Fail; page " ULINTPF - " invalid (fails log sequence number check)\n", ct); - if (!skip_corrupt) - goto error; - page_ok = 0; - } - - /* check old method of checksumming */ - oldcsum= buf_calc_page_old_checksum(buf); - oldcsumfield= mach_read_from_4(buf + logical_page_size - FIL_PAGE_END_LSN_OLD_CHKSUM); - if (debug) - printf("page " ULINTPF - ": old style: calculated = " ULINTPF - "; recorded = " ULINTPF "\n", - ct, oldcsum, oldcsumfield); - if (oldcsumfield != mach_read_from_4(buf + FIL_PAGE_LSN) && oldcsumfield != oldcsum) - { - fprintf(stderr, "Fail; page " ULINTPF - " invalid (fails old style checksum)\n", ct); - if (!skip_corrupt) - goto error; - page_ok = 0; - } - } - - /* now check the new method */ - csum= buf_calc_page_new_checksum(buf); - crc32= buf_calc_page_crc32(buf); - csumfield= mach_read_from_4(buf + FIL_PAGE_SPACE_OR_CHKSUM); - - if (key_version) - csumfield = encryption_checksum; - - if (debug) - printf("page " ULINTPF - ": new style: calculated = " ULINTPF - "; crc32 = " ULINTPF "; recorded = " ULINTPF "\n", - ct, csum, crc32, csumfield); - if (csumfield != 0 && crc32 != csumfield && csum != csumfield) - { - fprintf(stderr, "Fail; page " ULINTPF - " invalid (fails innodb and crc32 checksum)\n", ct); - if (!skip_corrupt) - goto error; - page_ok = 0; - } - } - /* end if this was the last page we were supposed to check */ - if (use_end_page && (ct >= end_page)) - goto ok; - - if (per_page_details) - { - printf("page " ULINTPF " ", ct); - } - - /* do counter increase and progress printing */ - ct++; - - if (!page_ok) - { - if (per_page_details) - { - printf("BAD_CHECKSUM\n"); - } - n_bad_checksum++; - continue; - } - - /* Can't parse compressed or/and encrypted pages */ - if (page_type != FIL_PAGE_PAGE_COMPRESSED && !encrypted) { - parse_page(buf, xdes); - } - - if (verbose) - { - if (ct % 64 == 0) - { - now= time(0); - if (!lastt) lastt= now; - if (now - lastt >= 1) - { - printf("page " ULINTPF " okay: %.3f%% done\n", - (ct - 1), (float) ct / pages * 100); - lastt= now; - } - } - } - } - -ok: - if (!just_count) - print_stats(); - free(big_xdes); - free(big_buf); - fclose(f); - my_end(0); - exit(0); - -error: - free(big_xdes); -error_big_buf: - free(big_buf); -error_f: - fclose(f); -error_out: - my_end(0); - exit(1); + return(false); +} + +static +bool +get_options( + int *argc, + char ***argv) +{ + if (handle_options(argc, argv, innochecksum_options, + innochecksum_get_one_option)) + exit(true); + + /* The next arg must be the filename */ + if (!*argc) { + usage(); + return (true); + } + + return (false); +} + +/** Check from page 0 if table is encrypted. */ +static +bool check_encryption( + const char* filename, + ulint zip_size, + byte * page) +{ + ulint offset = (FSP_HEADER_OFFSET + (XDES_ARR_OFFSET + XDES_SIZE * + (zip_size ? zip_size : UNIV_PAGE_SIZE) / FSP_EXTENT_SIZE)); + + if (memcmp(page + offset, CRYPT_MAGIC, MAGIC_SZ) != 0) { + return false; + } + + ulint type = mach_read_from_1(page + offset + MAGIC_SZ + 0); + + if (! (type == CRYPT_SCHEME_UNENCRYPTED || + type == CRYPT_SCHEME_1)) { + return false; + } + + ulint iv_length = mach_read_from_1(page + offset + MAGIC_SZ + 1); + + if (iv_length != CRYPT_SCHEME_1_IV_LEN) { + return false; + } + + uint min_key_version = mach_read_from_4 + (page + offset + MAGIC_SZ + 2 + iv_length); + + uint key_id = mach_read_from_4 + (page + offset + MAGIC_SZ + 2 + iv_length + 4); + + if (type == CRYPT_SCHEME_1) { + if (is_log_enabled) { + fprintf(log_file,"Tablespace %s encrypted key_version %u key_id %u\n", + filename, min_key_version, key_id); + } + } + + return (type == CRYPT_SCHEME_1); +} + +int main( + int argc, + char **argv) +{ + /* our input file. */ + FILE* fil_in = NULL; + /* our input filename. */ + char* filename; + /* Buffer to store pages read. */ + byte* buf = NULL; + byte* xdes = NULL; + /* bytes read count */ + ulong bytes; + /* current time */ + time_t now; + /* last time */ + time_t lastt; + /* stat, to get file size. */ +#ifdef _WIN32 + struct _stat64 st; +#else + struct stat st; +#endif /* _WIN32 */ + + int exit_status = 0; + + /* size of file (has to be 64 bits) */ + unsigned long long int size = 0; + /* number of pages in file */ + ulint pages; + + off_t offset = 0; + /* count the no. of page corrupted. */ + ulint mismatch_count = 0; + /* Variable to ack the page is corrupted or not. */ + bool is_corrupted = false; + + bool partial_page_read = false; + /* Enabled when read from stdin is done. */ + bool read_from_stdin = false; + FILE* fil_page_type = NULL; + fpos_t pos; + + /* Use to check the space id of given file. If space_id is zero, + then check whether page is doublewrite buffer.*/ + ulint space_id = 0UL; + /* enable when space_id of given file is zero. */ + bool is_system_tablespace = false; + + ut_crc32_init(); + MY_INIT(argv[0]); + DBUG_ENTER("main"); + DBUG_PROCESS(argv[0]); + + if (get_options(&argc,&argv)) { + exit_status = 1; + goto my_exit; + } + + if (strict_verify && no_check) { + fprintf(stderr, "Error: --strict-check option cannot be used " + "together with --no-check option.\n"); + exit_status = 1; + goto my_exit; + } + + if (no_check && !do_write) { + fprintf(stderr, "Error: --no-check must be associated with " + "--write option.\n"); + exit_status = 1; + goto my_exit; + } + + if (page_type_dump) { + fil_page_type = create_file(page_dump_filename); + if (!fil_page_type) { + exit_status = 1; + goto my_exit; + } + } + + if (is_log_enabled) { + log_file = create_file(log_filename); + if (!log_file) { + exit_status = 1; + goto my_exit; + } + fprintf(log_file, "InnoDB File Checksum Utility.\n"); + } + + if (verbose) { + my_print_variables(innochecksum_options); + } + + + buf = (byte*) malloc(UNIV_PAGE_SIZE_MAX * 2); + xdes = (byte*)malloc(UNIV_PAGE_SIZE_MAX * 2); + + /* The file name is not optional. */ + for (int i = 0; i < argc; ++i) { + /* Reset parameters for each file. */ + filename = argv[i]; + memset(&page_type, 0, sizeof(innodb_page_type)); + is_corrupted = false; + partial_page_read = false; + skip_page = false; + + if (is_log_enabled) { + fprintf(log_file, "Filename = %s\n", filename); + } + + if (*filename == '-') { + /* read from stdin. */ + fil_in = stdin; + read_from_stdin = true; + + } + + /* stat the file to get size and page count. */ + if (!read_from_stdin && +#ifdef _WIN32 + _stat64(filename, &st)) { +#else + stat(filename, &st)) { +#endif /* _WIN32 */ + fprintf(stderr, "Error: %s cannot be found\n", + filename); + + exit_status = 1; + goto my_exit; + } + + if (!read_from_stdin) { + size = st.st_size; + fil_in = open_file(filename); + /*If fil_in is NULL, terminate as some error encountered */ + if(fil_in == NULL) { + exit_status = 1; + goto my_exit; + } + /* Save the current file pointer in pos variable.*/ + if (0 != fgetpos(fil_in, &pos)) { + perror("fgetpos"); + exit_status = 1; + goto my_exit; + } + } + + /* Read the minimum page size. */ + bytes = ulong(fread(buf, 1, UNIV_ZIP_SIZE_MIN, fil_in)); + partial_page_read = true; + + if (bytes != UNIV_ZIP_SIZE_MIN) { + fprintf(stderr, "Error: Was not able to read the " + "minimum page size "); + fprintf(stderr, "of %d bytes. Bytes read was %lu\n", + UNIV_ZIP_SIZE_MIN, bytes); + + exit_status = 1; + goto my_exit; + } + + /* enable variable is_system_tablespace when space_id of given + file is zero. Use to skip the checksum verification and rewrite + for doublewrite pages. */ + is_system_tablespace = (!memcmp(&space_id, buf + + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID, 4)) + ? true : false; + + /* Determine page size, zip_size and page compression + from fsp_flags and encryption metadata from page 0 */ + ulint flags = mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + buf); + ulint page_size = fsp_flags_get_page_size(flags); + ulint zip_size = fsp_flags_get_zip_size(flags); + logical_page_size = zip_size; + physical_page_size = page_size; + srv_page_size = page_size; + bool is_compressed = FSP_FLAGS_HAS_PAGE_COMPRESSION(flags); + + if (page_size > UNIV_ZIP_SIZE_MIN) { + /* Read rest of the page 0 to determine crypt_data */ + bytes = ulong(read_file(buf, partial_page_read, page_size, fil_in)); + + if (bytes != page_size) { + fprintf(stderr, "Error: Was not able to read the " + "rest of the page "); + fprintf(stderr, "of %lu bytes. Bytes read was %lu\n", + page_size - UNIV_ZIP_SIZE_MIN, bytes); + + exit_status = 1; + goto my_exit; + } + partial_page_read = false; + } + + /* Now that we have full page 0 in buffer, check encryption */ + bool is_encrypted = check_encryption(filename, zip_size, buf); + + pages = (ulint) (size / page_size); + + if (just_count) { + if (read_from_stdin) { + fprintf(stderr, "Number of pages:" ULINTPF "\n", pages); + } else { + printf("Number of pages:" ULINTPF "\n", pages); + } + continue; + } else if (verbose && !read_from_stdin) { + if (is_log_enabled) { + fprintf(log_file, "file %s = %llu bytes " + "(" ULINTPF " pages)\n", filename, size, pages); + if (do_one_page) { + fprintf(log_file, "Innochecksum: " + "checking page %" PRIuMAX "\n", + do_page); + } + } + } else { + if (is_log_enabled) { + fprintf(log_file, "Innochecksum: checking " + "pages in range %" PRIuMAX " to %" PRIuMAX "\n", + start_page, use_end_page ? + end_page : (pages - 1)); + } + } + + /* seek to the necessary position */ + if (start_page) { + if (!read_from_stdin) { + /* If read is not from stdin, we can use + fseeko() to position the file pointer to + the desired page. */ + partial_page_read = false; + + offset = (off_t) start_page + * (off_t) page_size; +#ifdef _WIN32 + if (_fseeki64(fil_in, offset, SEEK_SET)) { +#else + if (fseeko(fil_in, offset, SEEK_SET)) { +#endif /* _WIN32 */ + perror("Error: Unable to seek to " + "necessary offset"); + + exit_status = 1; + goto my_exit; + } + /* Save the current file pointer in + pos variable. */ + if (0 != fgetpos(fil_in, &pos)) { + perror("fgetpos"); + + exit_status = 1; + goto my_exit; + } + } else { + + ulong count = 0; + + while (!feof(fil_in)) { + if (start_page == count) { + break; + } + /* We read a part of page to find the + minimum page size. We cannot reset + the file pointer to the beginning of + the page if we are reading from stdin + (fseeko() on stdin doesn't work). So + read only the remaining part of page, + if partial_page_read is enable. */ + bytes = read_file(buf, + partial_page_read, + static_cast( + page_size), + fil_in); + + partial_page_read = false; + count++; + + if (!bytes || feof(fil_in)) { + fprintf(stderr, "Error: Unable " + "to seek to necessary " + "offset"); + + exit_status = 1; + goto my_exit; + } + } + } + } + + if (page_type_dump) { + fprintf(fil_page_type, + "\n\nFilename::%s\n", filename); + fprintf(fil_page_type, + "========================================" + "======================================\n"); + fprintf(fil_page_type, + "\tPAGE_NO\t\t|\t\tPAGE_TYPE\t\t" + "\t|\tEXTRA INFO\n"); + fprintf(fil_page_type, + "========================================" + "======================================\n"); + } + + /* main checksumming loop */ + cur_page_num = start_page; + lastt = 0; + while (!feof(fil_in)) { + + bytes = read_file(buf, partial_page_read, + static_cast( + page_size), fil_in); + partial_page_read = false; + + if (!bytes && feof(fil_in)) { + break; + } + + if (ferror(fil_in)) { + fprintf(stderr, "Error reading " ULINTPF " bytes", + page_size); + perror(" "); + + exit_status = 1; + goto my_exit; + } + + if (bytes != page_size) { + fprintf(stderr, "Error: bytes read (%lu) " + "doesn't match page size (" ULINTPF ")\n", + bytes, page_size); + exit_status = 1; + goto my_exit; + } + + if (is_system_tablespace) { + /* enable when page is double write buffer.*/ + skip_page = is_page_doublewritebuffer(buf); + } else { + skip_page = false; + } + + /* If no-check is enabled, skip the + checksum verification.*/ + if (!no_check) { + /* Checksum verification */ + if (!skip_page) { + is_corrupted = is_page_corrupted( + buf, page_size, zip_size, is_encrypted, is_compressed); + + if (is_corrupted) { + fprintf(stderr, "Fail: page " + "%" PRIuMAX " invalid\n", + cur_page_num); + + mismatch_count++; + + if(mismatch_count > allow_mismatches) { + fprintf(stderr, + "Exceeded the " + "maximum allowed " + "checksum mismatch " + "count::%" PRIuMAX "\n", + allow_mismatches); + + exit_status = 1; + goto my_exit; + } + } + } + } + + /* Rewrite checksum. Note that for encrypted and + page compressed tables this is not currently supported. */ + if (do_write && + !is_encrypted && + !is_compressed + && !write_file(filename, fil_in, buf, + zip_size != 0, &pos, + static_cast(page_size))) { + + exit_status = 1; + goto my_exit; + } + + /* end if this was the last page we were supposed to check */ + if (use_end_page && (cur_page_num >= end_page)) { + break; + } + + if (per_page_details) { + printf("page %ld ", cur_page_num); + } + + if (page_type_summary || page_type_dump) { + parse_page(buf, xdes, fil_page_type); + } + + /* do counter increase and progress printing */ + cur_page_num++; + if (verbose && !read_from_stdin) { + if ((cur_page_num % 64) == 0) { + now = time(0); + if (!lastt) { + lastt= now; + } + if (now - lastt >= 1 + && is_log_enabled) { + fprintf(log_file, "page %" PRIuMAX " " + "okay: %.3f%% done\n", + (cur_page_num - 1), + (float) cur_page_num / pages * 100); + lastt = now; + } + } + } + } + + if (!read_from_stdin) { + /* flcose() will flush the data and release the lock if + any acquired. */ + fclose(fil_in); + } + + /* Enabled for page type summary. */ + if (page_type_summary) { + if (!read_from_stdin) { + fprintf(stdout, "\nFile::%s",filename); + print_summary(stdout); + } else { + print_summary(stderr); + } + } + } + + if (is_log_enabled) { + fclose(log_file); + } + +my_exit: + if (buf) { + free(buf); + } + my_end(exit_status); + exit(exit_status); } diff --git a/mysql-test/suite/encryption/r/innochecksum.result b/mysql-test/suite/encryption/r/innochecksum.result index 955d150eb04..6ea54f3d053 100644 --- a/mysql-test/suite/encryption/r/innochecksum.result +++ b/mysql-test/suite/encryption/r/innochecksum.result @@ -7,6 +7,7 @@ CREATE TABLE t2 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB ROW_FOR CREATE TABLE t3 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB ROW_FORMAT=COMPRESSED ENCRYPTED=NO; CREATE TABLE t4 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB PAGE_COMPRESSED=1; CREATE TABLE t5 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB PAGE_COMPRESSED=1 ENCRYPTED=YES ENCRYPTION_KEY_ID=4; +CREATE TABLE t6 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB; # Write file to make mysql-test-run.pl expect the "crash", but don't # start it until it's told to # We give 30 seconds to do a clean shutdown because we do not want @@ -18,6 +19,27 @@ CREATE TABLE t5 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB PAGE_CO # Run innochecksum on t3 # Run innochecksum on t4 # Run innochecksum on t4 +# Run innochecksum on t5 +# Run innochecksum on t6 +# Backup tables before corrupting +# Corrupt FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION +# Run innochecksum on t2 +# Run innochecksum on t3 +# no encryption corrupting the field should not have effect +# Run innochecksum on t6 +# no encryption corrupting the field should not have effect +# Restore the original tables +# Corrupt FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION+4 (post encryption checksum) +# Run innochecksum on t2 +# Run innochecksum on t3 +# Run innochecksum on t6 +# no encryption corrupting the field should not have effect +# Restore the original tables +# Corrupt FIL_DATA+10 (data) +# Run innochecksum on t2 +# Run innochecksum on t3 +# Run innochecksum on t6 +# Restore the original tables # Write file to make mysql-test-run.pl start up the server again # Cleanup -DROP TABLE t1, t2, t3, t4, t5; +DROP TABLE t1, t2, t3, t4, t5, t6; diff --git a/mysql-test/suite/encryption/t/innochecksum.test b/mysql-test/suite/encryption/t/innochecksum.test index a176cad6c0e..74fe7bcf7a0 100644 --- a/mysql-test/suite/encryption/t/innochecksum.test +++ b/mysql-test/suite/encryption/t/innochecksum.test @@ -2,7 +2,7 @@ # MDEV-8773: InnoDB innochecksum does not work with encrypted or page compressed tables # -# Don't test under embedded +# Don't test under embedded as we restart server -- source include/not_embedded.inc # Require InnoDB -- source include/have_innodb.inc @@ -13,16 +13,12 @@ if (!$INNOCHECKSUM) { --die Need innochecksum binary } ---disable_query_log -let $innodb_compression_algorithm_orig=`SELECT @@innodb_compression_algorithm`; -let $innodb_file_format_orig = `SELECT @@innodb_file_format`; -let $innodb_file_per_table_orig = `SELECT @@innodb_file_per_table`; ---enable_query_log - +--disable_warnings SET GLOBAL innodb_file_format = `Barracuda`; SET GLOBAL innodb_file_per_table = ON; # zlib set global innodb_compression_algorithm = 1; +--enable_warnings --echo # Create and populate a tables CREATE TABLE t1 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=4; @@ -30,9 +26,11 @@ CREATE TABLE t2 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB ROW_FOR CREATE TABLE t3 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB ROW_FORMAT=COMPRESSED ENCRYPTED=NO; CREATE TABLE t4 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB PAGE_COMPRESSED=1; CREATE TABLE t5 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB PAGE_COMPRESSED=1 ENCRYPTED=YES ENCRYPTION_KEY_ID=4; +CREATE TABLE t6 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB; --disable_query_log --let $i = 1000 +begin; while ($i) { INSERT INTO t1 (b) VALUES (REPEAT('abcdefghijklmnopqrstuvwxyz', 100)); @@ -42,6 +40,8 @@ INSERT INTO t2 SELECT * FROM t1; INSERT INTO t3 SELECT * FROM t1; INSERT INTO t4 SELECT * FROM t1; INSERT INTO t5 SELECT * FROM t1; +INSERT INTO t6 SELECT * FROM t1; +commit; --enable_query_log let $MYSQLD_DATADIR=`select @@datadir`; @@ -50,6 +50,10 @@ let t2_IBD = $MYSQLD_DATADIR/test/t2.ibd; let t3_IBD = $MYSQLD_DATADIR/test/t3.ibd; let t4_IBD = $MYSQLD_DATADIR/test/t4.ibd; let t5_IBD = $MYSQLD_DATADIR/test/t5.ibd; +let t6_IBD = $MYSQLD_DATADIR/test/t6.ibd; + +let INNODB_PAGE_SIZE=`select @@innodb_page_size`; +let MYSQLD_DATADIR=`select @@datadir`; --echo # Write file to make mysql-test-run.pl expect the "crash", but don't --echo # start it until it's told to @@ -83,19 +87,198 @@ shutdown_server 30; --exec $INNOCHECKSUM $t4_IBD +--echo # Run innochecksum on t5 + +--exec $INNOCHECKSUM $t5_IBD + +--echo # Run innochecksum on t6 + +--exec $INNOCHECKSUM $t6_IBD + --enable_result_log +--echo # Backup tables before corrupting +--copy_file $MYSQLD_DATADIR/test/t1.ibd $MYSQLD_DATADIR/test/t1.ibd.backup +--copy_file $MYSQLD_DATADIR/test/t2.ibd $MYSQLD_DATADIR/test/t2.ibd.backup +--copy_file $MYSQLD_DATADIR/test/t3.ibd $MYSQLD_DATADIR/test/t3.ibd.backup +--copy_file $MYSQLD_DATADIR/test/t4.ibd $MYSQLD_DATADIR/test/t4.ibd.backup +--copy_file $MYSQLD_DATADIR/test/t5.ibd $MYSQLD_DATADIR/test/t5.ibd.backup +--copy_file $MYSQLD_DATADIR/test/t6.ibd $MYSQLD_DATADIR/test/t6.ibd.backup + +# +# MDEV-11939: innochecksum mistakes a file for an encrypted one +# + +--echo # Corrupt FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + +perl; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t1.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 26, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t2.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 26, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t3.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 26, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t6.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 26, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +EOF + +-- disable_result_log +--error 1 +--exec $INNOCHECKSUM $t1_IBD + +--echo # Run innochecksum on t2 + +--error 1 +--exec $INNOCHECKSUM $t2_IBD + +--echo # Run innochecksum on t3 +--echo # no encryption corrupting the field should not have effect +--exec $INNOCHECKSUM $t3_IBD + +--echo # Run innochecksum on t6 +--echo # no encryption corrupting the field should not have effect +--exec $INNOCHECKSUM $t6_IBD + +--enable_result_log + +--echo # Restore the original tables +--remove_file $MYSQLD_DATADIR/test/t1.ibd +--remove_file $MYSQLD_DATADIR/test/t2.ibd +--remove_file $MYSQLD_DATADIR/test/t3.ibd +--remove_file $MYSQLD_DATADIR/test/t4.ibd +--remove_file $MYSQLD_DATADIR/test/t5.ibd +--remove_file $MYSQLD_DATADIR/test/t6.ibd +--copy_file $MYSQLD_DATADIR/test/t1.ibd.backup $MYSQLD_DATADIR/test/t1.ibd +--copy_file $MYSQLD_DATADIR/test/t2.ibd.backup $MYSQLD_DATADIR/test/t2.ibd +--copy_file $MYSQLD_DATADIR/test/t3.ibd.backup $MYSQLD_DATADIR/test/t3.ibd +--copy_file $MYSQLD_DATADIR/test/t4.ibd.backup $MYSQLD_DATADIR/test/t4.ibd +--copy_file $MYSQLD_DATADIR/test/t5.ibd.backup $MYSQLD_DATADIR/test/t5.ibd +--copy_file $MYSQLD_DATADIR/test/t6.ibd.backup $MYSQLD_DATADIR/test/t6.ibd + +--echo # Corrupt FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION+4 (post encryption checksum) + +perl; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t1.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 30, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t2.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 30, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t3.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 30, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t6.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 30, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +EOF + +-- disable_result_log +--error 1 +--exec $INNOCHECKSUM $t1_IBD + +--echo # Run innochecksum on t2 +--error 1 +--exec $INNOCHECKSUM $t2_IBD + +--echo # Run innochecksum on t3 +--error 1 +--exec $INNOCHECKSUM $t3_IBD + +--echo # Run innochecksum on t6 +--echo # no encryption corrupting the field should not have effect +--exec $INNOCHECKSUM $t6_IBD + +--enable_result_log + +--echo # Restore the original tables +--remove_file $MYSQLD_DATADIR/test/t1.ibd +--remove_file $MYSQLD_DATADIR/test/t2.ibd +--remove_file $MYSQLD_DATADIR/test/t3.ibd +--remove_file $MYSQLD_DATADIR/test/t4.ibd +--remove_file $MYSQLD_DATADIR/test/t5.ibd +--remove_file $MYSQLD_DATADIR/test/t6.ibd +--copy_file $MYSQLD_DATADIR/test/t1.ibd.backup $MYSQLD_DATADIR/test/t1.ibd +--copy_file $MYSQLD_DATADIR/test/t2.ibd.backup $MYSQLD_DATADIR/test/t2.ibd +--copy_file $MYSQLD_DATADIR/test/t3.ibd.backup $MYSQLD_DATADIR/test/t3.ibd +--copy_file $MYSQLD_DATADIR/test/t4.ibd.backup $MYSQLD_DATADIR/test/t4.ibd +--copy_file $MYSQLD_DATADIR/test/t5.ibd.backup $MYSQLD_DATADIR/test/t5.ibd +--copy_file $MYSQLD_DATADIR/test/t6.ibd.backup $MYSQLD_DATADIR/test/t6.ibd + +--echo # Corrupt FIL_DATA+10 (data) + +perl; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t1.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 48, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t2.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 48, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t3.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 48, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t6.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 48, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +EOF + +-- disable_result_log +--error 1 +--exec $INNOCHECKSUM $t1_IBD + +--echo # Run innochecksum on t2 +--error 1 +--exec $INNOCHECKSUM $t2_IBD + +--echo # Run innochecksum on t3 +--error 1 +--exec $INNOCHECKSUM $t3_IBD + +--echo # Run innochecksum on t6 +--error 1 +--exec $INNOCHECKSUM $t6_IBD + +--enable_result_log + +--echo # Restore the original tables +--move_file $MYSQLD_DATADIR/test/t1.ibd.backup $MYSQLD_DATADIR/test/t1.ibd +--move_file $MYSQLD_DATADIR/test/t2.ibd.backup $MYSQLD_DATADIR/test/t2.ibd +--move_file $MYSQLD_DATADIR/test/t3.ibd.backup $MYSQLD_DATADIR/test/t3.ibd +--move_file $MYSQLD_DATADIR/test/t4.ibd.backup $MYSQLD_DATADIR/test/t4.ibd +--move_file $MYSQLD_DATADIR/test/t5.ibd.backup $MYSQLD_DATADIR/test/t5.ibd +--move_file $MYSQLD_DATADIR/test/t6.ibd.backup $MYSQLD_DATADIR/test/t6.ibd + --echo # Write file to make mysql-test-run.pl start up the server again --exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect --enable_reconnect --source include/wait_until_connected_again.inc --echo # Cleanup -DROP TABLE t1, t2, t3, t4, t5; - -# reset system ---disable_query_log -EVAL SET GLOBAL innodb_compression_algorithm = $innodb_compression_algorithm_orig; -EVAL SET GLOBAL innodb_file_per_table = $innodb_file_per_table_orig; -EVAL SET GLOBAL innodb_file_format = $innodb_file_format_orig; ---enable_query_log +DROP TABLE t1, t2, t3, t4, t5, t6; diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 334b2fbddb5..50d067b3f85 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -31,12 +31,16 @@ The database buffer buf_pool Created 11/5/1995 Heikki Tuuri *******************************************************/ +#include "univ.i" +#include "mach0data.h" #include "buf0buf.h" +#include #ifdef UNIV_NONINL #include "buf0buf.ic" #endif +#ifndef UNIV_INNOCHECKSUM #include "mem0mem.h" #include "btr0btr.h" #include "fil0fil.h" @@ -52,13 +56,15 @@ Created 11/5/1995 Heikki Tuuri #include "srv0srv.h" #include "dict0dict.h" #include "log0recv.h" -#include "page0zip.h" #include "srv0mon.h" -#include "buf0checksum.h" #ifdef HAVE_LIBNUMA #include #include #endif // HAVE_LIBNUMA +#endif /* !UNIV_INNOCHECKSUM */ +#include "page0zip.h" +#include "buf0checksum.h" +#ifndef UNIV_INNOCHECKSUM #include "fil0pagecompress.h" #include "ha_prototypes.h" #include "ut0byte.h" @@ -528,6 +534,7 @@ buf_block_alloc( return(block); } #endif /* !UNIV_HOTBACKUP */ +#endif /* !UNIV_INNOCHECKSUM */ /** Check if a page is all zeroes. @param[in] read_buf database page @@ -561,6 +568,17 @@ buf_page_is_checksum_valid_crc32( { ib_uint32_t crc32 = buf_calc_page_crc32(read_buf); +#ifdef UNIV_INNOCHECKSUM + if (log_file + && srv_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_STRICT_CRC32) { + fprintf(log_file, "page::%lu;" + " crc32 calculated = %u;" + " recorded checksum field1 = %lu recorded" + " checksum field2 =%lu\n", cur_page_num, + crc32, checksum_field1, checksum_field2); + } +#endif /* UNIV_INNOCHECKSUM */ + if (!(checksum_field1 == crc32 && checksum_field2 == crc32)) { DBUG_PRINT("buf_checksum", ("Page checksum crc32 not valid field1 " ULINTPF @@ -595,12 +613,45 @@ buf_page_is_checksum_valid_innodb( 2. Newer InnoDB versions store the old formula checksum (buf_calc_page_old_checksum()). */ + ulint old_checksum = buf_calc_page_old_checksum(read_buf); + ulint new_checksum = buf_calc_page_new_checksum(read_buf); + +#ifdef UNIV_INNOCHECKSUM + if (log_file + && srv_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_INNODB) { + fprintf(log_file, "page::%lu;" + " old style: calculated =" + " %lu; recorded = %lu\n", + cur_page_num, old_checksum, + checksum_field2); + fprintf(log_file, "page::%lu;" + " new style: calculated =" + " %lu; crc32 = %u; recorded = %lu\n", + cur_page_num, new_checksum, + buf_calc_page_crc32(read_buf), checksum_field1); + } + + if (log_file + && srv_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_STRICT_INNODB) { + fprintf(log_file, "page::%lu;" + " old style: calculated =" + " %lu; recorded checksum = %lu\n", + cur_page_num, old_checksum, + checksum_field2); + fprintf(log_file, "page::%lu;" + " new style: calculated =" + " %lu; recorded checksum = %lu\n", + cur_page_num, new_checksum, + checksum_field1); + } +#endif /* UNIV_INNOCHECKSUM */ + if (checksum_field2 != mach_read_from_4(read_buf + FIL_PAGE_LSN) - && checksum_field2 != buf_calc_page_old_checksum(read_buf)) { + && checksum_field2 != old_checksum) { DBUG_PRINT("buf_checksum", ("Page checksum innodb not valid field1 " ULINTPF " field2 " ULINTPF "crc32 " ULINTPF " lsn " ULINTPF ".", - checksum_field1, checksum_field2, buf_calc_page_old_checksum(read_buf), + checksum_field1, checksum_field2, old_checksum, mach_read_from_4(read_buf + FIL_PAGE_LSN))); return(false); @@ -612,11 +663,11 @@ buf_page_is_checksum_valid_innodb( (always equal to 0), to FIL_PAGE_SPACE_OR_CHKSUM */ if (checksum_field1 != 0 - && checksum_field1 != buf_calc_page_new_checksum(read_buf)) { + && checksum_field1 != new_checksum) { DBUG_PRINT("buf_checksum", ("Page checksum innodb not valid field1 " ULINTPF " field2 " ULINTPF "crc32 " ULINTPF " lsn " ULINTPF ".", - checksum_field1, checksum_field2, buf_calc_page_new_checksum(read_buf), + checksum_field1, checksum_field2, new_checksum, mach_read_from_4(read_buf + FIL_PAGE_LSN))); return(false); @@ -646,6 +697,18 @@ buf_page_is_checksum_valid_none( mach_read_from_4(read_buf + FIL_PAGE_LSN))); } +#ifdef UNIV_INNOCHECKSUM + if (log_file + && srv_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_STRICT_NONE) { + fprintf(log_file, + "page::%lu; none checksum: calculated" + " = %lu; recorded checksum_field1 = %lu" + " recorded checksum_field2 = %lu\n", + cur_page_num, BUF_NO_CHECKSUM_MAGIC, + checksum_field1, checksum_field2); + } +#endif /* UNIV_INNOCHECKSUM */ + return(checksum_field1 == checksum_field2 && checksum_field1 == BUF_NO_CHECKSUM_MAGIC); } @@ -662,14 +725,18 @@ buf_page_is_corrupted( bool check_lsn, const byte* read_buf, ulint zip_size, +#ifndef UNIV_INNOCHECKSUM const fil_space_t* space) +#else + const void* space) +#endif { - ulint checksum_field1; - ulint checksum_field2; - ulint space_id = mach_read_from_4( - read_buf + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); - ulint page_type = mach_read_from_2( - read_buf + FIL_PAGE_TYPE); + ulint checksum_field1 = 0; + ulint checksum_field2 = 0; +#ifndef UNIV_INNOCHECKSUM + ulint space_id = mach_read_from_4(read_buf + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); +#endif + ulint page_type = mach_read_from_2(read_buf + FIL_PAGE_TYPE); /* We can trust page type if page compression is set on tablespace flags because page compression flag means file must have been @@ -682,7 +749,10 @@ buf_page_is_corrupted( decompressed at this stage). */ if ((page_type == FIL_PAGE_PAGE_COMPRESSED || page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) - && space && FSP_FLAGS_HAS_PAGE_COMPRESSION(space->flags)) { +#ifndef UNIV_INNOCHECKSUM + && space && FSP_FLAGS_HAS_PAGE_COMPRESSION(space->flags) +#endif + ) { return (false); } @@ -693,16 +763,17 @@ buf_page_is_corrupted( /* Stored log sequence numbers at the start and the end of page do not match */ - +#ifndef UNIV_INNOCHECKSUM ib_logf(IB_LOG_LEVEL_INFO, "Log sequence number at the start %lu and the end %lu do not match.", mach_read_from_4(read_buf + FIL_PAGE_LSN + 4), mach_read_from_4(read_buf + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM + 4)); +#endif /* UNIV_INNOCHECKSUM */ return(true); } -#ifndef UNIV_HOTBACKUP +#if !defined(UNIV_HOTBACKUP) && !defined(UNIV_INNOCHECKSUM) if (check_lsn && recv_lsn_checks_on) { lsn_t current_lsn; @@ -742,7 +813,7 @@ buf_page_is_corrupted( } if (zip_size) { - return(!page_zip_verify_checksum(read_buf, zip_size)); + return(!page_zip_verify_checksum((const void *)read_buf, zip_size)); } checksum_field1 = mach_read_from_4( @@ -762,9 +833,10 @@ buf_page_is_corrupted( /* make sure that the page is really empty */ for (ulint i = 0; i < UNIV_PAGE_SIZE; i++) { if (read_buf[i] != 0) { +#ifndef UNIV_INNOCHECKSUM ib_logf(IB_LOG_LEVEL_INFO, "Checksum fields zero but page is not empty."); - +#endif return(true); } } @@ -774,7 +846,9 @@ buf_page_is_corrupted( DBUG_EXECUTE_IF("buf_page_is_corrupt_failure", return(true); ); +#ifndef UNIV_INNOCHECKSUM ulint page_no = mach_read_from_4(read_buf + FIL_PAGE_OFFSET); +#endif const srv_checksum_algorithm_t curr_algo = static_cast(srv_checksum_algorithm); @@ -792,12 +866,32 @@ buf_page_is_corrupted( checksum_field1, checksum_field2)) { if (curr_algo == SRV_CHECKSUM_ALGORITHM_STRICT_CRC32) { +#ifndef UNIV_INNOCHECKSUM page_warn_strict_checksum( curr_algo, SRV_CHECKSUM_ALGORITHM_NONE, space_id, page_no); +#endif /* !UNIV_INNOCHECKSUM */ } +#ifdef UNIV_INNOCHECKSUM + if (log_file) { + fprintf(log_file, "page::" ULINTPF ";" + " old style: calculated = " ULINTPF ";" + " recorded = " ULINTPF "\n", + cur_page_num, + buf_calc_page_old_checksum(read_buf), + checksum_field2); + fprintf(log_file, "page::" ULINTPF ";" + " new style: calculated = " ULINTPF ";" + " crc32 = %u; recorded = " ULINTPF "\n", + cur_page_num, + buf_calc_page_new_checksum(read_buf), + buf_calc_page_crc32(read_buf), + checksum_field1); + } +#endif /* UNIV_INNOCHECKSUM */ + return(false); } @@ -805,15 +899,24 @@ buf_page_is_corrupted( checksum_field1, checksum_field2)) { if (curr_algo == SRV_CHECKSUM_ALGORITHM_STRICT_CRC32) { +#ifndef UNIV_INNOCHECKSUM page_warn_strict_checksum( curr_algo, SRV_CHECKSUM_ALGORITHM_INNODB, space_id, page_no); +#endif } return(false); } +#ifdef UNIV_INNOCHECKSUM + if (log_file) { + fprintf(log_file, "Fail; page " ULINTPF + " invalid (fails crc32 checksum)\n", + cur_page_num); + } +#endif /* UNIV_INNOCHECKSUM */ return(true); case SRV_CHECKSUM_ALGORITHM_INNODB: @@ -828,11 +931,29 @@ buf_page_is_corrupted( checksum_field1, checksum_field2)) { if (curr_algo == SRV_CHECKSUM_ALGORITHM_STRICT_INNODB) { +#ifndef UNIV_INNOCHECKSUM page_warn_strict_checksum( curr_algo, SRV_CHECKSUM_ALGORITHM_NONE, space_id, page_no); +#endif } +#ifdef UNIV_INNOCHECKSUM + if (log_file) { + fprintf(log_file, "page::" ULINTPF ";" + " old style: calculated = " ULINTPF ";" + " recorded = " ULINTPF "\n", cur_page_num, + buf_calc_page_old_checksum(read_buf), + checksum_field2); + fprintf(log_file, "page::" ULINTPF ";" + " new style: calculated = " ULINTPF ";" + " crc32 = %u; recorded = " ULINTPF "\n", + cur_page_num, + buf_calc_page_new_checksum(read_buf), + buf_calc_page_crc32(read_buf), + checksum_field1); + } +#endif /* UNIV_INNOCHECKSUM */ return(false); } @@ -841,15 +962,25 @@ buf_page_is_corrupted( checksum_field1, checksum_field2)) { if (curr_algo == SRV_CHECKSUM_ALGORITHM_STRICT_INNODB) { +#ifndef UNIV_INNOCHECKSUM page_warn_strict_checksum( curr_algo, SRV_CHECKSUM_ALGORITHM_CRC32, space_id, page_no); +#endif } return(false); } +#ifdef UNIV_INNOCHECKSUM + if (log_file) { + fprintf(log_file, "Fail; page " ULINTPF + " invalid (fails innodb checksum)\n", + cur_page_num); + } +#endif /* UNIV_INNOCHECKSUM */ + return(true); case SRV_CHECKSUM_ALGORITHM_STRICT_NONE: @@ -861,22 +992,34 @@ buf_page_is_corrupted( if (buf_page_is_checksum_valid_crc32(read_buf, checksum_field1, checksum_field2)) { +#ifndef UNIV_INNOCHECKSUM page_warn_strict_checksum( curr_algo, SRV_CHECKSUM_ALGORITHM_CRC32, space_id, page_no); +#endif return(false); } if (buf_page_is_checksum_valid_innodb(read_buf, checksum_field1, checksum_field2)) { +#ifndef UNIV_INNOCHECKSUM page_warn_strict_checksum( curr_algo, SRV_CHECKSUM_ALGORITHM_INNODB, space_id, page_no); +#endif return(false); } +#ifdef UNIV_INNOCHECKSUM + if (log_file) { + fprintf(log_file, "Fail; page " ULINTPF + " invalid (fails none checksum)\n", + cur_page_num); + } +#endif /* UNIV_INNOCHECKSUM */ + return(true); case SRV_CHECKSUM_ALGORITHM_NONE: @@ -890,6 +1033,7 @@ buf_page_is_corrupted( return(false); } +#ifndef UNIV_INNOCHECKSUM /********************************************************************//** Prints a page to stderr. */ UNIV_INTERN @@ -6352,3 +6496,4 @@ buf_page_decrypt_after_read(buf_page_t* bpage, fil_space_t* space) ut_ad(space->n_pending_ios > 0); return (success); } +#endif /* !UNIV_INNOCHECKSUM */ diff --git a/storage/innobase/buf/buf0checksum.cc b/storage/innobase/buf/buf0checksum.cc index 4101d117896..9e5f1dfe475 100644 --- a/storage/innobase/buf/buf0checksum.cc +++ b/storage/innobase/buf/buf0checksum.cc @@ -128,8 +128,6 @@ buf_calc_page_old_checksum( return(checksum); } -#ifndef UNIV_INNOCHECKSUM - /********************************************************************//** Return a printable string describing the checksum algorithm. @return algorithm name */ @@ -158,4 +156,3 @@ buf_checksum_algorithm_name( return(NULL); } -#endif /* !UNIV_INNOCHECKSUM */ diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index df5c250df90..b5b762c2cd9 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -25,13 +25,18 @@ Modified Jan Lindström jan.lindstrom@mariadb.com #include "fil0fil.h" #include "fil0crypt.h" +#include "mach0data.h" +#include "page0zip.h" +#include "buf0buf.h" +#include "buf0checksum.h" + +#ifndef UNIV_INNOCHECKSUM + #include "srv0srv.h" #include "srv0start.h" -#include "mach0data.h" #include "log0recv.h" #include "mtr0mtr.h" #include "mtr0log.h" -#include "page0zip.h" #include "ut0ut.h" #include "btr0scrub.h" #include "fsp0fsp.h" @@ -107,13 +112,20 @@ UNIV_INTERN mysql_pfs_key_t fil_crypt_data_mutex_key; extern my_bool srv_background_scrub_data_uncompressed; extern my_bool srv_background_scrub_data_compressed; +/*********************************************************************** +Check if a key needs rotation given a key_state +@param[in] encrypt_mode Encryption mode +@param[in] key_version Current key version +@param[in] latest_key_version Latest key version +@param[in] rotate_key_age when to rotate +@return true if key needs rotation, false if not */ static bool fil_crypt_needs_rotation( - fil_encryption_t encrypt_mode, /*!< in: Encryption - mode */ - uint key_version, /*!< in: Key version */ - uint latest_key_version, /*!< in: Latest key version */ - uint rotate_key_age); /*!< in: When to rotate */ + fil_encryption_t encrypt_mode, + uint key_version, + uint latest_key_version, + uint rotate_key_age) + MY_ATTRIBUTE((warn_unused_result)); /********************************************************************* Init space crypt */ @@ -908,137 +920,6 @@ fil_crypt_calculate_checksum( return checksum; } -/********************************************************************* -Verify that post encryption checksum match calculated checksum. -This function should be called only if tablespace contains crypt_data -metadata (this is strong indication that tablespace is encrypted). -Function also verifies that traditional checksum does not match -calculated checksum as if it does page could be valid unencrypted, -encrypted, or corrupted. - -@param[in] page Page to verify -@param[in] zip_size zip size -@param[in] space Tablespace -@param[in] pageno Page no -@return true if page is encrypted AND OK, false otherwise */ -UNIV_INTERN -bool -fil_space_verify_crypt_checksum( - byte* page, - ulint zip_size, - const fil_space_t* space, - ulint pageno) -{ - uint key_version = mach_read_from_4(page+ FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); - - /* If page is not encrypted, return false */ - if (key_version == 0) { - return(false); - } - - /* Read stored post encryption checksum. */ - ib_uint32_t checksum = mach_read_from_4( - page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4); - - /* Declare empty pages non-corrupted */ - if (checksum == 0 - && *reinterpret_cast(page + FIL_PAGE_LSN) == 0 - && buf_page_is_zeroes(page, zip_size)) { - return(true); - } - - /* Compressed and encrypted pages do not have checksum. Assume not - corrupted. Page verification happens after decompression in - buf_page_io_complete() using buf_page_is_corrupted(). */ - if (mach_read_from_2(page+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { - return (true); - } - - ib_uint32_t cchecksum1 = 0; - ib_uint32_t cchecksum2 = 0; - - /* Calculate checksums */ - if (zip_size) { - cchecksum1 = page_zip_calc_checksum( - page, zip_size, SRV_CHECKSUM_ALGORITHM_CRC32); - - if(cchecksum1 != checksum) { - cchecksum2 = page_zip_calc_checksum( - page, zip_size, - SRV_CHECKSUM_ALGORITHM_INNODB); - } - } else { - cchecksum1 = buf_calc_page_crc32(page); - - if (cchecksum1 != checksum) { - cchecksum2 = (ib_uint32_t) buf_calc_page_new_checksum( - page); - } - } - - /* If stored checksum matches one of the calculated checksums - page is not corrupted. */ - - bool encrypted = (checksum == cchecksum1 || checksum == cchecksum2 - || checksum == BUF_NO_CHECKSUM_MAGIC); - - /* MySQL 5.6 and MariaDB 10.0 and 10.1 will write an LSN to the - first page of each system tablespace file at - FIL_PAGE_FILE_FLUSH_LSN offset. On other pages and in other files, - the field might have been uninitialized until MySQL 5.5. In MySQL 5.7 - (and MariaDB Server 10.2.2) WL#7990 stopped writing the field for other - than page 0 of the system tablespace. - - Starting from MariaDB 10.1 the field has been repurposed for - encryption key_version. - - Starting with MySQL 5.7 (and MariaDB Server 10.2), the - field has been repurposed for SPATIAL INDEX pages for - FIL_RTREE_SPLIT_SEQ_NUM. - - Note that FIL_PAGE_FILE_FLUSH_LSN is not included in the InnoDB page - checksum. - - Thus, FIL_PAGE_FILE_FLUSH_LSN could contain any value. While the - field would usually be 0 for pages that are not encrypted, we cannot - assume that a nonzero value means that the page is encrypted. - Therefore we must validate the page both as encrypted and unencrypted - when FIL_PAGE_FILE_FLUSH_LSN does not contain 0. - */ - - ulint checksum1 = mach_read_from_4( - page + FIL_PAGE_SPACE_OR_CHKSUM); - - ulint checksum2 = checksum1; - - bool valid; - - if (zip_size) { - valid = (checksum1 == cchecksum1); - } else { - checksum2 = mach_read_from_4( - page + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM); - valid = (buf_page_is_checksum_valid_crc32(page,checksum1,checksum2) - || buf_page_is_checksum_valid_innodb(page,checksum1, checksum2)); - } - - if (encrypted && valid) { - /* If page is encrypted and traditional checksums match, - page could be still encrypted, or not encrypted and valid or - corrupted. */ - ib_logf(IB_LOG_LEVEL_ERROR, - " Page %lu in space %s (%lu) maybe corrupted." - " Post encryption checksum %u stored [%lu:%lu] key_version %u", - pageno, - space ? space->name : "N/A", - mach_read_from_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID), - checksum, checksum1, checksum2, key_version); - encrypted = false; - } - - return(encrypted); -} - /***********************************************************************/ /** A copy of global key state */ @@ -2555,8 +2436,9 @@ fil_space_crypt_close_tablespace( if (now >= last + 30) { ib_logf(IB_LOG_LEVEL_WARN, - "Waited %ld seconds to drop space: %s(" ULINTPF ").", - now - start, space->name, space->id); + "Waited %ld seconds to drop space: %s (" ULINTPF + ") active threads %u flushing=%d.", + now - start, space->name, space->id, cnt, flushing); last = now; } } @@ -2659,3 +2541,159 @@ fil_space_get_scrub_status( mutex_exit(&crypt_data->mutex); } } + +#endif /* !UNIV_INNOCHECKSUM */ + +/********************************************************************* +Verify that post encryption checksum match calculated checksum. +This function should be called only if tablespace contains crypt_data +metadata (this is strong indication that tablespace is encrypted). +Function also verifies that traditional checksum does not match +calculated checksum as if it does page could be valid unencrypted, +encrypted, or corrupted. + +@param[in] page Page to verify +@param[in] zip_size zip size +@param[in] space Tablespace +@param[in] pageno Page no +@return true if page is encrypted AND OK, false otherwise */ +UNIV_INTERN +bool +fil_space_verify_crypt_checksum( + byte* page, + ulint zip_size, +#ifndef UNIV_INNOCHECKSUM + const fil_space_t* space, +#else + const void* space, +#endif + ulint pageno) +{ + uint key_version = mach_read_from_4(page+ FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); + + /* If page is not encrypted, return false */ + if (key_version == 0) { + return(false); + } + + srv_checksum_algorithm_t algorithm = + static_cast(srv_checksum_algorithm); + + /* If no checksum is used, can't continue checking. */ + if (algorithm == SRV_CHECKSUM_ALGORITHM_NONE) { + return(true); + } + + /* Read stored post encryption checksum. */ + ib_uint32_t checksum = mach_read_from_4( + page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4); + + /* Declare empty pages non-corrupted */ + if (checksum == 0 + && *reinterpret_cast(page + FIL_PAGE_LSN) == 0 + && buf_page_is_zeroes(page, zip_size)) { + return(true); + } + + /* Compressed and encrypted pages do not have checksum. Assume not + corrupted. Page verification happens after decompression in + buf_page_io_complete() using buf_page_is_corrupted(). */ + if (mach_read_from_2(page+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { + return (true); + } + + ib_uint32_t cchecksum1 = 0; + ib_uint32_t cchecksum2 = 0; + + /* Calculate checksums */ + if (zip_size) { + cchecksum1 = page_zip_calc_checksum( + page, zip_size, SRV_CHECKSUM_ALGORITHM_CRC32); + + cchecksum2 = (cchecksum1 == checksum) + ? 0 + : page_zip_calc_checksum( + page, zip_size, + SRV_CHECKSUM_ALGORITHM_INNODB); + } else { + cchecksum1 = buf_calc_page_crc32(page); + cchecksum2 = (cchecksum1 == checksum) + ? 0 + : buf_calc_page_new_checksum(page); + } + + /* If stored checksum matches one of the calculated checksums + page is not corrupted. */ + + bool encrypted = (checksum == cchecksum1 || checksum == cchecksum2 + || checksum == BUF_NO_CHECKSUM_MAGIC); + + /* MySQL 5.6 and MariaDB 10.0 and 10.1 will write an LSN to the + first page of each system tablespace file at + FIL_PAGE_FILE_FLUSH_LSN offset. On other pages and in other files, + the field might have been uninitialized until MySQL 5.5. In MySQL 5.7 + (and MariaDB Server 10.2.2) WL#7990 stopped writing the field for other + than page 0 of the system tablespace. + + Starting from MariaDB 10.1 the field has been repurposed for + encryption key_version. + + Starting with MySQL 5.7 (and MariaDB Server 10.2), the + field has been repurposed for SPATIAL INDEX pages for + FIL_RTREE_SPLIT_SEQ_NUM. + + Note that FIL_PAGE_FILE_FLUSH_LSN is not included in the InnoDB page + checksum. + + Thus, FIL_PAGE_FILE_FLUSH_LSN could contain any value. While the + field would usually be 0 for pages that are not encrypted, we cannot + assume that a nonzero value means that the page is encrypted. + Therefore we must validate the page both as encrypted and unencrypted + when FIL_PAGE_FILE_FLUSH_LSN does not contain 0. + */ + + uint32_t checksum1 = mach_read_from_4(page + FIL_PAGE_SPACE_OR_CHKSUM); + uint32_t checksum2; + + bool valid; + + if (zip_size) { + valid = (checksum1 == cchecksum1); + checksum2 = checksum1; + } else { + checksum2 = mach_read_from_4( + page + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM); + valid = (buf_page_is_checksum_valid_crc32(page,checksum1,checksum2) + || buf_page_is_checksum_valid_innodb(page,checksum1, checksum2)); + } + + if (encrypted && valid) { + /* If page is encrypted and traditional checksums match, + page could be still encrypted, or not encrypted and valid or + corrupted. */ +#ifndef UNIV_INNOCHECKSUM + ib_logf(IB_LOG_LEVEL_ERROR, + " Page " ULINTPF " in space %s (" ULINTPF ") maybe corrupted." + " Post encryption checksum %u stored [%u:%u] key_version %u", + pageno, + space ? space->name : "N/A", + mach_read_from_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID), + checksum, checksum1, checksum2, key_version); +#else + if (log_file) { + fprintf(log_file, + "Page " ULINTPF ":" ULINTPF " may be corrupted." + " Post encryption checksum %u" + " stored [%u:%u] key_version %u\n", + pageno, + mach_read_from_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID), + checksum, checksum1, checksum2, + key_version); + } +#endif /* UNIV_INNOCHECKSUM */ + + encrypted = false; + } + + return(encrypted); +} diff --git a/storage/innobase/include/buf0buf.h b/storage/innobase/include/buf0buf.h index cabc6f64150..5fd3e05370f 100644 --- a/storage/innobase/include/buf0buf.h +++ b/storage/innobase/include/buf0buf.h @@ -31,6 +31,7 @@ Created 11/5/1995 Heikki Tuuri #include "fil0fil.h" #include "mtr0types.h" #include "buf0types.h" +#ifndef UNIV_INNOCHECKSUM #include "hash0hash.h" #include "ut0byte.h" #include "page0types.h" @@ -643,6 +644,8 @@ buf_block_unfix( # define buf_block_modify_clock_inc(block) ((void) 0) #endif /* !UNIV_HOTBACKUP */ +#endif /* !UNIV_INNOCHECKSUM */ + /** Checks if the page is in crc32 checksum format. @param[in] read_buf database page @param[in] checksum_field1 new checksum field @@ -691,8 +694,13 @@ buf_page_is_corrupted( bool check_lsn, const byte* read_buf, ulint zip_size, +#ifndef UNIV_INNOCHECKSUM const fil_space_t* space) +#else + const void* space = NULL) +#endif MY_ATTRIBUTE((warn_unused_result)); + /** Check if a page is all zeroes. @param[in] read_buf database page @param[in] zip_size ROW_FORMAT=COMPRESSED page size, or 0 @@ -700,6 +708,9 @@ buf_page_is_corrupted( UNIV_INTERN bool buf_page_is_zeroes(const byte* read_buf, ulint zip_size); + +#ifndef UNIV_INNOCHECKSUM + #ifndef UNIV_HOTBACKUP /**********************************************************************//** Gets the space id, page offset, and byte offset within page of a @@ -2470,4 +2481,5 @@ struct CheckUnzipLRUAndLRUList { #include "buf0buf.ic" #endif +#endif /*! UNIV_INNOCHECKSUM */ #endif diff --git a/storage/innobase/include/fil0crypt.h b/storage/innobase/include/fil0crypt.h index 228dfb895fe..7eb8b46d901 100644 --- a/storage/innobase/include/fil0crypt.h +++ b/storage/innobase/include/fil0crypt.h @@ -71,9 +71,11 @@ struct key_struct /** is encryption enabled */ extern ulong srv_encrypt_tables; +#ifndef UNIV_INNOCHECKSUM #ifdef UNIV_PFS_MUTEX extern mysql_pfs_key_t fil_crypt_data_mutex_key; #endif +#endif /* !UNIV_INNOCHECKSUM */ /** Mutex helper for crypt_data->scheme @param[in, out] schme encryption scheme @@ -102,6 +104,8 @@ struct fil_space_rotate_state_t } scrubbing; }; +#ifndef UNIV_INNOCHECKSUM + struct fil_space_crypt_t : st_encryption_scheme { public: @@ -399,6 +403,8 @@ fil_crypt_calculate_checksum( const byte* dst_frame) MY_ATTRIBUTE((warn_unused_result)); +#endif /* UNIV_INNOCHECKSUM */ + /********************************************************************* Verify that post encryption checksum match calculated checksum. This function should be called only if tablespace contains crypt_data @@ -417,10 +423,16 @@ bool fil_space_verify_crypt_checksum( byte* page, ulint zip_size, +#ifndef UNIV_INNOCHECKSUM const fil_space_t* space, +#else + const void* space, +#endif ulint pageno) MY_ATTRIBUTE((warn_unused_result)); +#ifndef UNIV_INNOCHECKSUM + /********************************************************************* Adjust thread count for key rotation @param[in] enw_cnt Number of threads to be used */ @@ -508,4 +520,5 @@ fil_space_get_scrub_status( #include "fil0crypt.ic" #endif +#endif /* !UNIV_INNOCHECKSUM */ #endif /* fil0crypt_h */ diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index 8033d0cc839..e481ea3a8aa 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -183,8 +183,6 @@ extern fil_addr_t fil_addr_null; #define FIL_LOG 502 /*!< redo log */ /* @} */ -#ifndef UNIV_INNOCHECKSUM - /** Structure containing encryption specification */ struct fil_space_crypt_t; @@ -209,6 +207,10 @@ extern ulint fil_n_pending_tablespace_flushes; /** Number of files currently open */ extern ulint fil_n_file_opened; +#ifndef UNIV_INNOCHECKSUM + +struct fil_space_t; + struct fsp_open_info { ibool success; /*!< Has the tablespace been opened? */ const char* check_msg; /*!< fil_check_first_page() message */ @@ -225,8 +227,6 @@ struct fsp_open_info { dict_table_t* table; /*!< table */ }; -struct fil_space_t; - /** File node of a tablespace or the log data space */ struct fil_node_t { fil_space_t* space; /*!< backpointer to the space where this node diff --git a/storage/innobase/include/mach0data.ic b/storage/innobase/include/mach0data.ic index 33b6405c8b5..72e793da8fd 100644 --- a/storage/innobase/include/mach0data.ic +++ b/storage/innobase/include/mach0data.ic @@ -154,6 +154,8 @@ mach_read_from_3( ); } +#endif /* !UNIV_INNOCHECKSUM */ + /*******************************************************//** The following function is used to store data in four consecutive bytes. We store the most significant byte to the lowest address. */ @@ -172,8 +174,6 @@ mach_write_to_4( b[3] = (byte) n; } -#endif /* !UNIV_INNOCHECKSUM */ - /********************************************************//** The following function is used to fetch data from 4 consecutive bytes. The most significant byte is at the lowest address. diff --git a/storage/innobase/include/univ.i b/storage/innobase/include/univ.i index 1cc89ae503c..aafb84db6f1 100644 --- a/storage/innobase/include/univ.i +++ b/storage/innobase/include/univ.i @@ -482,6 +482,12 @@ typedef long int lint; typedef unsigned long long int ullint; #endif /* UNIV_HOTBACKUP */ +#ifdef UNIV_INNOCHECKSUM +extern bool strict_verify; +extern FILE* log_file; +extern ulint cur_page_num; +#endif /* UNIV_INNOCHECKSUM */ + #ifndef __WIN__ #if SIZEOF_LONG != SIZEOF_VOIDP #error "Error: InnoDB's ulint must be of the same size as void*" diff --git a/storage/innobase/page/page0zip.cc b/storage/innobase/page/page0zip.cc index b4a8719c920..6e26f830fe6 100644 --- a/storage/innobase/page/page0zip.cc +++ b/storage/innobase/page/page0zip.cc @@ -4937,26 +4937,26 @@ page_zip_verify_checksum( #error "FIL_PAGE_LSN must be 64 bit aligned" #endif -#ifndef UNIV_INNOCHECKSUM - /* innochecksum doesn't compile with ut_d. Since we don't - need to check for empty pages when running innochecksum, - just don't include this code. */ /* Check if page is empty */ if (stored == 0 && *reinterpret_cast(static_cast( data) + FIL_PAGE_LSN) == 0) { /* make sure that the page is really empty */ - ulint i; - for (i = 0; i < size; i++) { + for (ulint i = 0; i < size; i++) { if (*((const char*) data + i) != 0) { return(FALSE); } } +#ifdef UNIV_INNOCHECKSUM + if (log_file) { + fprintf(log_file, "Page::%lu is empty and" + " uncorrupted\n", cur_page_num); + } +#endif /* UNIV_INNOCHECKSUM */ /* Empty page */ return(TRUE); } -#endif const srv_checksum_algorithm_t curr_algo = static_cast(srv_checksum_algorithm); @@ -4968,6 +4968,33 @@ page_zip_verify_checksum( calc = static_cast(page_zip_calc_checksum( data, size, curr_algo)); +#ifdef UNIV_INNOCHECKSUM + if (log_file) { + fprintf(log_file, "page::%lu;" + " %s checksum: calculated = %u;" + " recorded = %u\n", cur_page_num, + buf_checksum_algorithm_name( + static_cast( + srv_checksum_algorithm)), + calc, stored); + } + + if (!strict_verify) { + + const uint32_t crc32 = page_zip_calc_checksum( + data, size, SRV_CHECKSUM_ALGORITHM_CRC32); + + if (log_file) { + fprintf(log_file, "page::%lu: crc32 checksum:" + " calculated = %u; recorded = %u\n", + cur_page_num, crc32, stored); + fprintf(log_file, "page::%lu: none checksum:" + " calculated = %lu; recorded = %u\n", + cur_page_num, BUF_NO_CHECKSUM_MAGIC, stored); + } + } +#endif /* UNIV_INNOCHECKSUM */ + if (stored == calc) { return(TRUE); } diff --git a/storage/xtradb/fil/fil0crypt.cc b/storage/xtradb/fil/fil0crypt.cc index 6d3c2e98010..b5b762c2cd9 100644 --- a/storage/xtradb/fil/fil0crypt.cc +++ b/storage/xtradb/fil/fil0crypt.cc @@ -25,13 +25,18 @@ Modified Jan Lindström jan.lindstrom@mariadb.com #include "fil0fil.h" #include "fil0crypt.h" +#include "mach0data.h" +#include "page0zip.h" +#include "buf0buf.h" +#include "buf0checksum.h" + +#ifndef UNIV_INNOCHECKSUM + #include "srv0srv.h" #include "srv0start.h" -#include "mach0data.h" #include "log0recv.h" #include "mtr0mtr.h" #include "mtr0log.h" -#include "page0zip.h" #include "ut0ut.h" #include "btr0scrub.h" #include "fsp0fsp.h" @@ -107,13 +112,20 @@ UNIV_INTERN mysql_pfs_key_t fil_crypt_data_mutex_key; extern my_bool srv_background_scrub_data_uncompressed; extern my_bool srv_background_scrub_data_compressed; +/*********************************************************************** +Check if a key needs rotation given a key_state +@param[in] encrypt_mode Encryption mode +@param[in] key_version Current key version +@param[in] latest_key_version Latest key version +@param[in] rotate_key_age when to rotate +@return true if key needs rotation, false if not */ static bool fil_crypt_needs_rotation( - fil_encryption_t encrypt_mode, /*!< in: Encryption - mode */ - uint key_version, /*!< in: Key version */ - uint latest_key_version, /*!< in: Latest key version */ - uint rotate_key_age); /*!< in: When to rotate */ + fil_encryption_t encrypt_mode, + uint key_version, + uint latest_key_version, + uint rotate_key_age) + MY_ATTRIBUTE((warn_unused_result)); /********************************************************************* Init space crypt */ @@ -908,137 +920,6 @@ fil_crypt_calculate_checksum( return checksum; } -/********************************************************************* -Verify that post encryption checksum match calculated checksum. -This function should be called only if tablespace contains crypt_data -metadata (this is strong indication that tablespace is encrypted). -Function also verifies that traditional checksum does not match -calculated checksum as if it does page could be valid unencrypted, -encrypted, or corrupted. - -@param[in] page Page to verify -@param[in] zip_size zip size -@param[in] space Tablespace -@param[in] pageno Page no -@return true if page is encrypted AND OK, false otherwise */ -UNIV_INTERN -bool -fil_space_verify_crypt_checksum( - byte* page, - ulint zip_size, - const fil_space_t* space, - ulint pageno) -{ - uint key_version = mach_read_from_4(page+ FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); - - /* If page is not encrypted, return false */ - if (key_version == 0) { - return(false); - } - - /* Read stored post encryption checksum. */ - ib_uint32_t checksum = mach_read_from_4( - page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4); - - /* Declare empty pages non-corrupted */ - if (checksum == 0 - && *reinterpret_cast(page + FIL_PAGE_LSN) == 0 - && buf_page_is_zeroes(page, zip_size)) { - return(true); - } - - /* Compressed and encrypted pages do not have checksum. Assume not - corrupted. Page verification happens after decompression in - buf_page_io_complete() using buf_page_is_corrupted(). */ - if (mach_read_from_2(page+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { - return (true); - } - - ib_uint32_t cchecksum1 = 0; - ib_uint32_t cchecksum2 = 0; - - /* Calculate checksums */ - if (zip_size) { - cchecksum1 = page_zip_calc_checksum( - page, zip_size, SRV_CHECKSUM_ALGORITHM_CRC32); - - if(cchecksum1 != checksum) { - cchecksum2 = page_zip_calc_checksum( - page, zip_size, - SRV_CHECKSUM_ALGORITHM_INNODB); - } - } else { - cchecksum1 = buf_calc_page_crc32(page); - - if (cchecksum1 != checksum) { - cchecksum2 = (ib_uint32_t) buf_calc_page_new_checksum( - page); - } - } - - /* If stored checksum matches one of the calculated checksums - page is not corrupted. */ - - bool encrypted = (checksum == cchecksum1 || checksum == cchecksum2 - || checksum == BUF_NO_CHECKSUM_MAGIC); - - /* MySQL 5.6 and MariaDB 10.0 and 10.1 will write an LSN to the - first page of each system tablespace file at - FIL_PAGE_FILE_FLUSH_LSN offset. On other pages and in other files, - the field might have been uninitialized until MySQL 5.5. In MySQL 5.7 - (and MariaDB Server 10.2.2) WL#7990 stopped writing the field for other - than page 0 of the system tablespace. - - Starting from MariaDB 10.1 the field has been repurposed for - encryption key_version. - - Starting with MySQL 5.7 (and MariaDB Server 10.2), the - field has been repurposed for SPATIAL INDEX pages for - FIL_RTREE_SPLIT_SEQ_NUM. - - Note that FIL_PAGE_FILE_FLUSH_LSN is not included in the InnoDB page - checksum. - - Thus, FIL_PAGE_FILE_FLUSH_LSN could contain any value. While the - field would usually be 0 for pages that are not encrypted, we cannot - assume that a nonzero value means that the page is encrypted. - Therefore we must validate the page both as encrypted and unencrypted - when FIL_PAGE_FILE_FLUSH_LSN does not contain 0. - */ - - ulint checksum1 = mach_read_from_4( - page + FIL_PAGE_SPACE_OR_CHKSUM); - - ulint checksum2 = checksum1; - - bool valid; - - if (zip_size) { - valid = (checksum1 == cchecksum1); - } else { - checksum1 = mach_read_from_4( - page + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM); - valid = (buf_page_is_checksum_valid_crc32(page,checksum1,checksum2) - || buf_page_is_checksum_valid_innodb(page,checksum1, checksum2)); - } - - if (encrypted && valid) { - /* If page is encrypted and traditional checksums match, - page could be still encrypted, or not encrypted and valid or - corrupted. */ - ib_logf(IB_LOG_LEVEL_ERROR, - " Page %lu in space %s (%lu) maybe corrupted." - " Post encryption checksum %u stored [%lu:%lu] key_version %u", - pageno, - space ? space->name : "N/A", - mach_read_from_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID), - checksum, checksum1, checksum2, key_version); - encrypted = false; - } - - return(encrypted); -} - /***********************************************************************/ /** A copy of global key state */ @@ -2660,3 +2541,159 @@ fil_space_get_scrub_status( mutex_exit(&crypt_data->mutex); } } + +#endif /* !UNIV_INNOCHECKSUM */ + +/********************************************************************* +Verify that post encryption checksum match calculated checksum. +This function should be called only if tablespace contains crypt_data +metadata (this is strong indication that tablespace is encrypted). +Function also verifies that traditional checksum does not match +calculated checksum as if it does page could be valid unencrypted, +encrypted, or corrupted. + +@param[in] page Page to verify +@param[in] zip_size zip size +@param[in] space Tablespace +@param[in] pageno Page no +@return true if page is encrypted AND OK, false otherwise */ +UNIV_INTERN +bool +fil_space_verify_crypt_checksum( + byte* page, + ulint zip_size, +#ifndef UNIV_INNOCHECKSUM + const fil_space_t* space, +#else + const void* space, +#endif + ulint pageno) +{ + uint key_version = mach_read_from_4(page+ FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); + + /* If page is not encrypted, return false */ + if (key_version == 0) { + return(false); + } + + srv_checksum_algorithm_t algorithm = + static_cast(srv_checksum_algorithm); + + /* If no checksum is used, can't continue checking. */ + if (algorithm == SRV_CHECKSUM_ALGORITHM_NONE) { + return(true); + } + + /* Read stored post encryption checksum. */ + ib_uint32_t checksum = mach_read_from_4( + page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4); + + /* Declare empty pages non-corrupted */ + if (checksum == 0 + && *reinterpret_cast(page + FIL_PAGE_LSN) == 0 + && buf_page_is_zeroes(page, zip_size)) { + return(true); + } + + /* Compressed and encrypted pages do not have checksum. Assume not + corrupted. Page verification happens after decompression in + buf_page_io_complete() using buf_page_is_corrupted(). */ + if (mach_read_from_2(page+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { + return (true); + } + + ib_uint32_t cchecksum1 = 0; + ib_uint32_t cchecksum2 = 0; + + /* Calculate checksums */ + if (zip_size) { + cchecksum1 = page_zip_calc_checksum( + page, zip_size, SRV_CHECKSUM_ALGORITHM_CRC32); + + cchecksum2 = (cchecksum1 == checksum) + ? 0 + : page_zip_calc_checksum( + page, zip_size, + SRV_CHECKSUM_ALGORITHM_INNODB); + } else { + cchecksum1 = buf_calc_page_crc32(page); + cchecksum2 = (cchecksum1 == checksum) + ? 0 + : buf_calc_page_new_checksum(page); + } + + /* If stored checksum matches one of the calculated checksums + page is not corrupted. */ + + bool encrypted = (checksum == cchecksum1 || checksum == cchecksum2 + || checksum == BUF_NO_CHECKSUM_MAGIC); + + /* MySQL 5.6 and MariaDB 10.0 and 10.1 will write an LSN to the + first page of each system tablespace file at + FIL_PAGE_FILE_FLUSH_LSN offset. On other pages and in other files, + the field might have been uninitialized until MySQL 5.5. In MySQL 5.7 + (and MariaDB Server 10.2.2) WL#7990 stopped writing the field for other + than page 0 of the system tablespace. + + Starting from MariaDB 10.1 the field has been repurposed for + encryption key_version. + + Starting with MySQL 5.7 (and MariaDB Server 10.2), the + field has been repurposed for SPATIAL INDEX pages for + FIL_RTREE_SPLIT_SEQ_NUM. + + Note that FIL_PAGE_FILE_FLUSH_LSN is not included in the InnoDB page + checksum. + + Thus, FIL_PAGE_FILE_FLUSH_LSN could contain any value. While the + field would usually be 0 for pages that are not encrypted, we cannot + assume that a nonzero value means that the page is encrypted. + Therefore we must validate the page both as encrypted and unencrypted + when FIL_PAGE_FILE_FLUSH_LSN does not contain 0. + */ + + uint32_t checksum1 = mach_read_from_4(page + FIL_PAGE_SPACE_OR_CHKSUM); + uint32_t checksum2; + + bool valid; + + if (zip_size) { + valid = (checksum1 == cchecksum1); + checksum2 = checksum1; + } else { + checksum2 = mach_read_from_4( + page + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM); + valid = (buf_page_is_checksum_valid_crc32(page,checksum1,checksum2) + || buf_page_is_checksum_valid_innodb(page,checksum1, checksum2)); + } + + if (encrypted && valid) { + /* If page is encrypted and traditional checksums match, + page could be still encrypted, or not encrypted and valid or + corrupted. */ +#ifndef UNIV_INNOCHECKSUM + ib_logf(IB_LOG_LEVEL_ERROR, + " Page " ULINTPF " in space %s (" ULINTPF ") maybe corrupted." + " Post encryption checksum %u stored [%u:%u] key_version %u", + pageno, + space ? space->name : "N/A", + mach_read_from_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID), + checksum, checksum1, checksum2, key_version); +#else + if (log_file) { + fprintf(log_file, + "Page " ULINTPF ":" ULINTPF " may be corrupted." + " Post encryption checksum %u" + " stored [%u:%u] key_version %u\n", + pageno, + mach_read_from_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID), + checksum, checksum1, checksum2, + key_version); + } +#endif /* UNIV_INNOCHECKSUM */ + + encrypted = false; + } + + return(encrypted); +}