From 8b019f87dd51b1d2755655d2f136d5429fc47e1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Thu, 3 Aug 2017 08:29:36 +0300 Subject: [PATCH] MDEV-11939: innochecksum mistakes a file for an encrypted one (page 0 invalid) Always read full page 0 to determine does tablespace contain encryption metadata. Tablespaces that are page compressed or page compressed and encrypted do not compare checksum as it does not exists. For encrypted tables use checksum verification written for encrypted tables and normal tables use normal method. buf_page_is_checksum_valid_crc32 buf_page_is_checksum_valid_innodb buf_page_is_checksum_valid_none Add Innochecksum logging to file buf_page_is_corrupted Remove ib_logf and page_warn_strict_checksum calls in innochecksum compilation. Add innochecksum logging to file. fil0crypt.cc fil0crypt.h Modify to be able to use in innochecksum compilation and move fil_space_verify_crypt_checksum to end of the file. Add innochecksum logging to file. univ.i Add innochecksum strict_verify, log_file and cur_page_num variables as extern. page_zip_verify_checksum Add innochecksum logging to file. innochecksum.cc Lot of changes most notable able to read encryption metadata from page 0 of the tablespace. Added test case where we corrupt intentionally FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION (encryption key version) FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION+4 (post encryption checksum) FIL_DATA+10 (data) --- extra/CMakeLists.txt | 2 + extra/innochecksum.cc | 2592 +++++++++++------ .../suite/encryption/r/innochecksum.result | 24 +- .../suite/encryption/t/innochecksum.test | 213 +- storage/innobase/buf/buf0buf.cc | 179 +- storage/innobase/buf/buf0checksum.cc | 3 - storage/innobase/fil/fil0crypt.cc | 318 +- storage/innobase/include/buf0buf.h | 12 + storage/innobase/include/fil0crypt.h | 13 + storage/innobase/include/fil0fil.h | 8 +- storage/innobase/include/mach0data.ic | 4 +- storage/innobase/include/univ.i | 6 + storage/innobase/page/page0zip.cc | 41 +- storage/xtradb/fil/fil0crypt.cc | 313 +- 14 files changed, 2473 insertions(+), 1255 deletions(-) diff --git a/extra/CMakeLists.txt b/extra/CMakeLists.txt index 5b650413bf3..c8e15dd4fb4 100644 --- a/extra/CMakeLists.txt +++ b/extra/CMakeLists.txt @@ -77,7 +77,9 @@ IF(WITH_INNOBASE_STORAGE_ENGINE OR WITH_XTRADB_STORAGE_ENGINE) ../storage/innobase/buf/buf0checksum.cc ../storage/innobase/ut/ut0crc32.cc ../storage/innobase/ut/ut0ut.cc + ../storage/innobase/buf/buf0buf.cc ../storage/innobase/page/page0zip.cc + ../storage/innobase/fil/fil0crypt.cc ) IF(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64le") diff --git a/extra/innochecksum.cc b/extra/innochecksum.cc index 28191da2c48..e258a8c1d08 100644 --- a/extra/innochecksum.cc +++ b/extra/innochecksum.cc @@ -1,6 +1,6 @@ /* - Copyright (c) 2005, 2012, Oracle and/or its affiliates. - Copyright (c) 2014, 2015, MariaDB Corporation. + Copyright (c) 2005, 2016, Oracle and/or its affiliates. All rights reserved. + Copyright (c) 2014, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -42,6 +42,8 @@ /* Only parts of these files are included from the InnoDB codebase. The parts not included are excluded by #ifndef UNIV_INNOCHECKSUM. */ +typedef void fil_space_t; + #include "univ.i" /* include all of this */ #define FLST_BASE_NODE_SIZE (4 + 2 * FIL_ADDR_SIZE) @@ -55,6 +57,7 @@ The parts not included are excluded by #ifndef UNIV_INNOCHECKSUM. */ #include "fsp0types.h" #include "rem0rec.h" #include "buf0checksum.h" /* buf_calc_page_*() */ +#include "buf0buf.h" /* buf_page_is_corrupted */ #include "fil0fil.h" /* FIL_* */ #include "page0page.h" /* PAGE_* */ #include "page0zip.h" /* page_zip_*() */ @@ -63,6 +66,9 @@ The parts not included are excluded by #ifndef UNIV_INNOCHECKSUM. */ fsp_flags_get_zip_size() */ #include "ut0crc32.h" /* ut_crc32_init() */ #include "fsp0pagecompress.h" /* fil_get_compression_alg_name */ +#include "fil0crypt.h" /* fil_space_verify_crypt_checksum */ + +#include #ifdef UNIV_NONINL # include "fsp0fsp.ic" @@ -70,50 +76,104 @@ The parts not included are excluded by #ifndef UNIV_INNOCHECKSUM. */ # include "ut0rnd.ic" #endif +#ifndef PRIuMAX +#define PRIuMAX "llu" +#endif + /* Global variables */ -static my_bool verbose; -static my_bool debug; -static my_bool skip_corrupt; -static my_bool just_count; -static ulong start_page; -static ulong end_page; -static ulong do_page; -static my_bool use_end_page; -static my_bool do_one_page; -static my_bool per_page_details; +static bool verbose; +static bool just_count; +static ulint start_page; +static ulint end_page; +static ulint do_page; +static bool use_end_page; +static bool do_one_page; static my_bool do_leaf; +static my_bool per_page_details; static ulong n_merge; -ulong srv_page_size; /* replaces declaration in srv0srv.c */ +extern ulong srv_checksum_algorithm; static ulong physical_page_size; /* Page size in bytes on disk. */ static ulong logical_page_size; /* Page size when uncompressed. */ -static bool compressed= false; /* Is tablespace compressed */ +ulong srv_page_size; +/* Current page number (0 based). */ +ulint cur_page_num; +/* Skip the checksum verification. */ +static bool no_check; +/* Enabled for strict checksum verification. */ +bool strict_verify = 0; +/* Enabled for rewrite checksum. */ +static bool do_write; +/* Mismatches count allowed (0 by default). */ +static ulint allow_mismatches; +static bool page_type_summary; +static bool page_type_dump; +/* Store filename for page-type-dump option. */ +char* page_dump_filename = 0; +/* skip the checksum verification & rewrite if page is doublewrite buffer. */ +static bool skip_page = 0; +const char *dbug_setting = "FALSE"; +char* log_filename = NULL; +/* User defined filename for logging. */ +FILE* log_file = NULL; +/* Enabled for log write option. */ +static bool is_log_enabled = false; -int n_undo_state_active; -int n_undo_state_cached; -int n_undo_state_to_free; -int n_undo_state_to_purge; -int n_undo_state_prepared; -int n_undo_state_other; -int n_undo_insert, n_undo_update, n_undo_other; -int n_bad_checksum; -int n_fil_page_index; -int n_fil_page_undo_log; -int n_fil_page_inode; -int n_fil_page_ibuf_free_list; -int n_fil_page_allocated; -int n_fil_page_ibuf_bitmap; -int n_fil_page_type_sys; -int n_fil_page_type_trx_sys; -int n_fil_page_type_fsp_hdr; -int n_fil_page_type_allocated; -int n_fil_page_type_xdes; -int n_fil_page_type_blob; -int n_fil_page_type_zblob; -int n_fil_page_type_other; -int n_fil_page_type_page_compressed; -int n_fil_page_type_page_compressed_encrypted; +#ifndef _WIN32 +/* advisory lock for non-window system. */ +struct flock lk; +#endif /* _WIN32 */ -int n_fil_page_max_index_id; +/* Strict check algorithm name. */ +static ulong strict_check; +/* Rewrite checksum algorithm name. */ +static ulong write_check; + +/* Innodb page type. */ +struct innodb_page_type { + int n_undo_state_active; + int n_undo_state_cached; + int n_undo_state_to_free; + int n_undo_state_to_purge; + int n_undo_state_prepared; + int n_undo_state_other; + int n_undo_insert; + int n_undo_update; + int n_undo_other; + int n_fil_page_index; + int n_fil_page_undo_log; + int n_fil_page_inode; + int n_fil_page_ibuf_free_list; + int n_fil_page_ibuf_bitmap; + int n_fil_page_type_sys; + int n_fil_page_type_trx_sys; + int n_fil_page_type_fsp_hdr; + int n_fil_page_type_allocated; + int n_fil_page_type_xdes; + int n_fil_page_type_blob; + int n_fil_page_type_zblob; + int n_fil_page_type_other; + int n_fil_page_type_zblob2; + int n_fil_page_type_page_compressed; + int n_fil_page_type_page_compressed_encrypted; +} page_type; + +/* Possible values for "--strict-check" for strictly verify checksum +and "--write" for rewrite checksum. */ +static const char *innochecksum_algorithms[] = { + "crc32", + "crc32", + "innodb", + "innodb", + "none", + "none", + NullS +}; + +/* Used to define an enumerate type of the "innochecksum algorithm". */ +static TYPELIB innochecksum_algorithms_typelib = { + array_elements(innochecksum_algorithms)-1,"", + innochecksum_algorithms, NULL +}; #define SIZE_RANGES_FOR_PAGE 10 #define NUM_RETRIES 3 @@ -143,7 +203,7 @@ struct per_index_stats { last element for pages with more than logical_page_size */ unsigned long long pages_in_size_range[SIZE_RANGES_FOR_PAGE+2]; - std::map leaves; + std::map leaves; per_index_stats():pages(0), leaf_pages(0), first_leaf_page(0), count(0), free_pages(0), max_data_size(0), total_n_recs(0), @@ -155,64 +215,940 @@ struct per_index_stats { std::map index_ids; -bool encrypted = false; +void print_index_leaf_stats( + unsigned long long id, + const per_index_stats& index, + FILE* fil_out) -/* Get the page size of the filespace from the filespace header. */ -static -my_bool -get_page_size( -/*==========*/ - FILE* f, /*!< in: file pointer, must be open - and set to start of file */ - byte* buf, /*!< in: buffer used to read the page */ - ulong* logical_page_size, /*!< out: Logical/Uncompressed page size */ - ulong* physical_page_size) /*!< out: Physical/Commpressed page size */ { - ulong flags; - - int bytes= fread(buf, 1, UNIV_PAGE_SIZE_MIN, f); - - if (ferror(f)) - { - perror("Error reading file header"); - return FALSE; - } - - if (bytes != UNIV_PAGE_SIZE_MIN) - { - fprintf(stderr, "Error; Was not able to read the minimum page size "); - fprintf(stderr, "of %d bytes. Bytes read was %d\n", UNIV_PAGE_SIZE_MIN, bytes); - return FALSE; - } - - rewind(f); - - flags = mach_read_from_4(buf + FIL_PAGE_DATA + FSP_SPACE_FLAGS); - - /* srv_page_size is used by InnoDB code as UNIV_PAGE_SIZE */ - srv_page_size = *logical_page_size = fsp_flags_get_page_size(flags); - - /* fsp_flags_get_zip_size() will return zero if not compressed. */ - *physical_page_size = fsp_flags_get_zip_size(flags); - if (*physical_page_size == 0) - { - *physical_page_size= *logical_page_size; - } - else - { - compressed= true; - } - - - return TRUE; + ulint page_no = index.first_leaf_page; + std::map::const_iterator it_page = index.leaves.find(page_no); + fprintf(fil_out, "\nindex: %llu leaf page stats: n_pages = %llu\n", + id, index.leaf_pages); + fprintf(fil_out, "page_no\tdata_size\tn_recs\n"); + while (it_page != index.leaves.end()) { + const per_page_stats& stat = it_page->second; + fprintf(fil_out, "%llu\t%lu\t%lu\n", it_page->first, stat.data_size, stat.n_recs); + page_no = stat.right_page_no; + it_page = index.leaves.find(page_no); + } } - -/* command line argument to do page checks (that's it) */ -/* another argument to specify page ranges... seek to right spot and go from there */ - -static struct my_option innochecksum_options[] = +void defrag_analysis( + unsigned long long id, + const per_index_stats& index, + FILE* fil_out) { + // TODO: make it work for compressed pages too + std::map::const_iterator it = index.leaves.find(index.first_leaf_page); + ulint n_pages = 0; + ulint n_leaf_pages = 0; + while (it != index.leaves.end()) { + ulint data_size_total = 0; + for (ulong i = 0; i < n_merge; i++) { + const per_page_stats& stat = it->second; + n_leaf_pages ++; + data_size_total += stat.data_size; + it = index.leaves.find(stat.right_page_no); + if (it == index.leaves.end()) { + break; + } + } + + if (index.max_data_size) { + n_pages += data_size_total / index.max_data_size; + if (data_size_total % index.max_data_size != 0) { + n_pages += 1; + } + } + } + + if (index.leaf_pages) { + fprintf(fil_out, "count = %lu free = %lu\n", index.count, index.free_pages); + } + + if (!n_leaf_pages) { + n_leaf_pages = 1; + } + + fprintf(fil_out, "%llu\t\t%llu\t\t%lu\t\t%lu\t\t%lu\t\t%.2f\t%lu\n", + id, index.leaf_pages, n_leaf_pages, n_merge, n_pages, + 1.0 - (double)n_pages / (double)n_leaf_pages, index.max_data_size); +} + +void print_leaf_stats( + FILE* fil_out) +{ + fprintf(fil_out, "\n**************************************************\n"); + fprintf(fil_out, "index_id\t#leaf_pages\t#actual_leaf_pages\tn_merge\t" + "#leaf_after_merge\tdefrag\n"); + for (std::map::const_iterator it = index_ids.begin(); + it != index_ids.end(); it++) { + const per_index_stats& index = it->second; + + if (verbose) { + print_index_leaf_stats(it->first, index, fil_out); + } + + if (n_merge) { + defrag_analysis(it->first, index, fil_out); + } + } +} + +#ifdef _WIN32 +/***********************************************//* + @param [in] error error no. from the getLastError(). + + @retval error message corresponding to error no. +*/ +static +char* +error_message( + int error) +{ + static char err_msg[1024] = {'\0'}; + FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM, + NULL, error, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + (LPTSTR)err_msg, sizeof(err_msg), NULL ); + + return (err_msg); +} +#endif /* _WIN32 */ + +/***********************************************//* + @param>>_______[in] name>_____name of file. + @retval file pointer; file pointer is NULL when error occured. +*/ + +FILE* +open_file( + const char* name) +{ + int fd; /* file descriptor. */ + FILE* fil_in; +#ifdef _WIN32 + HANDLE hFile; /* handle to open file. */ + DWORD access; /* define access control */ + int flags = 0; /* define the mode for file + descriptor */ + + if (do_write) { + access = GENERIC_READ | GENERIC_WRITE; + flags = _O_RDWR | _O_BINARY; + } else { + access = GENERIC_READ; + flags = _O_RDONLY | _O_BINARY; + } + + /* CreateFile() also provide advisory lock with the usage of + access and share mode of the file.*/ + hFile = CreateFile( + (LPCTSTR) name, access, 0L, NULL, + OPEN_EXISTING, NULL, NULL); + + if (hFile == INVALID_HANDLE_VALUE) { + /* print the error message. */ + fprintf(stderr, "Filename::%s %s\n", name, + error_message(GetLastError())); + + return (NULL); + } + + /* get the file descriptor. */ + fd= _open_osfhandle((intptr_t)hFile, flags); +#else /* _WIN32 */ + + int create_flag; + /* define the advisory lock and open file mode. */ + if (do_write) { + create_flag = O_RDWR; + lk.l_type = F_WRLCK; + } else { + create_flag = O_RDONLY; + lk.l_type = F_RDLCK; + } + + fd = open(name, create_flag); + + lk.l_whence = SEEK_SET; + lk.l_start = lk.l_len = 0; + + if (fcntl(fd, F_SETLK, &lk) == -1) { + fprintf(stderr, "Error: Unable to lock file::" + " %s\n", name); + perror("fcntl"); + return (NULL); + } +#endif /* _WIN32 */ + + if (do_write) { + fil_in = fdopen(fd, "rb+"); + } else { + fil_in = fdopen(fd, "rb"); + } + + return (fil_in); +} + +/************************************************************//* + Read the content of file + + @param [in,out] buf read the file in buffer + @param [in] partial_page_read enable when to read the + remaining buffer for first page. + @param [in] physical_page_size Physical/Commpressed page size. + @param [in,out] fil_in file pointer created for the + tablespace. + @retval no. of bytes read. +*/ +ulong read_file( + byte* buf, + bool partial_page_read, + ulong physical_page_size, + FILE* fil_in) +{ + ulong bytes = 0; + + DBUG_ASSERT(physical_page_size >= UNIV_ZIP_SIZE_MIN); + + if (partial_page_read) { + buf += UNIV_ZIP_SIZE_MIN; + physical_page_size -= UNIV_ZIP_SIZE_MIN; + bytes = UNIV_ZIP_SIZE_MIN; + } + + bytes += ulong(fread(buf, 1, physical_page_size, fil_in)); + + return bytes; +} + +/** Check if page is corrupted or not. +@param[in] buf page frame +@param[in] page_size page size +@param[in] zip_size != if page row compressed +@param[in] is_encrypted true if page0 contained cryp_data + with crypt_scheme encrypted +@param[in] is_compressed true if page0 fsp_flags contained + page compression flag +@retval true if page is corrupted otherwise false. */ +static +bool +is_page_corrupted( + byte* buf, + ulint page_size, + ulint zip_size, + bool is_encrypted, + bool is_compressed) +{ + + /* enable if page is corrupted. */ + bool is_corrupted; + /* use to store LSN values. */ + ulint logseq; + ulint logseqfield; + ulint page_type = mach_read_from_2(buf+FIL_PAGE_TYPE); + ulint key_version = mach_read_from_4(buf+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); + ulint space_id = mach_read_from_4( + buf + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); + + /* We can't trust only a page type, thus we take account + also fsp_flags or crypt_data on page 0 */ + if ((page_type == FIL_PAGE_PAGE_COMPRESSED && is_compressed) || + (page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED && + is_compressed && is_encrypted)) { + /* Page compressed tables do not contain post compression + checksum. */ + return (false); + } + + if (!zip_size) { + /* check the stored log sequence numbers + for uncompressed tablespace. */ + logseq = mach_read_from_4(buf + FIL_PAGE_LSN + 4); + logseqfield = mach_read_from_4( + buf + page_size - + FIL_PAGE_END_LSN_OLD_CHKSUM + 4); + + if (is_log_enabled) { + fprintf(log_file, + "space::%" PRIuMAX " page::%" PRIuMAX + "; log sequence number:first = " ULINTPF + "; second = " ULINTPF "\n", + space_id, cur_page_num, logseq, logseqfield); + if (logseq != logseqfield) { + fprintf(log_file, + "Fail; space %" PRIuMAX " page %" PRIuMAX + " invalid (fails log " + "sequence number check)\n", + space_id, cur_page_num); + } + } + } + + /* Again we can't trust only FIL_PAGE_FILE_FLUSH_LSN field + now repurposed as FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, + we need to check also crypt_data contents. */ + if (is_encrypted && key_version != 0) { + is_corrupted = !fil_space_verify_crypt_checksum(buf, + zip_size, NULL, cur_page_num); + } else { + is_corrupted = buf_page_is_corrupted( + true, buf, zip_size, NULL); + } + + return(is_corrupted); +} + +/********************************************//* + Check if page is doublewrite buffer or not. + @param [in] page buffer page + + @retval true if page is doublewrite buffer otherwise false. +*/ +static +bool +is_page_doublewritebuffer( + const byte* page) +{ + if ((cur_page_num >= FSP_EXTENT_SIZE) + && (cur_page_num < FSP_EXTENT_SIZE * 3)) { + /* page is doublewrite buffer. */ + return (true); + } + + return (false); +} + +/*******************************************************//* +Check if page is empty or not. + @param [in] page page to checked for empty. + @param [in] len size of page. + + @retval true if page is empty. + @retval false if page is not empty. +*/ +static +bool +is_page_empty( + const byte* page, + size_t len) +{ + while (len--) { + if (*page++) { + return (false); + } + } + return (true); +} + +/********************************************************************//** +Rewrite the checksum for the page. +@param [in/out] page page buffer +@param [in] physical_page_size page size in bytes on disk. +@param [in] iscompressed Is compressed/Uncompressed Page. + +@retval true : do rewrite +@retval false : skip the rewrite as checksum stored match with + calculated or page is doublwrite buffer. +*/ + +bool +update_checksum( + byte* page, + ulong physical_page_size, + bool iscompressed) +{ + ib_uint32_t checksum = 0; + byte stored1[4]; /* get FIL_PAGE_SPACE_OR_CHKSUM field checksum */ + byte stored2[4]; /* get FIL_PAGE_END_LSN_OLD_CHKSUM field checksum */ + + ut_ad(page); + /* If page is doublewrite buffer, skip the rewrite of checksum. */ + if (skip_page) { + return (false); + } + + memcpy(stored1, page + FIL_PAGE_SPACE_OR_CHKSUM, 4); + memcpy(stored2, page + physical_page_size - + FIL_PAGE_END_LSN_OLD_CHKSUM, 4); + + /* Check if page is empty, exclude the checksum field */ + if (is_page_empty(page + 4, physical_page_size - 12) + && is_page_empty(page + physical_page_size - 4, 4)) { + + memset(page + FIL_PAGE_SPACE_OR_CHKSUM, 0, 4); + memset(page + physical_page_size - + FIL_PAGE_END_LSN_OLD_CHKSUM, 0, 4); + + goto func_exit; + } + + if (iscompressed) { + /* page is compressed */ + checksum = page_zip_calc_checksum( + page, physical_page_size, + static_cast(write_check)); + + mach_write_to_4(page + FIL_PAGE_SPACE_OR_CHKSUM, checksum); + if (is_log_enabled) { + fprintf(log_file, "page::%" PRIuMAX "; Updated checksum =" + " %u\n", cur_page_num, checksum); + } + + } else { + /* page is uncompressed. */ + + /* Store the new formula checksum */ + switch ((srv_checksum_algorithm_t) write_check) { + + case SRV_CHECKSUM_ALGORITHM_CRC32: + case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32: + checksum = buf_calc_page_crc32(page); + break; + + case SRV_CHECKSUM_ALGORITHM_INNODB: + case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB: + checksum = (ib_uint32_t) + buf_calc_page_new_checksum(page); + break; + + case SRV_CHECKSUM_ALGORITHM_NONE: + case SRV_CHECKSUM_ALGORITHM_STRICT_NONE: + checksum = BUF_NO_CHECKSUM_MAGIC; + break; + /* no default so the compiler will emit a warning if new + enum is added and not handled here */ + } + + mach_write_to_4(page + FIL_PAGE_SPACE_OR_CHKSUM, checksum); + if (is_log_enabled) { + fprintf(log_file, "page::%" PRIuMAX "; Updated checksum field1" + " = %u\n", cur_page_num, checksum); + } + + if (write_check == SRV_CHECKSUM_ALGORITHM_STRICT_INNODB + || write_check == SRV_CHECKSUM_ALGORITHM_INNODB) { + checksum = (ib_uint32_t) + buf_calc_page_old_checksum(page); + } + + mach_write_to_4(page + physical_page_size - + FIL_PAGE_END_LSN_OLD_CHKSUM,checksum); + + if (is_log_enabled) { + fprintf(log_file, "page::%" PRIuMAX "; Updated checksum " + "field2 = %u\n", cur_page_num, checksum); + } + + } + + func_exit: + /* The following code is to check the stored checksum with the + calculated checksum. If it matches, then return FALSE to skip + the rewrite of checksum, otherwise return TRUE. */ + if (iscompressed) { + if (!memcmp(stored1, page + FIL_PAGE_SPACE_OR_CHKSUM, 4)) { + return (false); + } + return (true); + } + + if (!memcmp(stored1, page + FIL_PAGE_SPACE_OR_CHKSUM, 4) + && !memcmp(stored2, page + physical_page_size - + FIL_PAGE_END_LSN_OLD_CHKSUM, 4)) { + return (false); + + } + + return (true); +} + +/** + Write the content to the file +@param[in] filename name of the file. +@param[in,out] file file pointer where content + have to be written +@param[in] buf file buffer read +@param[in] compressed Enabled if tablespace is + compressed. +@param[in,out] pos current file position. +@param[in] page_size page size in bytes on disk. + +@retval true if successfully written +@retval false if a non-recoverable error occurred +*/ +static +bool +write_file( + const char* filename, + FILE* file, + byte* buf, + bool compressed, + fpos_t* pos, + ulong page_size) +{ + bool do_update; + + do_update = update_checksum(buf, page_size, compressed); + + if (file != stdin) { + if (do_update) { + /* Set the previous file pointer position + saved in pos to current file position. */ + if (0 != fsetpos(file, pos)) { + perror("fsetpos"); + return(false); + } + } else { + /* Store the current file position in pos */ + if (0 != fgetpos(file, pos)) { + perror("fgetpos"); + return(false); + } + return(true); + } + } + + if (page_size + != fwrite(buf, 1, page_size, file == stdin ? stdout : file)) { + fprintf(stderr, "Failed to write page %" PRIuMAX " to %s: %s\n", + cur_page_num, filename, strerror(errno)); + + return(false); + } + if (file != stdin) { + fflush(file); + /* Store the current file position in pos */ + if (0 != fgetpos(file, pos)) { + perror("fgetpos"); + return(false); + } + } + + return(true); +} + +/* +Parse the page and collect/dump the information about page type +@param [in] page buffer page +@param [out] xdes extend descriptor page +@param [in] file file for diagnosis. +*/ +void +parse_page( + const byte* page, + byte* xdes, + FILE* file) +{ + unsigned long long id; + ulint undo_page_type; + char str[20]={'\0'}; + ulint n_recs; + ulint page_no; + ulint left_page_no; + ulint right_page_no; + ulint data_bytes; + int is_leaf; + int size_range_id; + + /* Check whether page is doublewrite buffer. */ + if(skip_page) { + strcpy(str, "Double_write_buffer"); + } else { + strcpy(str, "-"); + } + + switch (mach_read_from_2(page + FIL_PAGE_TYPE)) { + + case FIL_PAGE_INDEX: + page_type.n_fil_page_index++; + id = mach_read_from_8(page + PAGE_HEADER + PAGE_INDEX_ID); + n_recs = page_get_n_recs(page); + page_no = page_get_page_no(page); + left_page_no = mach_read_from_4(page + FIL_PAGE_PREV); + right_page_no = mach_read_from_4(page + FIL_PAGE_NEXT); + data_bytes = page_get_data_size(page); + is_leaf = page_is_leaf(page); + size_range_id = (data_bytes * SIZE_RANGES_FOR_PAGE + + logical_page_size - 1) / + logical_page_size; + if (size_range_id > SIZE_RANGES_FOR_PAGE + 1) { + /* data_bytes is bigger than logical_page_size */ + size_range_id = SIZE_RANGES_FOR_PAGE + 1; + } + if (per_page_details) { + printf("index %llu page %lu leaf %u n_recs %lu data_bytes %lu" + "\n", id, page_no, is_leaf, n_recs, data_bytes); + } + /* update per-index statistics */ + { + if (index_ids.count(id) == 0) { + index_ids[id] = per_index_stats(); + } + std::map::iterator it; + it = index_ids.find(id); + per_index_stats &index = (it->second); + const byte* des = xdes + XDES_ARR_OFFSET + + XDES_SIZE * ((page_no & (physical_page_size - 1)) + / FSP_EXTENT_SIZE); + if (xdes_get_bit(des, XDES_FREE_BIT, + page_no % FSP_EXTENT_SIZE)) { + index.free_pages++; + return; + } + index.pages++; + if (is_leaf) { + index.leaf_pages++; + if (data_bytes > index.max_data_size) { + index.max_data_size = data_bytes; + } + struct per_page_stats pp(n_recs, data_bytes, + left_page_no, right_page_no); + + index.leaves[page_no] = pp; + + if (left_page_no == ULINT32_UNDEFINED) { + index.first_leaf_page = page_no; + index.count++; + } + } + index.total_n_recs += n_recs; + index.total_data_bytes += data_bytes; + index.pages_in_size_range[size_range_id] ++; + } + + if (page_type_dump) { + fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tIndex page\t\t\t|" + "\tindex id=%llu,", cur_page_num, id); + + fprintf(file, + " page level=" ULINTPF + ", No. of records=" ULINTPF + ", garbage=" ULINTPF ", %s\n", + page_header_get_field(page, PAGE_LEVEL), + page_header_get_field(page, PAGE_N_RECS), + page_header_get_field(page, PAGE_GARBAGE), str); + } + break; + + case FIL_PAGE_UNDO_LOG: + page_type.n_fil_page_undo_log++; + undo_page_type = mach_read_from_2(page + + TRX_UNDO_PAGE_HDR + TRX_UNDO_PAGE_TYPE); + if (page_type_dump) { + fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tUndo log page\t\t\t|", + cur_page_num); + } + if (undo_page_type == TRX_UNDO_INSERT) { + page_type.n_undo_insert++; + if (page_type_dump) { + fprintf(file, "\t%s", + "Insert Undo log page"); + } + + } else if (undo_page_type == TRX_UNDO_UPDATE) { + page_type.n_undo_update++; + if (page_type_dump) { + fprintf(file, "\t%s", + "Update undo log page"); + } + } + + undo_page_type = mach_read_from_2(page + TRX_UNDO_SEG_HDR + + TRX_UNDO_STATE); + switch (undo_page_type) { + case TRX_UNDO_ACTIVE: + page_type.n_undo_state_active++; + if (page_type_dump) { + fprintf(file, ", %s", "Undo log of " + "an active transaction"); + } + break; + + case TRX_UNDO_CACHED: + page_type.n_undo_state_cached++; + if (page_type_dump) { + fprintf(file, ", %s", "Page is " + "cached for quick reuse"); + } + break; + + case TRX_UNDO_TO_FREE: + page_type.n_undo_state_to_free++; + if (page_type_dump) { + fprintf(file, ", %s", "Insert undo " + "segment that can be freed"); + } + break; + + case TRX_UNDO_TO_PURGE: + page_type.n_undo_state_to_purge++; + if (page_type_dump) { + fprintf(file, ", %s", "Will be " + "freed in purge when all undo" + "data in it is removed"); + } + break; + + case TRX_UNDO_PREPARED: + page_type.n_undo_state_prepared++; + if (page_type_dump) { + fprintf(file, ", %s", "Undo log of " + "an prepared transaction"); + } + break; + + default: + page_type.n_undo_state_other++; + break; + } + if(page_type_dump) { + fprintf(file, ", %s\n", str); + } + break; + + case FIL_PAGE_INODE: + page_type.n_fil_page_inode++; + if (page_type_dump) { + fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tInode page\t\t\t|" + "\t%s\n",cur_page_num, str); + } + break; + + case FIL_PAGE_IBUF_FREE_LIST: + page_type.n_fil_page_ibuf_free_list++; + if (page_type_dump) { + fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tInsert buffer free list" + " page\t|\t%s\n", cur_page_num, str); + } + break; + + case FIL_PAGE_TYPE_ALLOCATED: + page_type.n_fil_page_type_allocated++; + if (page_type_dump) { + fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tFreshly allocated " + "page\t\t|\t%s\n", cur_page_num, str); + } + break; + + case FIL_PAGE_IBUF_BITMAP: + page_type.n_fil_page_ibuf_bitmap++; + if (page_type_dump) { + fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tInsert Buffer " + "Bitmap\t\t|\t%s\n", cur_page_num, str); + } + break; + + case FIL_PAGE_TYPE_SYS: + page_type.n_fil_page_type_sys++; + if (page_type_dump) { + fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tSystem page\t\t\t|" + "\t%s\n",cur_page_num, str); + } + break; + + case FIL_PAGE_TYPE_TRX_SYS: + page_type.n_fil_page_type_trx_sys++; + if (page_type_dump) { + fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tTransaction system " + "page\t\t|\t%s\n", cur_page_num, str); + } + break; + + case FIL_PAGE_TYPE_FSP_HDR: + page_type.n_fil_page_type_fsp_hdr++; + memcpy(xdes, page, physical_page_size); + if (page_type_dump) { + fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tFile Space " + "Header\t\t|\t%s\n", cur_page_num, str); + } + break; + + case FIL_PAGE_TYPE_XDES: + page_type.n_fil_page_type_xdes++; + memcpy(xdes, page, physical_page_size); + if (page_type_dump) { + fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tExtent descriptor " + "page\t\t|\t%s\n", cur_page_num, str); + } + break; + + case FIL_PAGE_TYPE_BLOB: + page_type.n_fil_page_type_blob++; + if (page_type_dump) { + fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tBLOB page\t\t\t|\t%s\n", + cur_page_num, str); + } + break; + + case FIL_PAGE_TYPE_ZBLOB: + page_type.n_fil_page_type_zblob++; + if (page_type_dump) { + fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tCompressed BLOB " + "page\t\t|\t%s\n", cur_page_num, str); + } + break; + + case FIL_PAGE_TYPE_ZBLOB2: + page_type.n_fil_page_type_zblob2++; + if (page_type_dump) { + fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tSubsequent Compressed " + "BLOB page\t|\t%s\n", cur_page_num, str); + } + break; + + case FIL_PAGE_PAGE_COMPRESSED: + page_type.n_fil_page_type_page_compressed++; + if (page_type_dump) { + fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tPage compressed " + "page\t|\t%s\n", cur_page_num, str); + } + break; + + case FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED: + page_type.n_fil_page_type_page_compressed_encrypted++; + if (page_type_dump) { + fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tPage compressed encrypted " + "page\t|\t%s\n", cur_page_num, str); + } + break; + default: + page_type.n_fil_page_type_other++; + break; + } +} +/** +@param [in/out] file_name name of the filename + +@retval FILE pointer if successfully created else NULL when error occured. +*/ +FILE* +create_file( + char* file_name) +{ + FILE* file = NULL; + +#ifndef _WIN32 + file = fopen(file_name, "wb"); + if (file == NULL) { + fprintf(stderr, "Failed to create file: %s: %s\n", + file_name, strerror(errno)); + return(NULL); + } +#else + HANDLE hFile; /* handle to open file. */ + int fd = 0; + hFile = CreateFile((LPCTSTR) file_name, + GENERIC_READ | GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_DELETE, + NULL, CREATE_NEW, NULL, NULL); + + if (hFile == INVALID_HANDLE_VALUE) { + /* print the error message. */ + fprintf(stderr, "Filename::%s %s\n", + file_name, + error_message(GetLastError())); + + return(NULL); + } + + /* get the file descriptor. */ + fd= _open_osfhandle((intptr_t)hFile, _O_RDWR | _O_BINARY); + file = fdopen(fd, "wb"); +#endif /* _WIN32 */ + + return(file); +} + +/* + Print the page type count of a tablespace. + @param [in] fil_out stream where the output goes. +*/ +void +print_summary( + FILE* fil_out) +{ + fprintf(fil_out, "\n================PAGE TYPE SUMMARY==============\n"); + fprintf(fil_out, "#PAGE_COUNT\tPAGE_TYPE"); + fprintf(fil_out, "\n===============================================\n"); + fprintf(fil_out, "%8d\tIndex page\n", + page_type.n_fil_page_index); + fprintf(fil_out, "%8d\tUndo log page\n", + page_type.n_fil_page_undo_log); + fprintf(fil_out, "%8d\tInode page\n", + page_type.n_fil_page_inode); + fprintf(fil_out, "%8d\tInsert buffer free list page\n", + page_type.n_fil_page_ibuf_free_list); + fprintf(fil_out, "%8d\tFreshly allocated page\n", + page_type.n_fil_page_type_allocated); + fprintf(fil_out, "%8d\tInsert buffer bitmap\n", + page_type.n_fil_page_ibuf_bitmap); + fprintf(fil_out, "%8d\tSystem page\n", + page_type.n_fil_page_type_sys); + fprintf(fil_out, "%8d\tTransaction system page\n", + page_type.n_fil_page_type_trx_sys); + fprintf(fil_out, "%8d\tFile Space Header\n", + page_type.n_fil_page_type_fsp_hdr); + fprintf(fil_out, "%8d\tExtent descriptor page\n", + page_type.n_fil_page_type_xdes); + fprintf(fil_out, "%8d\tBLOB page\n", + page_type.n_fil_page_type_blob); + fprintf(fil_out, "%8d\tCompressed BLOB page\n", + page_type.n_fil_page_type_zblob); + fprintf(fil_out, "%8d\tOther type of page\n", + page_type.n_fil_page_type_other); + fprintf(fil_out, "%8d\tPage compressed page\n", + page_type.n_fil_page_type_page_compressed); + fprintf(fil_out, "%8d\tPage compressed encrypted page\n", + page_type.n_fil_page_type_page_compressed_encrypted); + fprintf(fil_out, "\n===============================================\n"); + fprintf(fil_out, "Additional information:\n"); + fprintf(fil_out, "Undo page type: %d insert, %d update, %d other\n", + page_type.n_undo_insert, + page_type.n_undo_update, + page_type.n_undo_other); + fprintf(fil_out, "Undo page state: %d active, %d cached, %d to_free, %d" + " to_purge, %d prepared, %d other\n", + page_type.n_undo_state_active, + page_type.n_undo_state_cached, + page_type.n_undo_state_to_free, + page_type.n_undo_state_to_purge, + page_type.n_undo_state_prepared, + page_type.n_undo_state_other); + + fprintf(fil_out, "index_id\t#pages\t\t#leaf_pages\t#recs_per_page" + "\t#bytes_per_page\n"); + + for (std::map::const_iterator it = index_ids.begin(); + it != index_ids.end(); it++) { + const per_index_stats& index = it->second; + fprintf(fil_out, "%lld\t\t%lld\t\t%lld\t\t%lld\t\t%lld\n", + it->first, index.pages, index.leaf_pages, + index.total_n_recs / index.pages, + index.total_data_bytes / index.pages); + } + + fprintf(fil_out, "\n"); + fprintf(fil_out, "index_id\tpage_data_bytes_histgram(empty,...,oversized)\n"); + + for (std::map::const_iterator it = index_ids.begin(); + it != index_ids.end(); it++) { + fprintf(fil_out, "%lld\t", it->first); + const per_index_stats& index = it->second; + for (ulint i = 0; i < SIZE_RANGES_FOR_PAGE+2; i++) { + fprintf(fil_out, "\t%lld", index.pages_in_size_range[i]); + } + fprintf(fil_out, "\n"); + } + + if (do_leaf) { + print_leaf_stats(fil_out); + } +} + +/* command line argument for innochecksum tool. */ +static struct my_option innochecksum_options[] = { {"help", '?', "Displays this help and exits.", 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, {"info", 'I', "Synonym for --help.", @@ -221,864 +1157,664 @@ static struct my_option innochecksum_options[] = 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, {"verbose", 'v', "Verbose (prints progress every 5 seconds).", &verbose, &verbose, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, - {"debug", 'd', "Debug mode (prints checksums for each page, implies verbose).", - &debug, &debug, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, - {"skip_corrupt", 'u', "Skip corrupt pages.", - &skip_corrupt, &skip_corrupt, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, - {"count", 'c', "Print the count of pages in the file.", +#ifndef DBUG_OFF + {"debug", '#', "Output debug log. See " REFMAN "dbug-package.html", + &dbug_setting, &dbug_setting, 0, GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0}, +#endif /* !DBUG_OFF */ + {"count", 'c', "Print the count of pages in the file and exits.", &just_count, &just_count, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, {"start_page", 's', "Start on this page number (0 based).", - &start_page, &start_page, 0, GET_ULONG, REQUIRED_ARG, - 0, 0, (longlong) 2L*1024L*1024L*1024L, 0, 1, 0}, + &start_page, &start_page, 0, GET_ULL, REQUIRED_ARG, + 0, 0, ULLONG_MAX, 0, 1, 0}, {"end_page", 'e', "End at this page number (0 based).", - &end_page, &end_page, 0, GET_ULONG, REQUIRED_ARG, - 0, 0, (longlong) 2L*1024L*1024L*1024L, 0, 1, 0}, + &end_page, &end_page, 0, GET_ULL, REQUIRED_ARG, + 0, 0, ULLONG_MAX, 0, 1, 0}, {"page", 'p', "Check only this page (0 based).", - &do_page, &do_page, 0, GET_ULONG, REQUIRED_ARG, - 0, 0, (longlong) 2L*1024L*1024L*1024L, 0, 1, 0}, - {"per_page_details", 'i', "Print out per-page detail information.", - &per_page_details, &per_page_details, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0} - , - {"leaf", 'l', "Examine leaf index pages", + &do_page, &do_page, 0, GET_ULL, REQUIRED_ARG, + 0, 0, ULLONG_MAX, 0, 1, 0}, + {"strict-check", 'C', "Specify the strict checksum algorithm by the user.", + &strict_check, &strict_check, &innochecksum_algorithms_typelib, + GET_ENUM, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"no-check", 'n', "Ignore the checksum verification.", + &no_check, &no_check, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"allow-mismatches", 'a', "Maximum checksum mismatch allowed.", + &allow_mismatches, &allow_mismatches, 0, + GET_ULL, REQUIRED_ARG, 0, 0, ULLONG_MAX, 0, 1, 0}, + {"write", 'w', "Rewrite the checksum algorithm by the user.", + &write_check, &write_check, &innochecksum_algorithms_typelib, + GET_ENUM, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"page-type-summary", 'S', "Display a count of each page type " + "in a tablespace.", &page_type_summary, &page_type_summary, 0, + GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"page-type-dump", 'D', "Dump the page type info for each page in a " + "tablespace.", &page_dump_filename, &page_dump_filename, 0, + GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"per-page-details", 'i', "Print out per-page detail information.", + &per_page_details, &per_page_details, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"log", 'l', "log output.", + &log_filename, &log_filename, 0, + GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"leaf", 'f', "Examine leaf index pages", &do_leaf, &do_leaf, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, {"merge", 'm', "leaf page count if merge given number of consecutive pages", - &n_merge, &n_merge, 0, GET_ULONG, REQUIRED_ARG, - 0, 0, (longlong)10L, 0, 1, 0}, + &n_merge, &n_merge, 0, GET_ULONG, REQUIRED_ARG, 0, 0, (longlong)10L, 0, 1, 0}, + {0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0} }; +/* Print out the Innodb version and machine information. */ static void print_version(void) { - printf("%s Ver %s, for %s (%s)\n", - my_progname, INNODB_VERSION_STR, - SYSTEM_TYPE, MACHINE_TYPE); +#ifdef DBUG_OFF + printf("%s Ver %s, for %s (%s)\n", + my_progname, INNODB_VERSION_STR, + SYSTEM_TYPE, MACHINE_TYPE); +#else + printf("%s-debug Ver %s, for %s (%s)\n", + my_progname, INNODB_VERSION_STR, + SYSTEM_TYPE, MACHINE_TYPE); +#endif /* DBUG_OFF */ } static void usage(void) { - print_version(); - puts(ORACLE_WELCOME_COPYRIGHT_NOTICE("2000")); - printf("InnoDB offline file checksum utility.\n"); - printf("Usage: %s [-c] [-s ] [-e ] [-p ] [-v] [-d] \n", my_progname); - my_print_help(innochecksum_options); - my_print_variables(innochecksum_options); + print_version(); + puts(ORACLE_WELCOME_COPYRIGHT_NOTICE("2000")); + printf("InnoDB offline file checksum utility.\n"); + printf("Usage: %s [-c] [-s ] [-e ] " + "[-p ] [-i] [-v] [-a ] [-n] " + "[-C ] [-w ] [-S] [-D ] " + "[-l ] [-l] [-m ] \n", my_progname); + printf("See " REFMAN "innochecksum.html for usage hints.\n"); + my_print_help(innochecksum_options); + my_print_variables(innochecksum_options); } extern "C" my_bool innochecksum_get_one_option( -/*========================*/ - int optid, - const struct my_option *opt __attribute__((unused)), - char *argument __attribute__((unused))) + int optid, + const struct my_option *opt MY_ATTRIBUTE((unused)), + char *argument MY_ATTRIBUTE((unused))) { - switch (optid) { - case 'd': - verbose=1; /* debug implies verbose... */ - break; - case 'e': - use_end_page= 1; - break; - case 'p': - end_page= start_page= do_page; - use_end_page= 1; - do_one_page= 1; - break; - case 'V': - print_version(); - exit(0); - break; - case 'I': - case '?': - usage(); - exit(0); - break; - } - return 0; -} + switch (optid) { +#ifndef DBUG_OFF + case '#': + dbug_setting = argument + ? argument + : IF_WIN("d:O,innochecksum.trace", + "d:o,/tmp/innochecksum.trace"); + DBUG_PUSH(dbug_setting); + break; +#endif /* !DBUG_OFF */ + case 'e': + use_end_page = true; + break; + case 'p': + end_page = start_page = do_page; + use_end_page = true; + do_one_page = true; + break; + case 'V': + print_version(); + exit(EXIT_SUCCESS); + break; + case 'C': + strict_verify = true; + switch ((srv_checksum_algorithm_t) strict_check) { -static int get_options( -/*===================*/ - int *argc, - char ***argv) -{ - int ho_error; + case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32: + case SRV_CHECKSUM_ALGORITHM_CRC32: + srv_checksum_algorithm = + SRV_CHECKSUM_ALGORITHM_STRICT_CRC32; + break; - if ((ho_error=handle_options(argc, argv, innochecksum_options, innochecksum_get_one_option))) - exit(ho_error); + case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB: + case SRV_CHECKSUM_ALGORITHM_INNODB: + srv_checksum_algorithm = + SRV_CHECKSUM_ALGORITHM_STRICT_INNODB; + break; - /* The next arg must be the filename */ - if (!*argc) - { - usage(); - return 1; - } - return 0; -} /* get_options */ - -/*********************************************************************//** -Gets the file page type. -@return type; NOTE that if the type has not been written to page, the -return value not defined */ -ulint -fil_page_get_type( -/*==============*/ - uchar* page) /*!< in: file page */ -{ - return(mach_read_from_2(page + FIL_PAGE_TYPE)); -} - -/**************************************************************//** -Gets the index id field of a page. -@return index id */ -ib_uint64_t -btr_page_get_index_id( -/*==================*/ - uchar* page) /*!< in: index page */ -{ - return(mach_read_from_8(page + PAGE_HEADER + PAGE_INDEX_ID)); -} - -/********************************************************//** -Gets the next index page number. -@return next page number */ -ulint -btr_page_get_next( -/*==============*/ - const page_t* page) /*!< in: index page */ -{ - return(mach_read_from_4(page + FIL_PAGE_NEXT)); -} - -/********************************************************//** -Gets the previous index page number. -@return prev page number */ -ulint -btr_page_get_prev( -/*==============*/ - const page_t* page) /*!< in: index page */ -{ - return(mach_read_from_4(page + FIL_PAGE_PREV)); -} - -void -parse_page( -/*=======*/ - uchar* page, /* in: buffer page */ - uchar* xdes) /* in: extend descriptor page */ -{ - ib_uint64_t id; - ulint x; - ulint n_recs; - ulint page_no; - ulint left_page_no; - ulint right_page_no; - ulint data_bytes; - int is_leaf; - int size_range_id; - - switch (fil_page_get_type(page)) { - case FIL_PAGE_INDEX: - n_fil_page_index++; - id = btr_page_get_index_id(page); - n_recs = page_get_n_recs(page); - page_no = page_get_page_no(page); - left_page_no = btr_page_get_prev(page); - right_page_no = btr_page_get_next(page); - data_bytes = page_get_data_size(page); - is_leaf = page_is_leaf(page); - size_range_id = (data_bytes * SIZE_RANGES_FOR_PAGE - + logical_page_size - 1) / - logical_page_size; - if (size_range_id > SIZE_RANGES_FOR_PAGE + 1) { - /* data_bytes is bigger than logical_page_size */ - size_range_id = SIZE_RANGES_FOR_PAGE + 1; - } - if (per_page_details) { - printf("index " IB_ID_FMT " page " ULINTPF - " leaf %d n_recs " ULINTPF " data_bytes " ULINTPF - "\n", id, page_no, is_leaf, n_recs, data_bytes); - } - /* update per-index statistics */ - { - if (index_ids.count(id) == 0) { - index_ids[id] = per_index_stats(); - } - std::map::iterator it; - it = index_ids.find(id); - per_index_stats &index = (it->second); - uchar* des = xdes + XDES_ARR_OFFSET - + XDES_SIZE * ((page_no & (physical_page_size - 1)) - / FSP_EXTENT_SIZE); - if (xdes_get_bit(des, XDES_FREE_BIT, - page_no % FSP_EXTENT_SIZE)) { - index.free_pages++; - return; - } - index.pages++; - if (is_leaf) { - index.leaf_pages++; - if (data_bytes > index.max_data_size) { - index.max_data_size = data_bytes; - } - struct per_page_stats pp(n_recs, data_bytes, - left_page_no, right_page_no); - - index.leaves[page_no] = pp; - - if (left_page_no == ULINT32_UNDEFINED) { - index.first_leaf_page = page_no; - index.count++; - } - } - index.total_n_recs += n_recs; - index.total_data_bytes += data_bytes; - index.pages_in_size_range[size_range_id] ++; - } - - break; - case FIL_PAGE_UNDO_LOG: - if (per_page_details) { - printf("FIL_PAGE_UNDO_LOG\n"); - } - n_fil_page_undo_log++; - x = mach_read_from_2(page + TRX_UNDO_PAGE_HDR + - TRX_UNDO_PAGE_TYPE); - if (x == TRX_UNDO_INSERT) - n_undo_insert++; - else if (x == TRX_UNDO_UPDATE) - n_undo_update++; - else - n_undo_other++; - - x = mach_read_from_2(page + TRX_UNDO_SEG_HDR + TRX_UNDO_STATE); - switch (x) { - case TRX_UNDO_ACTIVE: n_undo_state_active++; break; - case TRX_UNDO_CACHED: n_undo_state_cached++; break; - case TRX_UNDO_TO_FREE: n_undo_state_to_free++; break; - case TRX_UNDO_TO_PURGE: n_undo_state_to_purge++; break; - case TRX_UNDO_PREPARED: n_undo_state_prepared++; break; - default: n_undo_state_other++; break; - } - break; - case FIL_PAGE_INODE: - if (per_page_details) { - printf("FIL_PAGE_INODE\n"); - } - n_fil_page_inode++; - break; - case FIL_PAGE_IBUF_FREE_LIST: - if (per_page_details) { - printf("FIL_PAGE_IBUF_FREE_LIST\n"); - } - n_fil_page_ibuf_free_list++; - break; - case FIL_PAGE_TYPE_ALLOCATED: - if (per_page_details) { - printf("FIL_PAGE_TYPE_ALLOCATED\n"); - } - n_fil_page_type_allocated++; - break; - case FIL_PAGE_IBUF_BITMAP: - if (per_page_details) { - printf("FIL_PAGE_IBUF_BITMAP\n"); - } - n_fil_page_ibuf_bitmap++; - break; - case FIL_PAGE_TYPE_SYS: - if (per_page_details) { - printf("FIL_PAGE_TYPE_SYS\n"); - } - n_fil_page_type_sys++; - break; - case FIL_PAGE_TYPE_TRX_SYS: - if (per_page_details) { - printf("FIL_PAGE_TYPE_TRX_SYS\n"); - } - n_fil_page_type_trx_sys++; - break; - case FIL_PAGE_TYPE_FSP_HDR: - if (per_page_details) { - printf("FIL_PAGE_TYPE_FSP_HDR\n"); - } - memcpy(xdes, page, physical_page_size); - n_fil_page_type_fsp_hdr++; - break; - case FIL_PAGE_TYPE_XDES: - if (per_page_details) { - printf("FIL_PAGE_TYPE_XDES\n"); - } - memcpy(xdes, page, physical_page_size); - n_fil_page_type_xdes++; - break; - case FIL_PAGE_TYPE_BLOB: - if (per_page_details) { - printf("FIL_PAGE_TYPE_BLOB\n"); - } - n_fil_page_type_blob++; - break; - case FIL_PAGE_TYPE_ZBLOB: - case FIL_PAGE_TYPE_ZBLOB2: - if (per_page_details) { - printf("FIL_PAGE_TYPE_ZBLOB/2\n"); - } - n_fil_page_type_zblob++; - break; - case FIL_PAGE_PAGE_COMPRESSED: - if (per_page_details) { - printf("FIL_PAGE_PAGE_COMPRESSED\n"); - } - n_fil_page_type_page_compressed++; - break; - case FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED: - if (per_page_details) { - printf("FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED\n"); - } - n_fil_page_type_page_compressed_encrypted++; - break; - default: - if (per_page_details) { - printf("FIL_PAGE_TYPE_OTHER\n"); - } - n_fil_page_type_other++; - } -} - -void print_index_leaf_stats(unsigned long long id, const per_index_stats& index) -{ - ulint page_no = index.first_leaf_page; - std::map::const_iterator it_page = index.leaves.find(page_no); - printf("\nindex: %llu leaf page stats: n_pages = %llu\n", - id, index.leaf_pages); - printf("page_no\tdata_size\tn_recs\n"); - while (it_page != index.leaves.end()) { - const per_page_stats& stat = it_page->second; - printf(ULINTPF "\t" ULINTPF "\t" ULINTPF "\n", - it_page->first, stat.data_size, stat.n_recs); - page_no = stat.right_page_no; - it_page = index.leaves.find(page_no); - } -} - -void defrag_analysis(unsigned long long id, const per_index_stats& index) -{ - // TODO: make it work for compressed pages too - std::map::const_iterator it = index.leaves.find(index.first_leaf_page); - ulint n_pages = 0; - ulint n_leaf_pages = 0; - while (it != index.leaves.end()) { - ulint data_size_total = 0; - for (ulong i = 0; i < n_merge; i++) { - const per_page_stats& stat = it->second; - n_leaf_pages ++; - data_size_total += stat.data_size; - it = index.leaves.find(stat.right_page_no); - if (it == index.leaves.end()) { - break; - } - } - if (index.max_data_size) { - n_pages += data_size_total / index.max_data_size; - if (data_size_total % index.max_data_size != 0) { - n_pages += 1; - } - } - } - if (!n_leaf_pages) { - n_leaf_pages=1; - } - printf("count = " ULINTPF " free = " ULINTPF "\n", - index.count, index.free_pages); - if (n_leaf_pages) { - printf("%llu\t\t%llu\t\t" - ULINTPF "\t\t%lu\t\t" ULINTPF "\t\t%.2f\t" ULINTPF "\n", - id, index.leaf_pages, n_leaf_pages, n_merge, n_pages, - 1.0 - (double)n_pages / (double)n_leaf_pages, index.max_data_size); - } -} - -void print_leaf_stats() -{ - printf("\n**************************************************\n"); - printf("index_id\t#leaf_pages\t#actual_leaf_pages\tn_merge\t" - "#leaf_after_merge\tdefrag\n"); - for (std::map::const_iterator it = index_ids.begin(); it != index_ids.end(); it++) { - const per_index_stats& index = it->second; - if (verbose) { - print_index_leaf_stats(it->first, index); - } - if (n_merge) { - defrag_analysis(it->first, index); - } - } -} - -void -print_stats() -/*========*/ -{ - unsigned long long i; - - printf("%d\tbad checksum\n", n_bad_checksum); - printf("%d\tFIL_PAGE_INDEX\n", n_fil_page_index); - printf("%d\tFIL_PAGE_UNDO_LOG\n", n_fil_page_undo_log); - printf("%d\tFIL_PAGE_INODE\n", n_fil_page_inode); - printf("%d\tFIL_PAGE_IBUF_FREE_LIST\n", n_fil_page_ibuf_free_list); - printf("%d\tFIL_PAGE_TYPE_ALLOCATED\n", n_fil_page_type_allocated); - printf("%d\tFIL_PAGE_IBUF_BITMAP\n", n_fil_page_ibuf_bitmap); - printf("%d\tFIL_PAGE_TYPE_SYS\n", n_fil_page_type_sys); - printf("%d\tFIL_PAGE_TYPE_TRX_SYS\n", n_fil_page_type_trx_sys); - printf("%d\tFIL_PAGE_TYPE_FSP_HDR\n", n_fil_page_type_fsp_hdr); - printf("%d\tFIL_PAGE_TYPE_XDES\n", n_fil_page_type_xdes); - printf("%d\tFIL_PAGE_TYPE_BLOB\n", n_fil_page_type_blob); - printf("%d\tFIL_PAGE_TYPE_ZBLOB\n", n_fil_page_type_zblob); - printf("%d\tFIL_PAGE_PAGE_COMPRESSED\n", n_fil_page_type_page_compressed); - printf("%d\tFIL_PAGE_PAGE_COMPRESSED_ENCRYPTED\n", n_fil_page_type_page_compressed_encrypted); - printf("%d\tother\n", n_fil_page_type_other); - printf("%d\tmax index_id\n", n_fil_page_max_index_id); - printf("undo type: %d insert, %d update, %d other\n", - n_undo_insert, n_undo_update, n_undo_other); - printf("undo state: %d active, %d cached, %d to_free, %d to_purge," - " %d prepared, %d other\n", n_undo_state_active, - n_undo_state_cached, n_undo_state_to_free, - n_undo_state_to_purge, n_undo_state_prepared, - n_undo_state_other); - - printf("index_id\t#pages\t\t#leaf_pages\t#recs_per_page" - "\t#bytes_per_page\n"); - for (std::map::const_iterator it = index_ids.begin(); it != index_ids.end(); it++) { - const per_index_stats& index = it->second; - ulonglong recs_per_page = index.total_n_recs; - ulonglong bytes_per_page = index.total_data_bytes; - if (index.total_n_recs && index.pages) { - recs_per_page = index.total_n_recs / index.pages; - } - if (index.total_data_bytes && index.pages) { - bytes_per_page = index.total_data_bytes / index.pages; - } - printf("%llu\t\t%llu\t\t%llu\t\t%llu\t\t%llu\n", - it->first, index.pages, index.leaf_pages, - recs_per_page, - bytes_per_page); - } - printf("\n"); - printf("index_id\tpage_data_bytes_histgram(empty,...,oversized)\n"); - for (std::map::const_iterator it = index_ids.begin(); it != index_ids.end(); it++) { - printf("%llu\t", it->first); - const per_index_stats& index = it->second; - for (i = 0; i < SIZE_RANGES_FOR_PAGE+2; i++) { - printf("\t%llu", index.pages_in_size_range[i]); - } - printf("\n"); - } - if (do_leaf) { - print_leaf_stats(); - } -} - -int main(int argc, char **argv) -{ - FILE* f; /* our input file */ - char* filename; /* our input filename. */ - unsigned char *big_buf= 0, *buf; - unsigned char *big_xdes= 0, *xdes; - ulong bytes; /* bytes read count */ - ulint ct; /* current page number (0 based) */ - time_t now; /* current time */ - time_t lastt; /* last time */ - ulint oldcsum, oldcsumfield, csum, csumfield, crc32, logseq, logseqfield; - /* ulints for checksum storage */ - unsigned long long int size; /* size of file (has to be 64 bits) */ - ulint pages; /* number of pages in file */ - long long offset= 0; - int fd; - - printf("InnoDB offline file checksum utility.\n"); - - ut_crc32_init(); - - MY_INIT(argv[0]); - - if (get_options(&argc,&argv)) - exit(1); - - if (verbose) - my_print_variables(innochecksum_options); - - /* The file name is not optional */ - filename = *argv; - if (*filename == '\0') - { - fprintf(stderr, "Error; File name missing\n"); - goto error_out; - } - -#ifdef _WIN32 - /* Switch off OS file buffering for the file. */ - - HANDLE h = CreateFile(filename, GENERIC_READ, - FILE_SHARE_READ|FILE_SHARE_WRITE, 0, - OPEN_EXISTING, FILE_FLAG_NO_BUFFERING, 0); - - if (!h) - { - fprintf(stderr, "Error; cant open file\n"); - goto error; - } - - if (!GetFileSizeEx(h, (LARGE_INTEGER *)&size)) - { - fprintf(stderr, "Error; GetFileSize() failed\n"); - goto error; - } - - fd = _open_osfhandle ((intptr_t) h, _O_RDONLY); - if (fd < 0) - { - fprintf(stderr, "Error; _open_osfhandle() failed\n"); - goto error; - } - - f = _fdopen(fd, "rb"); - if (!f) - { - fprintf(stderr, "Error; fdopen() failed\n"); - goto error; - } - - /* - Disable stdio buffering (FILE_FLAG_NO_BUFFERING requires properly IO buffers - which stdio does not guarantee. - */ - setvbuf(f, NULL, _IONBF, 0); - -#else - struct stat st; - /* stat the file to get size and page count */ - if (stat(filename, &st)) - { - fprintf(stderr, "Error; %s cannot be found\n", filename); - goto error_out; - } - size= st.st_size; - - /* Open the file for reading */ - f= fopen(filename, "rb"); -#endif - - if (f == NULL) - { - fprintf(stderr, "Error; %s cannot be opened", filename); - perror(" "); - goto error_out; - } - - big_buf = (unsigned char *)malloc(2 * UNIV_PAGE_SIZE_MAX); - if (big_buf == NULL) - { - fprintf(stderr, "Error; failed to allocate memory\n"); - perror(""); - goto error_f; - } - - /* Make sure the page is aligned */ - buf = (unsigned char*)ut_align_down(big_buf - + UNIV_PAGE_SIZE_MAX, UNIV_PAGE_SIZE_MAX); - - big_xdes = (unsigned char *)malloc(2 * UNIV_PAGE_SIZE_MAX); - if (big_xdes == NULL) - { - fprintf(stderr, "Error; failed to allocate memory\n"); - perror(""); - goto error_big_buf; - } - - /* Make sure the page is aligned */ - xdes = (unsigned char*)ut_align_down(big_xdes - + UNIV_PAGE_SIZE_MAX, UNIV_PAGE_SIZE_MAX); - - - if (!get_page_size(f, buf, &logical_page_size, &physical_page_size)) - goto error; - - if (compressed) - { - printf("Table is compressed\n"); - printf("Key block size is %lu\n", physical_page_size); - } - else - { - printf("Table is uncompressed\n"); - printf("Page size is %lu\n", physical_page_size); - } - - pages= (ulint) (size / physical_page_size); - - if (just_count) - { - if (verbose) - printf("Number of pages: "); - printf(ULINTPF "\n", pages); - goto ok; - } - else if (verbose) - { - printf("file %s = %llu bytes (" ULINTPF " pages)...\n", - filename, size, pages); - if (do_one_page) - printf("InnoChecksum; checking page %lu\n", do_page); - else - printf("InnoChecksum; checking pages in range %lu to %lu\n", start_page, use_end_page ? end_page : (pages - 1)); - } - -#ifdef UNIV_LINUX - if (posix_fadvise(fileno(f), 0, 0, POSIX_FADV_SEQUENTIAL) || - posix_fadvise(fileno(f), 0, 0, POSIX_FADV_NOREUSE)) - { - perror("posix_fadvise failed"); - } -#endif - - /* seek to the necessary position */ - if (start_page) - { - fd= fileno(f); - if (!fd) - { - perror("Error; Unable to obtain file descriptor number"); - goto error; - } - - offset= (longlong)start_page * (longlong)physical_page_size; -#ifdef _WIN32 - if (_lseeki64(fd, offset, SEEK_SET) != offset) -#else - if (lseek(fd, offset, SEEK_SET) != offset) -#endif - { - perror("Error; Unable to seek to necessary offset"); - goto error; - } - } - - /* main checksumming loop */ - ct= start_page; - lastt= 0; - while (!feof(f)) - { - int page_ok = 1; - - bytes= fread(buf, 1, physical_page_size, f); - - if (!bytes && feof(f)) - goto ok; - - if (ferror(f)) - { - fprintf(stderr, "Error reading %lu bytes", physical_page_size); - perror(" "); - goto error; - } - - ulint page_type = mach_read_from_2(buf+FIL_PAGE_TYPE); - ulint key_version = mach_read_from_4(buf + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); - - if (key_version && page_type != FIL_PAGE_PAGE_COMPRESSED) { - encrypted = true; - } else { - encrypted = false; - } - - ulint comp_method = 0; - - if (encrypted) { - comp_method = mach_read_from_2(buf+FIL_PAGE_DATA+FIL_PAGE_COMPRESSED_SIZE); - } else { - comp_method = mach_read_from_8(buf+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); - } - - ulint comp_size = mach_read_from_2(buf+FIL_PAGE_DATA); - ib_uint32_t encryption_checksum = mach_read_from_4(buf+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4); - - - if (page_type == FIL_PAGE_PAGE_COMPRESSED) { - /* Page compressed tables do not have any checksum */ - if (debug) - fprintf(stderr, "Page " ULINTPF - " page compressed with method %s real_size " ULINTPF "\n", ct, - fil_get_compression_alg_name(comp_method), comp_size); - page_ok = 1; - } else if (compressed) { - /* compressed pages */ - ulint crccsum = page_zip_calc_checksum(buf, physical_page_size, SRV_CHECKSUM_ALGORITHM_CRC32); - ulint icsum = page_zip_calc_checksum(buf, physical_page_size, SRV_CHECKSUM_ALGORITHM_INNODB); - - if (debug) { - if (key_version != 0) { - fprintf(stderr, - "Page " ULINTPF - " encrypted key_version " ULINTPF - " calculated = " ULINTPF "; crc32 = " ULINTPF - "; recorded = %u\n", - ct, key_version, icsum, crccsum, encryption_checksum); - } - } - - if (encrypted) { - if (encryption_checksum != 0 && crccsum != encryption_checksum && icsum != encryption_checksum) { - if (debug) - fprintf(stderr, "page " ULINTPF - ": compressed: calculated = " ULINTPF - "; crc32 = " ULINTPF "; recorded = %u\n", - ct, icsum, crccsum, encryption_checksum); - fprintf(stderr, "Fail; page " ULINTPF - " invalid (fails compressed page checksum).\n", ct); - } - } else { - if (!page_zip_verify_checksum(buf, physical_page_size)) { - fprintf(stderr, "Fail; page " ULINTPF - " invalid (fails compressed page checksum).\n", ct); - if (!skip_corrupt) - goto error; - page_ok = 0; - } + case SRV_CHECKSUM_ALGORITHM_STRICT_NONE: + case SRV_CHECKSUM_ALGORITHM_NONE: + srv_checksum_algorithm = + SRV_CHECKSUM_ALGORITHM_STRICT_NONE; + break; + default: + return(true); + } + break; + case 'n': + no_check = true; + break; + case 'a': + case 'S': + break; + case 'w': + do_write = true; + break; + case 'D': + page_type_dump = true; + break; + case 'l': + is_log_enabled = true; + break; + case 'I': + case '?': + usage(); + my_end(0); + exit(EXIT_SUCCESS); + break; } - } else { - if (key_version != 0) { - /* Encrypted page */ - if (debug) { - if (page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { - fprintf(stderr, - "Page " ULINTPF - " page compressed with method %s real_size " ULINTPF - " and encrypted key_version " ULINTPF " checksum %u\n", - ct, fil_get_compression_alg_name(comp_method), comp_size, key_version, encryption_checksum); - } else { - fprintf(stderr, - "Page " ULINTPF - " encrypted key_version " ULINTPF " checksum %u\n", - ct, key_version, encryption_checksum); - } - } - } - /* Page compressed tables do not contain FIL tailer */ - if (page_type != FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED && page_type != FIL_PAGE_PAGE_COMPRESSED) { - /* check the "stored log sequence numbers" */ - logseq= mach_read_from_4(buf + FIL_PAGE_LSN + 4); - logseqfield= mach_read_from_4(buf + logical_page_size - FIL_PAGE_END_LSN_OLD_CHKSUM + 4); - if (debug) - printf("page " ULINTPF - ": log sequence number: first = " ULINTPF - "; second = " ULINTPF "\n", - ct, logseq, logseqfield); - if (logseq != logseqfield) - { - fprintf(stderr, "Fail; page " ULINTPF - " invalid (fails log sequence number check)\n", ct); - if (!skip_corrupt) - goto error; - page_ok = 0; - } - - /* check old method of checksumming */ - oldcsum= buf_calc_page_old_checksum(buf); - oldcsumfield= mach_read_from_4(buf + logical_page_size - FIL_PAGE_END_LSN_OLD_CHKSUM); - if (debug) - printf("page " ULINTPF - ": old style: calculated = " ULINTPF - "; recorded = " ULINTPF "\n", - ct, oldcsum, oldcsumfield); - if (oldcsumfield != mach_read_from_4(buf + FIL_PAGE_LSN) && oldcsumfield != oldcsum) - { - fprintf(stderr, "Fail; page " ULINTPF - " invalid (fails old style checksum)\n", ct); - if (!skip_corrupt) - goto error; - page_ok = 0; - } - } - - /* now check the new method */ - csum= buf_calc_page_new_checksum(buf); - crc32= buf_calc_page_crc32(buf); - csumfield= mach_read_from_4(buf + FIL_PAGE_SPACE_OR_CHKSUM); - - if (key_version) - csumfield = encryption_checksum; - - if (debug) - printf("page " ULINTPF - ": new style: calculated = " ULINTPF - "; crc32 = " ULINTPF "; recorded = " ULINTPF "\n", - ct, csum, crc32, csumfield); - if (csumfield != 0 && crc32 != csumfield && csum != csumfield) - { - fprintf(stderr, "Fail; page " ULINTPF - " invalid (fails innodb and crc32 checksum)\n", ct); - if (!skip_corrupt) - goto error; - page_ok = 0; - } - } - /* end if this was the last page we were supposed to check */ - if (use_end_page && (ct >= end_page)) - goto ok; - - if (per_page_details) - { - printf("page " ULINTPF " ", ct); - } - - /* do counter increase and progress printing */ - ct++; - - if (!page_ok) - { - if (per_page_details) - { - printf("BAD_CHECKSUM\n"); - } - n_bad_checksum++; - continue; - } - - /* Can't parse compressed or/and encrypted pages */ - if (page_type != FIL_PAGE_PAGE_COMPRESSED && !encrypted) { - parse_page(buf, xdes); - } - - if (verbose) - { - if (ct % 64 == 0) - { - now= time(0); - if (!lastt) lastt= now; - if (now - lastt >= 1) - { - printf("page " ULINTPF " okay: %.3f%% done\n", - (ct - 1), (float) ct / pages * 100); - lastt= now; - } - } - } - } - -ok: - if (!just_count) - print_stats(); - free(big_xdes); - free(big_buf); - fclose(f); - my_end(0); - exit(0); - -error: - free(big_xdes); -error_big_buf: - free(big_buf); -error_f: - fclose(f); -error_out: - my_end(0); - exit(1); + return(false); +} + +static +bool +get_options( + int *argc, + char ***argv) +{ + if (handle_options(argc, argv, innochecksum_options, + innochecksum_get_one_option)) + exit(true); + + /* The next arg must be the filename */ + if (!*argc) { + usage(); + return (true); + } + + return (false); +} + +/** Check from page 0 if table is encrypted. */ +static +bool check_encryption( + const char* filename, + ulint zip_size, + byte * page) +{ + ulint offset = (FSP_HEADER_OFFSET + (XDES_ARR_OFFSET + XDES_SIZE * + (zip_size ? zip_size : UNIV_PAGE_SIZE) / FSP_EXTENT_SIZE)); + + if (memcmp(page + offset, CRYPT_MAGIC, MAGIC_SZ) != 0) { + return false; + } + + ulint type = mach_read_from_1(page + offset + MAGIC_SZ + 0); + + if (! (type == CRYPT_SCHEME_UNENCRYPTED || + type == CRYPT_SCHEME_1)) { + return false; + } + + ulint iv_length = mach_read_from_1(page + offset + MAGIC_SZ + 1); + + if (iv_length != CRYPT_SCHEME_1_IV_LEN) { + return false; + } + + uint min_key_version = mach_read_from_4 + (page + offset + MAGIC_SZ + 2 + iv_length); + + uint key_id = mach_read_from_4 + (page + offset + MAGIC_SZ + 2 + iv_length + 4); + + if (type == CRYPT_SCHEME_1) { + if (is_log_enabled) { + fprintf(log_file,"Tablespace %s encrypted key_version %u key_id %u\n", + filename, min_key_version, key_id); + } + } + + return (type == CRYPT_SCHEME_1); +} + +int main( + int argc, + char **argv) +{ + /* our input file. */ + FILE* fil_in = NULL; + /* our input filename. */ + char* filename; + /* Buffer to store pages read. */ + byte* buf = NULL; + byte* xdes = NULL; + /* bytes read count */ + ulong bytes; + /* current time */ + time_t now; + /* last time */ + time_t lastt; + /* stat, to get file size. */ +#ifdef _WIN32 + struct _stat64 st; +#else + struct stat st; +#endif /* _WIN32 */ + + int exit_status = 0; + + /* size of file (has to be 64 bits) */ + unsigned long long int size = 0; + /* number of pages in file */ + ulint pages; + + off_t offset = 0; + /* count the no. of page corrupted. */ + ulint mismatch_count = 0; + /* Variable to ack the page is corrupted or not. */ + bool is_corrupted = false; + + bool partial_page_read = false; + /* Enabled when read from stdin is done. */ + bool read_from_stdin = false; + FILE* fil_page_type = NULL; + fpos_t pos; + + /* Use to check the space id of given file. If space_id is zero, + then check whether page is doublewrite buffer.*/ + ulint space_id = 0UL; + /* enable when space_id of given file is zero. */ + bool is_system_tablespace = false; + + ut_crc32_init(); + MY_INIT(argv[0]); + DBUG_ENTER("main"); + DBUG_PROCESS(argv[0]); + + if (get_options(&argc,&argv)) { + exit_status = 1; + goto my_exit; + } + + if (strict_verify && no_check) { + fprintf(stderr, "Error: --strict-check option cannot be used " + "together with --no-check option.\n"); + exit_status = 1; + goto my_exit; + } + + if (no_check && !do_write) { + fprintf(stderr, "Error: --no-check must be associated with " + "--write option.\n"); + exit_status = 1; + goto my_exit; + } + + if (page_type_dump) { + fil_page_type = create_file(page_dump_filename); + if (!fil_page_type) { + exit_status = 1; + goto my_exit; + } + } + + if (is_log_enabled) { + log_file = create_file(log_filename); + if (!log_file) { + exit_status = 1; + goto my_exit; + } + fprintf(log_file, "InnoDB File Checksum Utility.\n"); + } + + if (verbose) { + my_print_variables(innochecksum_options); + } + + + buf = (byte*) malloc(UNIV_PAGE_SIZE_MAX * 2); + xdes = (byte*)malloc(UNIV_PAGE_SIZE_MAX * 2); + + /* The file name is not optional. */ + for (int i = 0; i < argc; ++i) { + /* Reset parameters for each file. */ + filename = argv[i]; + memset(&page_type, 0, sizeof(innodb_page_type)); + is_corrupted = false; + partial_page_read = false; + skip_page = false; + + if (is_log_enabled) { + fprintf(log_file, "Filename = %s\n", filename); + } + + if (*filename == '-') { + /* read from stdin. */ + fil_in = stdin; + read_from_stdin = true; + + } + + /* stat the file to get size and page count. */ + if (!read_from_stdin && +#ifdef _WIN32 + _stat64(filename, &st)) { +#else + stat(filename, &st)) { +#endif /* _WIN32 */ + fprintf(stderr, "Error: %s cannot be found\n", + filename); + + exit_status = 1; + goto my_exit; + } + + if (!read_from_stdin) { + size = st.st_size; + fil_in = open_file(filename); + /*If fil_in is NULL, terminate as some error encountered */ + if(fil_in == NULL) { + exit_status = 1; + goto my_exit; + } + /* Save the current file pointer in pos variable.*/ + if (0 != fgetpos(fil_in, &pos)) { + perror("fgetpos"); + exit_status = 1; + goto my_exit; + } + } + + /* Read the minimum page size. */ + bytes = ulong(fread(buf, 1, UNIV_ZIP_SIZE_MIN, fil_in)); + partial_page_read = true; + + if (bytes != UNIV_ZIP_SIZE_MIN) { + fprintf(stderr, "Error: Was not able to read the " + "minimum page size "); + fprintf(stderr, "of %d bytes. Bytes read was %lu\n", + UNIV_ZIP_SIZE_MIN, bytes); + + exit_status = 1; + goto my_exit; + } + + /* enable variable is_system_tablespace when space_id of given + file is zero. Use to skip the checksum verification and rewrite + for doublewrite pages. */ + is_system_tablespace = (!memcmp(&space_id, buf + + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID, 4)) + ? true : false; + + /* Determine page size, zip_size and page compression + from fsp_flags and encryption metadata from page 0 */ + ulint flags = mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + buf); + ulint page_size = fsp_flags_get_page_size(flags); + ulint zip_size = fsp_flags_get_zip_size(flags); + logical_page_size = zip_size; + physical_page_size = page_size; + srv_page_size = page_size; + bool is_compressed = FSP_FLAGS_HAS_PAGE_COMPRESSION(flags); + + if (page_size > UNIV_ZIP_SIZE_MIN) { + /* Read rest of the page 0 to determine crypt_data */ + bytes = ulong(read_file(buf, partial_page_read, page_size, fil_in)); + + if (bytes != page_size) { + fprintf(stderr, "Error: Was not able to read the " + "rest of the page "); + fprintf(stderr, "of %lu bytes. Bytes read was %lu\n", + page_size - UNIV_ZIP_SIZE_MIN, bytes); + + exit_status = 1; + goto my_exit; + } + partial_page_read = false; + } + + /* Now that we have full page 0 in buffer, check encryption */ + bool is_encrypted = check_encryption(filename, zip_size, buf); + + pages = (ulint) (size / page_size); + + if (just_count) { + if (read_from_stdin) { + fprintf(stderr, "Number of pages:" ULINTPF "\n", pages); + } else { + printf("Number of pages:" ULINTPF "\n", pages); + } + continue; + } else if (verbose && !read_from_stdin) { + if (is_log_enabled) { + fprintf(log_file, "file %s = %llu bytes " + "(" ULINTPF " pages)\n", filename, size, pages); + if (do_one_page) { + fprintf(log_file, "Innochecksum: " + "checking page %" PRIuMAX "\n", + do_page); + } + } + } else { + if (is_log_enabled) { + fprintf(log_file, "Innochecksum: checking " + "pages in range %" PRIuMAX " to %" PRIuMAX "\n", + start_page, use_end_page ? + end_page : (pages - 1)); + } + } + + /* seek to the necessary position */ + if (start_page) { + if (!read_from_stdin) { + /* If read is not from stdin, we can use + fseeko() to position the file pointer to + the desired page. */ + partial_page_read = false; + + offset = (off_t) start_page + * (off_t) page_size; +#ifdef _WIN32 + if (_fseeki64(fil_in, offset, SEEK_SET)) { +#else + if (fseeko(fil_in, offset, SEEK_SET)) { +#endif /* _WIN32 */ + perror("Error: Unable to seek to " + "necessary offset"); + + exit_status = 1; + goto my_exit; + } + /* Save the current file pointer in + pos variable. */ + if (0 != fgetpos(fil_in, &pos)) { + perror("fgetpos"); + + exit_status = 1; + goto my_exit; + } + } else { + + ulong count = 0; + + while (!feof(fil_in)) { + if (start_page == count) { + break; + } + /* We read a part of page to find the + minimum page size. We cannot reset + the file pointer to the beginning of + the page if we are reading from stdin + (fseeko() on stdin doesn't work). So + read only the remaining part of page, + if partial_page_read is enable. */ + bytes = read_file(buf, + partial_page_read, + static_cast( + page_size), + fil_in); + + partial_page_read = false; + count++; + + if (!bytes || feof(fil_in)) { + fprintf(stderr, "Error: Unable " + "to seek to necessary " + "offset"); + + exit_status = 1; + goto my_exit; + } + } + } + } + + if (page_type_dump) { + fprintf(fil_page_type, + "\n\nFilename::%s\n", filename); + fprintf(fil_page_type, + "========================================" + "======================================\n"); + fprintf(fil_page_type, + "\tPAGE_NO\t\t|\t\tPAGE_TYPE\t\t" + "\t|\tEXTRA INFO\n"); + fprintf(fil_page_type, + "========================================" + "======================================\n"); + } + + /* main checksumming loop */ + cur_page_num = start_page; + lastt = 0; + while (!feof(fil_in)) { + + bytes = read_file(buf, partial_page_read, + static_cast( + page_size), fil_in); + partial_page_read = false; + + if (!bytes && feof(fil_in)) { + break; + } + + if (ferror(fil_in)) { + fprintf(stderr, "Error reading " ULINTPF " bytes", + page_size); + perror(" "); + + exit_status = 1; + goto my_exit; + } + + if (bytes != page_size) { + fprintf(stderr, "Error: bytes read (%lu) " + "doesn't match page size (" ULINTPF ")\n", + bytes, page_size); + exit_status = 1; + goto my_exit; + } + + if (is_system_tablespace) { + /* enable when page is double write buffer.*/ + skip_page = is_page_doublewritebuffer(buf); + } else { + skip_page = false; + } + + /* If no-check is enabled, skip the + checksum verification.*/ + if (!no_check) { + /* Checksum verification */ + if (!skip_page) { + is_corrupted = is_page_corrupted( + buf, page_size, zip_size, is_encrypted, is_compressed); + + if (is_corrupted) { + fprintf(stderr, "Fail: page " + "%" PRIuMAX " invalid\n", + cur_page_num); + + mismatch_count++; + + if(mismatch_count > allow_mismatches) { + fprintf(stderr, + "Exceeded the " + "maximum allowed " + "checksum mismatch " + "count::%" PRIuMAX "\n", + allow_mismatches); + + exit_status = 1; + goto my_exit; + } + } + } + } + + /* Rewrite checksum. Note that for encrypted and + page compressed tables this is not currently supported. */ + if (do_write && + !is_encrypted && + !is_compressed + && !write_file(filename, fil_in, buf, + zip_size != 0, &pos, + static_cast(page_size))) { + + exit_status = 1; + goto my_exit; + } + + /* end if this was the last page we were supposed to check */ + if (use_end_page && (cur_page_num >= end_page)) { + break; + } + + if (per_page_details) { + printf("page %ld ", cur_page_num); + } + + if (page_type_summary || page_type_dump) { + parse_page(buf, xdes, fil_page_type); + } + + /* do counter increase and progress printing */ + cur_page_num++; + if (verbose && !read_from_stdin) { + if ((cur_page_num % 64) == 0) { + now = time(0); + if (!lastt) { + lastt= now; + } + if (now - lastt >= 1 + && is_log_enabled) { + fprintf(log_file, "page %" PRIuMAX " " + "okay: %.3f%% done\n", + (cur_page_num - 1), + (float) cur_page_num / pages * 100); + lastt = now; + } + } + } + } + + if (!read_from_stdin) { + /* flcose() will flush the data and release the lock if + any acquired. */ + fclose(fil_in); + } + + /* Enabled for page type summary. */ + if (page_type_summary) { + if (!read_from_stdin) { + fprintf(stdout, "\nFile::%s",filename); + print_summary(stdout); + } else { + print_summary(stderr); + } + } + } + + if (is_log_enabled) { + fclose(log_file); + } + +my_exit: + if (buf) { + free(buf); + } + my_end(exit_status); + exit(exit_status); } diff --git a/mysql-test/suite/encryption/r/innochecksum.result b/mysql-test/suite/encryption/r/innochecksum.result index 955d150eb04..6ea54f3d053 100644 --- a/mysql-test/suite/encryption/r/innochecksum.result +++ b/mysql-test/suite/encryption/r/innochecksum.result @@ -7,6 +7,7 @@ CREATE TABLE t2 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB ROW_FOR CREATE TABLE t3 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB ROW_FORMAT=COMPRESSED ENCRYPTED=NO; CREATE TABLE t4 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB PAGE_COMPRESSED=1; CREATE TABLE t5 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB PAGE_COMPRESSED=1 ENCRYPTED=YES ENCRYPTION_KEY_ID=4; +CREATE TABLE t6 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB; # Write file to make mysql-test-run.pl expect the "crash", but don't # start it until it's told to # We give 30 seconds to do a clean shutdown because we do not want @@ -18,6 +19,27 @@ CREATE TABLE t5 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB PAGE_CO # Run innochecksum on t3 # Run innochecksum on t4 # Run innochecksum on t4 +# Run innochecksum on t5 +# Run innochecksum on t6 +# Backup tables before corrupting +# Corrupt FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION +# Run innochecksum on t2 +# Run innochecksum on t3 +# no encryption corrupting the field should not have effect +# Run innochecksum on t6 +# no encryption corrupting the field should not have effect +# Restore the original tables +# Corrupt FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION+4 (post encryption checksum) +# Run innochecksum on t2 +# Run innochecksum on t3 +# Run innochecksum on t6 +# no encryption corrupting the field should not have effect +# Restore the original tables +# Corrupt FIL_DATA+10 (data) +# Run innochecksum on t2 +# Run innochecksum on t3 +# Run innochecksum on t6 +# Restore the original tables # Write file to make mysql-test-run.pl start up the server again # Cleanup -DROP TABLE t1, t2, t3, t4, t5; +DROP TABLE t1, t2, t3, t4, t5, t6; diff --git a/mysql-test/suite/encryption/t/innochecksum.test b/mysql-test/suite/encryption/t/innochecksum.test index a176cad6c0e..74fe7bcf7a0 100644 --- a/mysql-test/suite/encryption/t/innochecksum.test +++ b/mysql-test/suite/encryption/t/innochecksum.test @@ -2,7 +2,7 @@ # MDEV-8773: InnoDB innochecksum does not work with encrypted or page compressed tables # -# Don't test under embedded +# Don't test under embedded as we restart server -- source include/not_embedded.inc # Require InnoDB -- source include/have_innodb.inc @@ -13,16 +13,12 @@ if (!$INNOCHECKSUM) { --die Need innochecksum binary } ---disable_query_log -let $innodb_compression_algorithm_orig=`SELECT @@innodb_compression_algorithm`; -let $innodb_file_format_orig = `SELECT @@innodb_file_format`; -let $innodb_file_per_table_orig = `SELECT @@innodb_file_per_table`; ---enable_query_log - +--disable_warnings SET GLOBAL innodb_file_format = `Barracuda`; SET GLOBAL innodb_file_per_table = ON; # zlib set global innodb_compression_algorithm = 1; +--enable_warnings --echo # Create and populate a tables CREATE TABLE t1 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=4; @@ -30,9 +26,11 @@ CREATE TABLE t2 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB ROW_FOR CREATE TABLE t3 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB ROW_FORMAT=COMPRESSED ENCRYPTED=NO; CREATE TABLE t4 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB PAGE_COMPRESSED=1; CREATE TABLE t5 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB PAGE_COMPRESSED=1 ENCRYPTED=YES ENCRYPTION_KEY_ID=4; +CREATE TABLE t6 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB; --disable_query_log --let $i = 1000 +begin; while ($i) { INSERT INTO t1 (b) VALUES (REPEAT('abcdefghijklmnopqrstuvwxyz', 100)); @@ -42,6 +40,8 @@ INSERT INTO t2 SELECT * FROM t1; INSERT INTO t3 SELECT * FROM t1; INSERT INTO t4 SELECT * FROM t1; INSERT INTO t5 SELECT * FROM t1; +INSERT INTO t6 SELECT * FROM t1; +commit; --enable_query_log let $MYSQLD_DATADIR=`select @@datadir`; @@ -50,6 +50,10 @@ let t2_IBD = $MYSQLD_DATADIR/test/t2.ibd; let t3_IBD = $MYSQLD_DATADIR/test/t3.ibd; let t4_IBD = $MYSQLD_DATADIR/test/t4.ibd; let t5_IBD = $MYSQLD_DATADIR/test/t5.ibd; +let t6_IBD = $MYSQLD_DATADIR/test/t6.ibd; + +let INNODB_PAGE_SIZE=`select @@innodb_page_size`; +let MYSQLD_DATADIR=`select @@datadir`; --echo # Write file to make mysql-test-run.pl expect the "crash", but don't --echo # start it until it's told to @@ -83,19 +87,198 @@ shutdown_server 30; --exec $INNOCHECKSUM $t4_IBD +--echo # Run innochecksum on t5 + +--exec $INNOCHECKSUM $t5_IBD + +--echo # Run innochecksum on t6 + +--exec $INNOCHECKSUM $t6_IBD + --enable_result_log +--echo # Backup tables before corrupting +--copy_file $MYSQLD_DATADIR/test/t1.ibd $MYSQLD_DATADIR/test/t1.ibd.backup +--copy_file $MYSQLD_DATADIR/test/t2.ibd $MYSQLD_DATADIR/test/t2.ibd.backup +--copy_file $MYSQLD_DATADIR/test/t3.ibd $MYSQLD_DATADIR/test/t3.ibd.backup +--copy_file $MYSQLD_DATADIR/test/t4.ibd $MYSQLD_DATADIR/test/t4.ibd.backup +--copy_file $MYSQLD_DATADIR/test/t5.ibd $MYSQLD_DATADIR/test/t5.ibd.backup +--copy_file $MYSQLD_DATADIR/test/t6.ibd $MYSQLD_DATADIR/test/t6.ibd.backup + +# +# MDEV-11939: innochecksum mistakes a file for an encrypted one +# + +--echo # Corrupt FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + +perl; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t1.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 26, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t2.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 26, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t3.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 26, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t6.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 26, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +EOF + +-- disable_result_log +--error 1 +--exec $INNOCHECKSUM $t1_IBD + +--echo # Run innochecksum on t2 + +--error 1 +--exec $INNOCHECKSUM $t2_IBD + +--echo # Run innochecksum on t3 +--echo # no encryption corrupting the field should not have effect +--exec $INNOCHECKSUM $t3_IBD + +--echo # Run innochecksum on t6 +--echo # no encryption corrupting the field should not have effect +--exec $INNOCHECKSUM $t6_IBD + +--enable_result_log + +--echo # Restore the original tables +--remove_file $MYSQLD_DATADIR/test/t1.ibd +--remove_file $MYSQLD_DATADIR/test/t2.ibd +--remove_file $MYSQLD_DATADIR/test/t3.ibd +--remove_file $MYSQLD_DATADIR/test/t4.ibd +--remove_file $MYSQLD_DATADIR/test/t5.ibd +--remove_file $MYSQLD_DATADIR/test/t6.ibd +--copy_file $MYSQLD_DATADIR/test/t1.ibd.backup $MYSQLD_DATADIR/test/t1.ibd +--copy_file $MYSQLD_DATADIR/test/t2.ibd.backup $MYSQLD_DATADIR/test/t2.ibd +--copy_file $MYSQLD_DATADIR/test/t3.ibd.backup $MYSQLD_DATADIR/test/t3.ibd +--copy_file $MYSQLD_DATADIR/test/t4.ibd.backup $MYSQLD_DATADIR/test/t4.ibd +--copy_file $MYSQLD_DATADIR/test/t5.ibd.backup $MYSQLD_DATADIR/test/t5.ibd +--copy_file $MYSQLD_DATADIR/test/t6.ibd.backup $MYSQLD_DATADIR/test/t6.ibd + +--echo # Corrupt FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION+4 (post encryption checksum) + +perl; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t1.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 30, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t2.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 30, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t3.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 30, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t6.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 30, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +EOF + +-- disable_result_log +--error 1 +--exec $INNOCHECKSUM $t1_IBD + +--echo # Run innochecksum on t2 +--error 1 +--exec $INNOCHECKSUM $t2_IBD + +--echo # Run innochecksum on t3 +--error 1 +--exec $INNOCHECKSUM $t3_IBD + +--echo # Run innochecksum on t6 +--echo # no encryption corrupting the field should not have effect +--exec $INNOCHECKSUM $t6_IBD + +--enable_result_log + +--echo # Restore the original tables +--remove_file $MYSQLD_DATADIR/test/t1.ibd +--remove_file $MYSQLD_DATADIR/test/t2.ibd +--remove_file $MYSQLD_DATADIR/test/t3.ibd +--remove_file $MYSQLD_DATADIR/test/t4.ibd +--remove_file $MYSQLD_DATADIR/test/t5.ibd +--remove_file $MYSQLD_DATADIR/test/t6.ibd +--copy_file $MYSQLD_DATADIR/test/t1.ibd.backup $MYSQLD_DATADIR/test/t1.ibd +--copy_file $MYSQLD_DATADIR/test/t2.ibd.backup $MYSQLD_DATADIR/test/t2.ibd +--copy_file $MYSQLD_DATADIR/test/t3.ibd.backup $MYSQLD_DATADIR/test/t3.ibd +--copy_file $MYSQLD_DATADIR/test/t4.ibd.backup $MYSQLD_DATADIR/test/t4.ibd +--copy_file $MYSQLD_DATADIR/test/t5.ibd.backup $MYSQLD_DATADIR/test/t5.ibd +--copy_file $MYSQLD_DATADIR/test/t6.ibd.backup $MYSQLD_DATADIR/test/t6.ibd + +--echo # Corrupt FIL_DATA+10 (data) + +perl; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t1.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 48, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t2.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 48, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t3.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 48, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t6.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 48, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +EOF + +-- disable_result_log +--error 1 +--exec $INNOCHECKSUM $t1_IBD + +--echo # Run innochecksum on t2 +--error 1 +--exec $INNOCHECKSUM $t2_IBD + +--echo # Run innochecksum on t3 +--error 1 +--exec $INNOCHECKSUM $t3_IBD + +--echo # Run innochecksum on t6 +--error 1 +--exec $INNOCHECKSUM $t6_IBD + +--enable_result_log + +--echo # Restore the original tables +--move_file $MYSQLD_DATADIR/test/t1.ibd.backup $MYSQLD_DATADIR/test/t1.ibd +--move_file $MYSQLD_DATADIR/test/t2.ibd.backup $MYSQLD_DATADIR/test/t2.ibd +--move_file $MYSQLD_DATADIR/test/t3.ibd.backup $MYSQLD_DATADIR/test/t3.ibd +--move_file $MYSQLD_DATADIR/test/t4.ibd.backup $MYSQLD_DATADIR/test/t4.ibd +--move_file $MYSQLD_DATADIR/test/t5.ibd.backup $MYSQLD_DATADIR/test/t5.ibd +--move_file $MYSQLD_DATADIR/test/t6.ibd.backup $MYSQLD_DATADIR/test/t6.ibd + --echo # Write file to make mysql-test-run.pl start up the server again --exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect --enable_reconnect --source include/wait_until_connected_again.inc --echo # Cleanup -DROP TABLE t1, t2, t3, t4, t5; - -# reset system ---disable_query_log -EVAL SET GLOBAL innodb_compression_algorithm = $innodb_compression_algorithm_orig; -EVAL SET GLOBAL innodb_file_per_table = $innodb_file_per_table_orig; -EVAL SET GLOBAL innodb_file_format = $innodb_file_format_orig; ---enable_query_log +DROP TABLE t1, t2, t3, t4, t5, t6; diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 334b2fbddb5..50d067b3f85 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -31,12 +31,16 @@ The database buffer buf_pool Created 11/5/1995 Heikki Tuuri *******************************************************/ +#include "univ.i" +#include "mach0data.h" #include "buf0buf.h" +#include #ifdef UNIV_NONINL #include "buf0buf.ic" #endif +#ifndef UNIV_INNOCHECKSUM #include "mem0mem.h" #include "btr0btr.h" #include "fil0fil.h" @@ -52,13 +56,15 @@ Created 11/5/1995 Heikki Tuuri #include "srv0srv.h" #include "dict0dict.h" #include "log0recv.h" -#include "page0zip.h" #include "srv0mon.h" -#include "buf0checksum.h" #ifdef HAVE_LIBNUMA #include #include #endif // HAVE_LIBNUMA +#endif /* !UNIV_INNOCHECKSUM */ +#include "page0zip.h" +#include "buf0checksum.h" +#ifndef UNIV_INNOCHECKSUM #include "fil0pagecompress.h" #include "ha_prototypes.h" #include "ut0byte.h" @@ -528,6 +534,7 @@ buf_block_alloc( return(block); } #endif /* !UNIV_HOTBACKUP */ +#endif /* !UNIV_INNOCHECKSUM */ /** Check if a page is all zeroes. @param[in] read_buf database page @@ -561,6 +568,17 @@ buf_page_is_checksum_valid_crc32( { ib_uint32_t crc32 = buf_calc_page_crc32(read_buf); +#ifdef UNIV_INNOCHECKSUM + if (log_file + && srv_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_STRICT_CRC32) { + fprintf(log_file, "page::%lu;" + " crc32 calculated = %u;" + " recorded checksum field1 = %lu recorded" + " checksum field2 =%lu\n", cur_page_num, + crc32, checksum_field1, checksum_field2); + } +#endif /* UNIV_INNOCHECKSUM */ + if (!(checksum_field1 == crc32 && checksum_field2 == crc32)) { DBUG_PRINT("buf_checksum", ("Page checksum crc32 not valid field1 " ULINTPF @@ -595,12 +613,45 @@ buf_page_is_checksum_valid_innodb( 2. Newer InnoDB versions store the old formula checksum (buf_calc_page_old_checksum()). */ + ulint old_checksum = buf_calc_page_old_checksum(read_buf); + ulint new_checksum = buf_calc_page_new_checksum(read_buf); + +#ifdef UNIV_INNOCHECKSUM + if (log_file + && srv_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_INNODB) { + fprintf(log_file, "page::%lu;" + " old style: calculated =" + " %lu; recorded = %lu\n", + cur_page_num, old_checksum, + checksum_field2); + fprintf(log_file, "page::%lu;" + " new style: calculated =" + " %lu; crc32 = %u; recorded = %lu\n", + cur_page_num, new_checksum, + buf_calc_page_crc32(read_buf), checksum_field1); + } + + if (log_file + && srv_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_STRICT_INNODB) { + fprintf(log_file, "page::%lu;" + " old style: calculated =" + " %lu; recorded checksum = %lu\n", + cur_page_num, old_checksum, + checksum_field2); + fprintf(log_file, "page::%lu;" + " new style: calculated =" + " %lu; recorded checksum = %lu\n", + cur_page_num, new_checksum, + checksum_field1); + } +#endif /* UNIV_INNOCHECKSUM */ + if (checksum_field2 != mach_read_from_4(read_buf + FIL_PAGE_LSN) - && checksum_field2 != buf_calc_page_old_checksum(read_buf)) { + && checksum_field2 != old_checksum) { DBUG_PRINT("buf_checksum", ("Page checksum innodb not valid field1 " ULINTPF " field2 " ULINTPF "crc32 " ULINTPF " lsn " ULINTPF ".", - checksum_field1, checksum_field2, buf_calc_page_old_checksum(read_buf), + checksum_field1, checksum_field2, old_checksum, mach_read_from_4(read_buf + FIL_PAGE_LSN))); return(false); @@ -612,11 +663,11 @@ buf_page_is_checksum_valid_innodb( (always equal to 0), to FIL_PAGE_SPACE_OR_CHKSUM */ if (checksum_field1 != 0 - && checksum_field1 != buf_calc_page_new_checksum(read_buf)) { + && checksum_field1 != new_checksum) { DBUG_PRINT("buf_checksum", ("Page checksum innodb not valid field1 " ULINTPF " field2 " ULINTPF "crc32 " ULINTPF " lsn " ULINTPF ".", - checksum_field1, checksum_field2, buf_calc_page_new_checksum(read_buf), + checksum_field1, checksum_field2, new_checksum, mach_read_from_4(read_buf + FIL_PAGE_LSN))); return(false); @@ -646,6 +697,18 @@ buf_page_is_checksum_valid_none( mach_read_from_4(read_buf + FIL_PAGE_LSN))); } +#ifdef UNIV_INNOCHECKSUM + if (log_file + && srv_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_STRICT_NONE) { + fprintf(log_file, + "page::%lu; none checksum: calculated" + " = %lu; recorded checksum_field1 = %lu" + " recorded checksum_field2 = %lu\n", + cur_page_num, BUF_NO_CHECKSUM_MAGIC, + checksum_field1, checksum_field2); + } +#endif /* UNIV_INNOCHECKSUM */ + return(checksum_field1 == checksum_field2 && checksum_field1 == BUF_NO_CHECKSUM_MAGIC); } @@ -662,14 +725,18 @@ buf_page_is_corrupted( bool check_lsn, const byte* read_buf, ulint zip_size, +#ifndef UNIV_INNOCHECKSUM const fil_space_t* space) +#else + const void* space) +#endif { - ulint checksum_field1; - ulint checksum_field2; - ulint space_id = mach_read_from_4( - read_buf + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); - ulint page_type = mach_read_from_2( - read_buf + FIL_PAGE_TYPE); + ulint checksum_field1 = 0; + ulint checksum_field2 = 0; +#ifndef UNIV_INNOCHECKSUM + ulint space_id = mach_read_from_4(read_buf + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); +#endif + ulint page_type = mach_read_from_2(read_buf + FIL_PAGE_TYPE); /* We can trust page type if page compression is set on tablespace flags because page compression flag means file must have been @@ -682,7 +749,10 @@ buf_page_is_corrupted( decompressed at this stage). */ if ((page_type == FIL_PAGE_PAGE_COMPRESSED || page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) - && space && FSP_FLAGS_HAS_PAGE_COMPRESSION(space->flags)) { +#ifndef UNIV_INNOCHECKSUM + && space && FSP_FLAGS_HAS_PAGE_COMPRESSION(space->flags) +#endif + ) { return (false); } @@ -693,16 +763,17 @@ buf_page_is_corrupted( /* Stored log sequence numbers at the start and the end of page do not match */ - +#ifndef UNIV_INNOCHECKSUM ib_logf(IB_LOG_LEVEL_INFO, "Log sequence number at the start %lu and the end %lu do not match.", mach_read_from_4(read_buf + FIL_PAGE_LSN + 4), mach_read_from_4(read_buf + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM + 4)); +#endif /* UNIV_INNOCHECKSUM */ return(true); } -#ifndef UNIV_HOTBACKUP +#if !defined(UNIV_HOTBACKUP) && !defined(UNIV_INNOCHECKSUM) if (check_lsn && recv_lsn_checks_on) { lsn_t current_lsn; @@ -742,7 +813,7 @@ buf_page_is_corrupted( } if (zip_size) { - return(!page_zip_verify_checksum(read_buf, zip_size)); + return(!page_zip_verify_checksum((const void *)read_buf, zip_size)); } checksum_field1 = mach_read_from_4( @@ -762,9 +833,10 @@ buf_page_is_corrupted( /* make sure that the page is really empty */ for (ulint i = 0; i < UNIV_PAGE_SIZE; i++) { if (read_buf[i] != 0) { +#ifndef UNIV_INNOCHECKSUM ib_logf(IB_LOG_LEVEL_INFO, "Checksum fields zero but page is not empty."); - +#endif return(true); } } @@ -774,7 +846,9 @@ buf_page_is_corrupted( DBUG_EXECUTE_IF("buf_page_is_corrupt_failure", return(true); ); +#ifndef UNIV_INNOCHECKSUM ulint page_no = mach_read_from_4(read_buf + FIL_PAGE_OFFSET); +#endif const srv_checksum_algorithm_t curr_algo = static_cast(srv_checksum_algorithm); @@ -792,12 +866,32 @@ buf_page_is_corrupted( checksum_field1, checksum_field2)) { if (curr_algo == SRV_CHECKSUM_ALGORITHM_STRICT_CRC32) { +#ifndef UNIV_INNOCHECKSUM page_warn_strict_checksum( curr_algo, SRV_CHECKSUM_ALGORITHM_NONE, space_id, page_no); +#endif /* !UNIV_INNOCHECKSUM */ } +#ifdef UNIV_INNOCHECKSUM + if (log_file) { + fprintf(log_file, "page::" ULINTPF ";" + " old style: calculated = " ULINTPF ";" + " recorded = " ULINTPF "\n", + cur_page_num, + buf_calc_page_old_checksum(read_buf), + checksum_field2); + fprintf(log_file, "page::" ULINTPF ";" + " new style: calculated = " ULINTPF ";" + " crc32 = %u; recorded = " ULINTPF "\n", + cur_page_num, + buf_calc_page_new_checksum(read_buf), + buf_calc_page_crc32(read_buf), + checksum_field1); + } +#endif /* UNIV_INNOCHECKSUM */ + return(false); } @@ -805,15 +899,24 @@ buf_page_is_corrupted( checksum_field1, checksum_field2)) { if (curr_algo == SRV_CHECKSUM_ALGORITHM_STRICT_CRC32) { +#ifndef UNIV_INNOCHECKSUM page_warn_strict_checksum( curr_algo, SRV_CHECKSUM_ALGORITHM_INNODB, space_id, page_no); +#endif } return(false); } +#ifdef UNIV_INNOCHECKSUM + if (log_file) { + fprintf(log_file, "Fail; page " ULINTPF + " invalid (fails crc32 checksum)\n", + cur_page_num); + } +#endif /* UNIV_INNOCHECKSUM */ return(true); case SRV_CHECKSUM_ALGORITHM_INNODB: @@ -828,11 +931,29 @@ buf_page_is_corrupted( checksum_field1, checksum_field2)) { if (curr_algo == SRV_CHECKSUM_ALGORITHM_STRICT_INNODB) { +#ifndef UNIV_INNOCHECKSUM page_warn_strict_checksum( curr_algo, SRV_CHECKSUM_ALGORITHM_NONE, space_id, page_no); +#endif } +#ifdef UNIV_INNOCHECKSUM + if (log_file) { + fprintf(log_file, "page::" ULINTPF ";" + " old style: calculated = " ULINTPF ";" + " recorded = " ULINTPF "\n", cur_page_num, + buf_calc_page_old_checksum(read_buf), + checksum_field2); + fprintf(log_file, "page::" ULINTPF ";" + " new style: calculated = " ULINTPF ";" + " crc32 = %u; recorded = " ULINTPF "\n", + cur_page_num, + buf_calc_page_new_checksum(read_buf), + buf_calc_page_crc32(read_buf), + checksum_field1); + } +#endif /* UNIV_INNOCHECKSUM */ return(false); } @@ -841,15 +962,25 @@ buf_page_is_corrupted( checksum_field1, checksum_field2)) { if (curr_algo == SRV_CHECKSUM_ALGORITHM_STRICT_INNODB) { +#ifndef UNIV_INNOCHECKSUM page_warn_strict_checksum( curr_algo, SRV_CHECKSUM_ALGORITHM_CRC32, space_id, page_no); +#endif } return(false); } +#ifdef UNIV_INNOCHECKSUM + if (log_file) { + fprintf(log_file, "Fail; page " ULINTPF + " invalid (fails innodb checksum)\n", + cur_page_num); + } +#endif /* UNIV_INNOCHECKSUM */ + return(true); case SRV_CHECKSUM_ALGORITHM_STRICT_NONE: @@ -861,22 +992,34 @@ buf_page_is_corrupted( if (buf_page_is_checksum_valid_crc32(read_buf, checksum_field1, checksum_field2)) { +#ifndef UNIV_INNOCHECKSUM page_warn_strict_checksum( curr_algo, SRV_CHECKSUM_ALGORITHM_CRC32, space_id, page_no); +#endif return(false); } if (buf_page_is_checksum_valid_innodb(read_buf, checksum_field1, checksum_field2)) { +#ifndef UNIV_INNOCHECKSUM page_warn_strict_checksum( curr_algo, SRV_CHECKSUM_ALGORITHM_INNODB, space_id, page_no); +#endif return(false); } +#ifdef UNIV_INNOCHECKSUM + if (log_file) { + fprintf(log_file, "Fail; page " ULINTPF + " invalid (fails none checksum)\n", + cur_page_num); + } +#endif /* UNIV_INNOCHECKSUM */ + return(true); case SRV_CHECKSUM_ALGORITHM_NONE: @@ -890,6 +1033,7 @@ buf_page_is_corrupted( return(false); } +#ifndef UNIV_INNOCHECKSUM /********************************************************************//** Prints a page to stderr. */ UNIV_INTERN @@ -6352,3 +6496,4 @@ buf_page_decrypt_after_read(buf_page_t* bpage, fil_space_t* space) ut_ad(space->n_pending_ios > 0); return (success); } +#endif /* !UNIV_INNOCHECKSUM */ diff --git a/storage/innobase/buf/buf0checksum.cc b/storage/innobase/buf/buf0checksum.cc index 4101d117896..9e5f1dfe475 100644 --- a/storage/innobase/buf/buf0checksum.cc +++ b/storage/innobase/buf/buf0checksum.cc @@ -128,8 +128,6 @@ buf_calc_page_old_checksum( return(checksum); } -#ifndef UNIV_INNOCHECKSUM - /********************************************************************//** Return a printable string describing the checksum algorithm. @return algorithm name */ @@ -158,4 +156,3 @@ buf_checksum_algorithm_name( return(NULL); } -#endif /* !UNIV_INNOCHECKSUM */ diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index df5c250df90..b5b762c2cd9 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -25,13 +25,18 @@ Modified Jan Lindström jan.lindstrom@mariadb.com #include "fil0fil.h" #include "fil0crypt.h" +#include "mach0data.h" +#include "page0zip.h" +#include "buf0buf.h" +#include "buf0checksum.h" + +#ifndef UNIV_INNOCHECKSUM + #include "srv0srv.h" #include "srv0start.h" -#include "mach0data.h" #include "log0recv.h" #include "mtr0mtr.h" #include "mtr0log.h" -#include "page0zip.h" #include "ut0ut.h" #include "btr0scrub.h" #include "fsp0fsp.h" @@ -107,13 +112,20 @@ UNIV_INTERN mysql_pfs_key_t fil_crypt_data_mutex_key; extern my_bool srv_background_scrub_data_uncompressed; extern my_bool srv_background_scrub_data_compressed; +/*********************************************************************** +Check if a key needs rotation given a key_state +@param[in] encrypt_mode Encryption mode +@param[in] key_version Current key version +@param[in] latest_key_version Latest key version +@param[in] rotate_key_age when to rotate +@return true if key needs rotation, false if not */ static bool fil_crypt_needs_rotation( - fil_encryption_t encrypt_mode, /*!< in: Encryption - mode */ - uint key_version, /*!< in: Key version */ - uint latest_key_version, /*!< in: Latest key version */ - uint rotate_key_age); /*!< in: When to rotate */ + fil_encryption_t encrypt_mode, + uint key_version, + uint latest_key_version, + uint rotate_key_age) + MY_ATTRIBUTE((warn_unused_result)); /********************************************************************* Init space crypt */ @@ -908,137 +920,6 @@ fil_crypt_calculate_checksum( return checksum; } -/********************************************************************* -Verify that post encryption checksum match calculated checksum. -This function should be called only if tablespace contains crypt_data -metadata (this is strong indication that tablespace is encrypted). -Function also verifies that traditional checksum does not match -calculated checksum as if it does page could be valid unencrypted, -encrypted, or corrupted. - -@param[in] page Page to verify -@param[in] zip_size zip size -@param[in] space Tablespace -@param[in] pageno Page no -@return true if page is encrypted AND OK, false otherwise */ -UNIV_INTERN -bool -fil_space_verify_crypt_checksum( - byte* page, - ulint zip_size, - const fil_space_t* space, - ulint pageno) -{ - uint key_version = mach_read_from_4(page+ FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); - - /* If page is not encrypted, return false */ - if (key_version == 0) { - return(false); - } - - /* Read stored post encryption checksum. */ - ib_uint32_t checksum = mach_read_from_4( - page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4); - - /* Declare empty pages non-corrupted */ - if (checksum == 0 - && *reinterpret_cast(page + FIL_PAGE_LSN) == 0 - && buf_page_is_zeroes(page, zip_size)) { - return(true); - } - - /* Compressed and encrypted pages do not have checksum. Assume not - corrupted. Page verification happens after decompression in - buf_page_io_complete() using buf_page_is_corrupted(). */ - if (mach_read_from_2(page+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { - return (true); - } - - ib_uint32_t cchecksum1 = 0; - ib_uint32_t cchecksum2 = 0; - - /* Calculate checksums */ - if (zip_size) { - cchecksum1 = page_zip_calc_checksum( - page, zip_size, SRV_CHECKSUM_ALGORITHM_CRC32); - - if(cchecksum1 != checksum) { - cchecksum2 = page_zip_calc_checksum( - page, zip_size, - SRV_CHECKSUM_ALGORITHM_INNODB); - } - } else { - cchecksum1 = buf_calc_page_crc32(page); - - if (cchecksum1 != checksum) { - cchecksum2 = (ib_uint32_t) buf_calc_page_new_checksum( - page); - } - } - - /* If stored checksum matches one of the calculated checksums - page is not corrupted. */ - - bool encrypted = (checksum == cchecksum1 || checksum == cchecksum2 - || checksum == BUF_NO_CHECKSUM_MAGIC); - - /* MySQL 5.6 and MariaDB 10.0 and 10.1 will write an LSN to the - first page of each system tablespace file at - FIL_PAGE_FILE_FLUSH_LSN offset. On other pages and in other files, - the field might have been uninitialized until MySQL 5.5. In MySQL 5.7 - (and MariaDB Server 10.2.2) WL#7990 stopped writing the field for other - than page 0 of the system tablespace. - - Starting from MariaDB 10.1 the field has been repurposed for - encryption key_version. - - Starting with MySQL 5.7 (and MariaDB Server 10.2), the - field has been repurposed for SPATIAL INDEX pages for - FIL_RTREE_SPLIT_SEQ_NUM. - - Note that FIL_PAGE_FILE_FLUSH_LSN is not included in the InnoDB page - checksum. - - Thus, FIL_PAGE_FILE_FLUSH_LSN could contain any value. While the - field would usually be 0 for pages that are not encrypted, we cannot - assume that a nonzero value means that the page is encrypted. - Therefore we must validate the page both as encrypted and unencrypted - when FIL_PAGE_FILE_FLUSH_LSN does not contain 0. - */ - - ulint checksum1 = mach_read_from_4( - page + FIL_PAGE_SPACE_OR_CHKSUM); - - ulint checksum2 = checksum1; - - bool valid; - - if (zip_size) { - valid = (checksum1 == cchecksum1); - } else { - checksum2 = mach_read_from_4( - page + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM); - valid = (buf_page_is_checksum_valid_crc32(page,checksum1,checksum2) - || buf_page_is_checksum_valid_innodb(page,checksum1, checksum2)); - } - - if (encrypted && valid) { - /* If page is encrypted and traditional checksums match, - page could be still encrypted, or not encrypted and valid or - corrupted. */ - ib_logf(IB_LOG_LEVEL_ERROR, - " Page %lu in space %s (%lu) maybe corrupted." - " Post encryption checksum %u stored [%lu:%lu] key_version %u", - pageno, - space ? space->name : "N/A", - mach_read_from_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID), - checksum, checksum1, checksum2, key_version); - encrypted = false; - } - - return(encrypted); -} - /***********************************************************************/ /** A copy of global key state */ @@ -2555,8 +2436,9 @@ fil_space_crypt_close_tablespace( if (now >= last + 30) { ib_logf(IB_LOG_LEVEL_WARN, - "Waited %ld seconds to drop space: %s(" ULINTPF ").", - now - start, space->name, space->id); + "Waited %ld seconds to drop space: %s (" ULINTPF + ") active threads %u flushing=%d.", + now - start, space->name, space->id, cnt, flushing); last = now; } } @@ -2659,3 +2541,159 @@ fil_space_get_scrub_status( mutex_exit(&crypt_data->mutex); } } + +#endif /* !UNIV_INNOCHECKSUM */ + +/********************************************************************* +Verify that post encryption checksum match calculated checksum. +This function should be called only if tablespace contains crypt_data +metadata (this is strong indication that tablespace is encrypted). +Function also verifies that traditional checksum does not match +calculated checksum as if it does page could be valid unencrypted, +encrypted, or corrupted. + +@param[in] page Page to verify +@param[in] zip_size zip size +@param[in] space Tablespace +@param[in] pageno Page no +@return true if page is encrypted AND OK, false otherwise */ +UNIV_INTERN +bool +fil_space_verify_crypt_checksum( + byte* page, + ulint zip_size, +#ifndef UNIV_INNOCHECKSUM + const fil_space_t* space, +#else + const void* space, +#endif + ulint pageno) +{ + uint key_version = mach_read_from_4(page+ FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); + + /* If page is not encrypted, return false */ + if (key_version == 0) { + return(false); + } + + srv_checksum_algorithm_t algorithm = + static_cast(srv_checksum_algorithm); + + /* If no checksum is used, can't continue checking. */ + if (algorithm == SRV_CHECKSUM_ALGORITHM_NONE) { + return(true); + } + + /* Read stored post encryption checksum. */ + ib_uint32_t checksum = mach_read_from_4( + page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4); + + /* Declare empty pages non-corrupted */ + if (checksum == 0 + && *reinterpret_cast(page + FIL_PAGE_LSN) == 0 + && buf_page_is_zeroes(page, zip_size)) { + return(true); + } + + /* Compressed and encrypted pages do not have checksum. Assume not + corrupted. Page verification happens after decompression in + buf_page_io_complete() using buf_page_is_corrupted(). */ + if (mach_read_from_2(page+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { + return (true); + } + + ib_uint32_t cchecksum1 = 0; + ib_uint32_t cchecksum2 = 0; + + /* Calculate checksums */ + if (zip_size) { + cchecksum1 = page_zip_calc_checksum( + page, zip_size, SRV_CHECKSUM_ALGORITHM_CRC32); + + cchecksum2 = (cchecksum1 == checksum) + ? 0 + : page_zip_calc_checksum( + page, zip_size, + SRV_CHECKSUM_ALGORITHM_INNODB); + } else { + cchecksum1 = buf_calc_page_crc32(page); + cchecksum2 = (cchecksum1 == checksum) + ? 0 + : buf_calc_page_new_checksum(page); + } + + /* If stored checksum matches one of the calculated checksums + page is not corrupted. */ + + bool encrypted = (checksum == cchecksum1 || checksum == cchecksum2 + || checksum == BUF_NO_CHECKSUM_MAGIC); + + /* MySQL 5.6 and MariaDB 10.0 and 10.1 will write an LSN to the + first page of each system tablespace file at + FIL_PAGE_FILE_FLUSH_LSN offset. On other pages and in other files, + the field might have been uninitialized until MySQL 5.5. In MySQL 5.7 + (and MariaDB Server 10.2.2) WL#7990 stopped writing the field for other + than page 0 of the system tablespace. + + Starting from MariaDB 10.1 the field has been repurposed for + encryption key_version. + + Starting with MySQL 5.7 (and MariaDB Server 10.2), the + field has been repurposed for SPATIAL INDEX pages for + FIL_RTREE_SPLIT_SEQ_NUM. + + Note that FIL_PAGE_FILE_FLUSH_LSN is not included in the InnoDB page + checksum. + + Thus, FIL_PAGE_FILE_FLUSH_LSN could contain any value. While the + field would usually be 0 for pages that are not encrypted, we cannot + assume that a nonzero value means that the page is encrypted. + Therefore we must validate the page both as encrypted and unencrypted + when FIL_PAGE_FILE_FLUSH_LSN does not contain 0. + */ + + uint32_t checksum1 = mach_read_from_4(page + FIL_PAGE_SPACE_OR_CHKSUM); + uint32_t checksum2; + + bool valid; + + if (zip_size) { + valid = (checksum1 == cchecksum1); + checksum2 = checksum1; + } else { + checksum2 = mach_read_from_4( + page + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM); + valid = (buf_page_is_checksum_valid_crc32(page,checksum1,checksum2) + || buf_page_is_checksum_valid_innodb(page,checksum1, checksum2)); + } + + if (encrypted && valid) { + /* If page is encrypted and traditional checksums match, + page could be still encrypted, or not encrypted and valid or + corrupted. */ +#ifndef UNIV_INNOCHECKSUM + ib_logf(IB_LOG_LEVEL_ERROR, + " Page " ULINTPF " in space %s (" ULINTPF ") maybe corrupted." + " Post encryption checksum %u stored [%u:%u] key_version %u", + pageno, + space ? space->name : "N/A", + mach_read_from_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID), + checksum, checksum1, checksum2, key_version); +#else + if (log_file) { + fprintf(log_file, + "Page " ULINTPF ":" ULINTPF " may be corrupted." + " Post encryption checksum %u" + " stored [%u:%u] key_version %u\n", + pageno, + mach_read_from_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID), + checksum, checksum1, checksum2, + key_version); + } +#endif /* UNIV_INNOCHECKSUM */ + + encrypted = false; + } + + return(encrypted); +} diff --git a/storage/innobase/include/buf0buf.h b/storage/innobase/include/buf0buf.h index cabc6f64150..5fd3e05370f 100644 --- a/storage/innobase/include/buf0buf.h +++ b/storage/innobase/include/buf0buf.h @@ -31,6 +31,7 @@ Created 11/5/1995 Heikki Tuuri #include "fil0fil.h" #include "mtr0types.h" #include "buf0types.h" +#ifndef UNIV_INNOCHECKSUM #include "hash0hash.h" #include "ut0byte.h" #include "page0types.h" @@ -643,6 +644,8 @@ buf_block_unfix( # define buf_block_modify_clock_inc(block) ((void) 0) #endif /* !UNIV_HOTBACKUP */ +#endif /* !UNIV_INNOCHECKSUM */ + /** Checks if the page is in crc32 checksum format. @param[in] read_buf database page @param[in] checksum_field1 new checksum field @@ -691,8 +694,13 @@ buf_page_is_corrupted( bool check_lsn, const byte* read_buf, ulint zip_size, +#ifndef UNIV_INNOCHECKSUM const fil_space_t* space) +#else + const void* space = NULL) +#endif MY_ATTRIBUTE((warn_unused_result)); + /** Check if a page is all zeroes. @param[in] read_buf database page @param[in] zip_size ROW_FORMAT=COMPRESSED page size, or 0 @@ -700,6 +708,9 @@ buf_page_is_corrupted( UNIV_INTERN bool buf_page_is_zeroes(const byte* read_buf, ulint zip_size); + +#ifndef UNIV_INNOCHECKSUM + #ifndef UNIV_HOTBACKUP /**********************************************************************//** Gets the space id, page offset, and byte offset within page of a @@ -2470,4 +2481,5 @@ struct CheckUnzipLRUAndLRUList { #include "buf0buf.ic" #endif +#endif /*! UNIV_INNOCHECKSUM */ #endif diff --git a/storage/innobase/include/fil0crypt.h b/storage/innobase/include/fil0crypt.h index 228dfb895fe..7eb8b46d901 100644 --- a/storage/innobase/include/fil0crypt.h +++ b/storage/innobase/include/fil0crypt.h @@ -71,9 +71,11 @@ struct key_struct /** is encryption enabled */ extern ulong srv_encrypt_tables; +#ifndef UNIV_INNOCHECKSUM #ifdef UNIV_PFS_MUTEX extern mysql_pfs_key_t fil_crypt_data_mutex_key; #endif +#endif /* !UNIV_INNOCHECKSUM */ /** Mutex helper for crypt_data->scheme @param[in, out] schme encryption scheme @@ -102,6 +104,8 @@ struct fil_space_rotate_state_t } scrubbing; }; +#ifndef UNIV_INNOCHECKSUM + struct fil_space_crypt_t : st_encryption_scheme { public: @@ -399,6 +403,8 @@ fil_crypt_calculate_checksum( const byte* dst_frame) MY_ATTRIBUTE((warn_unused_result)); +#endif /* UNIV_INNOCHECKSUM */ + /********************************************************************* Verify that post encryption checksum match calculated checksum. This function should be called only if tablespace contains crypt_data @@ -417,10 +423,16 @@ bool fil_space_verify_crypt_checksum( byte* page, ulint zip_size, +#ifndef UNIV_INNOCHECKSUM const fil_space_t* space, +#else + const void* space, +#endif ulint pageno) MY_ATTRIBUTE((warn_unused_result)); +#ifndef UNIV_INNOCHECKSUM + /********************************************************************* Adjust thread count for key rotation @param[in] enw_cnt Number of threads to be used */ @@ -508,4 +520,5 @@ fil_space_get_scrub_status( #include "fil0crypt.ic" #endif +#endif /* !UNIV_INNOCHECKSUM */ #endif /* fil0crypt_h */ diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index 8033d0cc839..e481ea3a8aa 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -183,8 +183,6 @@ extern fil_addr_t fil_addr_null; #define FIL_LOG 502 /*!< redo log */ /* @} */ -#ifndef UNIV_INNOCHECKSUM - /** Structure containing encryption specification */ struct fil_space_crypt_t; @@ -209,6 +207,10 @@ extern ulint fil_n_pending_tablespace_flushes; /** Number of files currently open */ extern ulint fil_n_file_opened; +#ifndef UNIV_INNOCHECKSUM + +struct fil_space_t; + struct fsp_open_info { ibool success; /*!< Has the tablespace been opened? */ const char* check_msg; /*!< fil_check_first_page() message */ @@ -225,8 +227,6 @@ struct fsp_open_info { dict_table_t* table; /*!< table */ }; -struct fil_space_t; - /** File node of a tablespace or the log data space */ struct fil_node_t { fil_space_t* space; /*!< backpointer to the space where this node diff --git a/storage/innobase/include/mach0data.ic b/storage/innobase/include/mach0data.ic index 33b6405c8b5..72e793da8fd 100644 --- a/storage/innobase/include/mach0data.ic +++ b/storage/innobase/include/mach0data.ic @@ -154,6 +154,8 @@ mach_read_from_3( ); } +#endif /* !UNIV_INNOCHECKSUM */ + /*******************************************************//** The following function is used to store data in four consecutive bytes. We store the most significant byte to the lowest address. */ @@ -172,8 +174,6 @@ mach_write_to_4( b[3] = (byte) n; } -#endif /* !UNIV_INNOCHECKSUM */ - /********************************************************//** The following function is used to fetch data from 4 consecutive bytes. The most significant byte is at the lowest address. diff --git a/storage/innobase/include/univ.i b/storage/innobase/include/univ.i index 1cc89ae503c..aafb84db6f1 100644 --- a/storage/innobase/include/univ.i +++ b/storage/innobase/include/univ.i @@ -482,6 +482,12 @@ typedef long int lint; typedef unsigned long long int ullint; #endif /* UNIV_HOTBACKUP */ +#ifdef UNIV_INNOCHECKSUM +extern bool strict_verify; +extern FILE* log_file; +extern ulint cur_page_num; +#endif /* UNIV_INNOCHECKSUM */ + #ifndef __WIN__ #if SIZEOF_LONG != SIZEOF_VOIDP #error "Error: InnoDB's ulint must be of the same size as void*" diff --git a/storage/innobase/page/page0zip.cc b/storage/innobase/page/page0zip.cc index b4a8719c920..6e26f830fe6 100644 --- a/storage/innobase/page/page0zip.cc +++ b/storage/innobase/page/page0zip.cc @@ -4937,26 +4937,26 @@ page_zip_verify_checksum( #error "FIL_PAGE_LSN must be 64 bit aligned" #endif -#ifndef UNIV_INNOCHECKSUM - /* innochecksum doesn't compile with ut_d. Since we don't - need to check for empty pages when running innochecksum, - just don't include this code. */ /* Check if page is empty */ if (stored == 0 && *reinterpret_cast(static_cast( data) + FIL_PAGE_LSN) == 0) { /* make sure that the page is really empty */ - ulint i; - for (i = 0; i < size; i++) { + for (ulint i = 0; i < size; i++) { if (*((const char*) data + i) != 0) { return(FALSE); } } +#ifdef UNIV_INNOCHECKSUM + if (log_file) { + fprintf(log_file, "Page::%lu is empty and" + " uncorrupted\n", cur_page_num); + } +#endif /* UNIV_INNOCHECKSUM */ /* Empty page */ return(TRUE); } -#endif const srv_checksum_algorithm_t curr_algo = static_cast(srv_checksum_algorithm); @@ -4968,6 +4968,33 @@ page_zip_verify_checksum( calc = static_cast(page_zip_calc_checksum( data, size, curr_algo)); +#ifdef UNIV_INNOCHECKSUM + if (log_file) { + fprintf(log_file, "page::%lu;" + " %s checksum: calculated = %u;" + " recorded = %u\n", cur_page_num, + buf_checksum_algorithm_name( + static_cast( + srv_checksum_algorithm)), + calc, stored); + } + + if (!strict_verify) { + + const uint32_t crc32 = page_zip_calc_checksum( + data, size, SRV_CHECKSUM_ALGORITHM_CRC32); + + if (log_file) { + fprintf(log_file, "page::%lu: crc32 checksum:" + " calculated = %u; recorded = %u\n", + cur_page_num, crc32, stored); + fprintf(log_file, "page::%lu: none checksum:" + " calculated = %lu; recorded = %u\n", + cur_page_num, BUF_NO_CHECKSUM_MAGIC, stored); + } + } +#endif /* UNIV_INNOCHECKSUM */ + if (stored == calc) { return(TRUE); } diff --git a/storage/xtradb/fil/fil0crypt.cc b/storage/xtradb/fil/fil0crypt.cc index 6d3c2e98010..b5b762c2cd9 100644 --- a/storage/xtradb/fil/fil0crypt.cc +++ b/storage/xtradb/fil/fil0crypt.cc @@ -25,13 +25,18 @@ Modified Jan Lindström jan.lindstrom@mariadb.com #include "fil0fil.h" #include "fil0crypt.h" +#include "mach0data.h" +#include "page0zip.h" +#include "buf0buf.h" +#include "buf0checksum.h" + +#ifndef UNIV_INNOCHECKSUM + #include "srv0srv.h" #include "srv0start.h" -#include "mach0data.h" #include "log0recv.h" #include "mtr0mtr.h" #include "mtr0log.h" -#include "page0zip.h" #include "ut0ut.h" #include "btr0scrub.h" #include "fsp0fsp.h" @@ -107,13 +112,20 @@ UNIV_INTERN mysql_pfs_key_t fil_crypt_data_mutex_key; extern my_bool srv_background_scrub_data_uncompressed; extern my_bool srv_background_scrub_data_compressed; +/*********************************************************************** +Check if a key needs rotation given a key_state +@param[in] encrypt_mode Encryption mode +@param[in] key_version Current key version +@param[in] latest_key_version Latest key version +@param[in] rotate_key_age when to rotate +@return true if key needs rotation, false if not */ static bool fil_crypt_needs_rotation( - fil_encryption_t encrypt_mode, /*!< in: Encryption - mode */ - uint key_version, /*!< in: Key version */ - uint latest_key_version, /*!< in: Latest key version */ - uint rotate_key_age); /*!< in: When to rotate */ + fil_encryption_t encrypt_mode, + uint key_version, + uint latest_key_version, + uint rotate_key_age) + MY_ATTRIBUTE((warn_unused_result)); /********************************************************************* Init space crypt */ @@ -908,137 +920,6 @@ fil_crypt_calculate_checksum( return checksum; } -/********************************************************************* -Verify that post encryption checksum match calculated checksum. -This function should be called only if tablespace contains crypt_data -metadata (this is strong indication that tablespace is encrypted). -Function also verifies that traditional checksum does not match -calculated checksum as if it does page could be valid unencrypted, -encrypted, or corrupted. - -@param[in] page Page to verify -@param[in] zip_size zip size -@param[in] space Tablespace -@param[in] pageno Page no -@return true if page is encrypted AND OK, false otherwise */ -UNIV_INTERN -bool -fil_space_verify_crypt_checksum( - byte* page, - ulint zip_size, - const fil_space_t* space, - ulint pageno) -{ - uint key_version = mach_read_from_4(page+ FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); - - /* If page is not encrypted, return false */ - if (key_version == 0) { - return(false); - } - - /* Read stored post encryption checksum. */ - ib_uint32_t checksum = mach_read_from_4( - page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4); - - /* Declare empty pages non-corrupted */ - if (checksum == 0 - && *reinterpret_cast(page + FIL_PAGE_LSN) == 0 - && buf_page_is_zeroes(page, zip_size)) { - return(true); - } - - /* Compressed and encrypted pages do not have checksum. Assume not - corrupted. Page verification happens after decompression in - buf_page_io_complete() using buf_page_is_corrupted(). */ - if (mach_read_from_2(page+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { - return (true); - } - - ib_uint32_t cchecksum1 = 0; - ib_uint32_t cchecksum2 = 0; - - /* Calculate checksums */ - if (zip_size) { - cchecksum1 = page_zip_calc_checksum( - page, zip_size, SRV_CHECKSUM_ALGORITHM_CRC32); - - if(cchecksum1 != checksum) { - cchecksum2 = page_zip_calc_checksum( - page, zip_size, - SRV_CHECKSUM_ALGORITHM_INNODB); - } - } else { - cchecksum1 = buf_calc_page_crc32(page); - - if (cchecksum1 != checksum) { - cchecksum2 = (ib_uint32_t) buf_calc_page_new_checksum( - page); - } - } - - /* If stored checksum matches one of the calculated checksums - page is not corrupted. */ - - bool encrypted = (checksum == cchecksum1 || checksum == cchecksum2 - || checksum == BUF_NO_CHECKSUM_MAGIC); - - /* MySQL 5.6 and MariaDB 10.0 and 10.1 will write an LSN to the - first page of each system tablespace file at - FIL_PAGE_FILE_FLUSH_LSN offset. On other pages and in other files, - the field might have been uninitialized until MySQL 5.5. In MySQL 5.7 - (and MariaDB Server 10.2.2) WL#7990 stopped writing the field for other - than page 0 of the system tablespace. - - Starting from MariaDB 10.1 the field has been repurposed for - encryption key_version. - - Starting with MySQL 5.7 (and MariaDB Server 10.2), the - field has been repurposed for SPATIAL INDEX pages for - FIL_RTREE_SPLIT_SEQ_NUM. - - Note that FIL_PAGE_FILE_FLUSH_LSN is not included in the InnoDB page - checksum. - - Thus, FIL_PAGE_FILE_FLUSH_LSN could contain any value. While the - field would usually be 0 for pages that are not encrypted, we cannot - assume that a nonzero value means that the page is encrypted. - Therefore we must validate the page both as encrypted and unencrypted - when FIL_PAGE_FILE_FLUSH_LSN does not contain 0. - */ - - ulint checksum1 = mach_read_from_4( - page + FIL_PAGE_SPACE_OR_CHKSUM); - - ulint checksum2 = checksum1; - - bool valid; - - if (zip_size) { - valid = (checksum1 == cchecksum1); - } else { - checksum1 = mach_read_from_4( - page + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM); - valid = (buf_page_is_checksum_valid_crc32(page,checksum1,checksum2) - || buf_page_is_checksum_valid_innodb(page,checksum1, checksum2)); - } - - if (encrypted && valid) { - /* If page is encrypted and traditional checksums match, - page could be still encrypted, or not encrypted and valid or - corrupted. */ - ib_logf(IB_LOG_LEVEL_ERROR, - " Page %lu in space %s (%lu) maybe corrupted." - " Post encryption checksum %u stored [%lu:%lu] key_version %u", - pageno, - space ? space->name : "N/A", - mach_read_from_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID), - checksum, checksum1, checksum2, key_version); - encrypted = false; - } - - return(encrypted); -} - /***********************************************************************/ /** A copy of global key state */ @@ -2660,3 +2541,159 @@ fil_space_get_scrub_status( mutex_exit(&crypt_data->mutex); } } + +#endif /* !UNIV_INNOCHECKSUM */ + +/********************************************************************* +Verify that post encryption checksum match calculated checksum. +This function should be called only if tablespace contains crypt_data +metadata (this is strong indication that tablespace is encrypted). +Function also verifies that traditional checksum does not match +calculated checksum as if it does page could be valid unencrypted, +encrypted, or corrupted. + +@param[in] page Page to verify +@param[in] zip_size zip size +@param[in] space Tablespace +@param[in] pageno Page no +@return true if page is encrypted AND OK, false otherwise */ +UNIV_INTERN +bool +fil_space_verify_crypt_checksum( + byte* page, + ulint zip_size, +#ifndef UNIV_INNOCHECKSUM + const fil_space_t* space, +#else + const void* space, +#endif + ulint pageno) +{ + uint key_version = mach_read_from_4(page+ FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); + + /* If page is not encrypted, return false */ + if (key_version == 0) { + return(false); + } + + srv_checksum_algorithm_t algorithm = + static_cast(srv_checksum_algorithm); + + /* If no checksum is used, can't continue checking. */ + if (algorithm == SRV_CHECKSUM_ALGORITHM_NONE) { + return(true); + } + + /* Read stored post encryption checksum. */ + ib_uint32_t checksum = mach_read_from_4( + page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4); + + /* Declare empty pages non-corrupted */ + if (checksum == 0 + && *reinterpret_cast(page + FIL_PAGE_LSN) == 0 + && buf_page_is_zeroes(page, zip_size)) { + return(true); + } + + /* Compressed and encrypted pages do not have checksum. Assume not + corrupted. Page verification happens after decompression in + buf_page_io_complete() using buf_page_is_corrupted(). */ + if (mach_read_from_2(page+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { + return (true); + } + + ib_uint32_t cchecksum1 = 0; + ib_uint32_t cchecksum2 = 0; + + /* Calculate checksums */ + if (zip_size) { + cchecksum1 = page_zip_calc_checksum( + page, zip_size, SRV_CHECKSUM_ALGORITHM_CRC32); + + cchecksum2 = (cchecksum1 == checksum) + ? 0 + : page_zip_calc_checksum( + page, zip_size, + SRV_CHECKSUM_ALGORITHM_INNODB); + } else { + cchecksum1 = buf_calc_page_crc32(page); + cchecksum2 = (cchecksum1 == checksum) + ? 0 + : buf_calc_page_new_checksum(page); + } + + /* If stored checksum matches one of the calculated checksums + page is not corrupted. */ + + bool encrypted = (checksum == cchecksum1 || checksum == cchecksum2 + || checksum == BUF_NO_CHECKSUM_MAGIC); + + /* MySQL 5.6 and MariaDB 10.0 and 10.1 will write an LSN to the + first page of each system tablespace file at + FIL_PAGE_FILE_FLUSH_LSN offset. On other pages and in other files, + the field might have been uninitialized until MySQL 5.5. In MySQL 5.7 + (and MariaDB Server 10.2.2) WL#7990 stopped writing the field for other + than page 0 of the system tablespace. + + Starting from MariaDB 10.1 the field has been repurposed for + encryption key_version. + + Starting with MySQL 5.7 (and MariaDB Server 10.2), the + field has been repurposed for SPATIAL INDEX pages for + FIL_RTREE_SPLIT_SEQ_NUM. + + Note that FIL_PAGE_FILE_FLUSH_LSN is not included in the InnoDB page + checksum. + + Thus, FIL_PAGE_FILE_FLUSH_LSN could contain any value. While the + field would usually be 0 for pages that are not encrypted, we cannot + assume that a nonzero value means that the page is encrypted. + Therefore we must validate the page both as encrypted and unencrypted + when FIL_PAGE_FILE_FLUSH_LSN does not contain 0. + */ + + uint32_t checksum1 = mach_read_from_4(page + FIL_PAGE_SPACE_OR_CHKSUM); + uint32_t checksum2; + + bool valid; + + if (zip_size) { + valid = (checksum1 == cchecksum1); + checksum2 = checksum1; + } else { + checksum2 = mach_read_from_4( + page + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM); + valid = (buf_page_is_checksum_valid_crc32(page,checksum1,checksum2) + || buf_page_is_checksum_valid_innodb(page,checksum1, checksum2)); + } + + if (encrypted && valid) { + /* If page is encrypted and traditional checksums match, + page could be still encrypted, or not encrypted and valid or + corrupted. */ +#ifndef UNIV_INNOCHECKSUM + ib_logf(IB_LOG_LEVEL_ERROR, + " Page " ULINTPF " in space %s (" ULINTPF ") maybe corrupted." + " Post encryption checksum %u stored [%u:%u] key_version %u", + pageno, + space ? space->name : "N/A", + mach_read_from_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID), + checksum, checksum1, checksum2, key_version); +#else + if (log_file) { + fprintf(log_file, + "Page " ULINTPF ":" ULINTPF " may be corrupted." + " Post encryption checksum %u" + " stored [%u:%u] key_version %u\n", + pageno, + mach_read_from_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID), + checksum, checksum1, checksum2, + key_version); + } +#endif /* UNIV_INNOCHECKSUM */ + + encrypted = false; + } + + return(encrypted); +}