From 2877c5ecc22ab0323ab86ce548a81909d7c865c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Mon, 19 Jan 2015 12:39:17 +0200 Subject: [PATCH] MDEV-7477: Make innochecksum work on compressed tables This patch ports the work that facebook has performed to make innochecksum handle compressed tables. the basic idea is to use actual innodb-code to perform checksum verification rather than duplicating in innochecksum.cc. to make this work, innodb code has been annotated with lots of #ifndef UNIV_INNOCHECKSUM so that it can be compiled outside of storage/innobase. A new testcase is also added that verifies that innochecksum works on compressed/non-compressed tables. Merged from commit fabc79d2ea976c4ff5b79bfe913e6bc03ef69d42 from https://code.google.com/p/google-mysql/ The actual steps to produce this patch are: take innochecksum from 5.6.14 apply changes in innodb from facebook patches needed to make innochecksum compile apply changes in innochecksum from facebook patches add handcrafted testcase The referenced facebook patches used are: https://github.com/facebook/mysql-5.6/commit/91e25120e75272db4cdbc07d0e45877d9dea5715 https://github.com/facebook/mysql-5.6/commit/847fe76ea5239b09fa361b023c56e6be76d32046 https://github.com/facebook/mysql-5.6/commit/1135628a5a9b3412621b93233478f3804bcef51a https://github.com/facebook/mysql-5.6/commit/4dbf7c240ce2f08b7d6572d9452c9779ce90641c --- extra/CMakeLists.txt | 22 +- extra/innochecksum.c | 325 ------------ extra/innochecksum.cc | 470 ++++++++++++++++++ mysql-test/mysql-test-run.pl | 11 + mysql-test/suite/innodb/r/innochecksum.result | 31 ++ mysql-test/suite/innodb/t/innochecksum.opt | 2 + mysql-test/suite/innodb/t/innochecksum.test | 70 +++ storage/innobase/buf/buf0checksum.cc | 5 +- storage/innobase/include/buf0buf.h | 3 - storage/innobase/include/buf0checksum.h | 9 +- storage/innobase/include/fil0fil.h | 9 +- storage/innobase/include/mach0data.ic | 8 + storage/innobase/include/page0page.h | 14 + storage/innobase/include/page0page.ic | 19 +- storage/innobase/include/page0types.h | 2 + storage/innobase/include/page0zip.h | 18 +- storage/innobase/include/rem0rec.h | 4 + storage/innobase/include/srv0srv.h | 1 - storage/innobase/include/trx0undo.h | 8 + storage/innobase/include/univ.i | 2 +- storage/innobase/include/ut0ut.h | 5 + storage/innobase/page/page0zip.cc | 22 +- 22 files changed, 713 insertions(+), 347 deletions(-) delete mode 100644 extra/innochecksum.c create mode 100644 extra/innochecksum.cc create mode 100644 mysql-test/suite/innodb/r/innochecksum.result create mode 100644 mysql-test/suite/innodb/t/innochecksum.opt create mode 100644 mysql-test/suite/innodb/t/innochecksum.test diff --git a/extra/CMakeLists.txt b/extra/CMakeLists.txt index f8f71b00743..585b5aef6f6 100644 --- a/extra/CMakeLists.txt +++ b/extra/CMakeLists.txt @@ -75,11 +75,29 @@ ENDIF() MYSQL_ADD_EXECUTABLE(replace replace.c COMPONENT Server) TARGET_LINK_LIBRARIES(replace mysys) IF(UNIX) - MYSQL_ADD_EXECUTABLE(innochecksum innochecksum.c) - MYSQL_ADD_EXECUTABLE(resolve_stack_dump resolve_stack_dump.c) TARGET_LINK_LIBRARIES(resolve_stack_dump mysys) MYSQL_ADD_EXECUTABLE(mysql_waitpid mysql_waitpid.c COMPONENT Client) TARGET_LINK_LIBRARIES(mysql_waitpid mysys) ENDIF() + + + # Add path to the InnoDB headers + INCLUDE_DIRECTORIES( + ${CMAKE_SOURCE_DIR}/storage/innobase/include + ${CMAKE_SOURCE_DIR}/sql) + + # We use the InnoDB code directly in case the code changes. + ADD_DEFINITIONS("-DUNIV_INNOCHECKSUM") + SET(INNOBASE_SOURCES + ../storage/innobase/buf/buf0checksum.cc + ../storage/innobase/ut/ut0crc32.cc + ../storage/innobase/ut/ut0ut.cc + ../storage/innobase/page/page0zip.cc + ) + + MYSQL_ADD_EXECUTABLE(innochecksum innochecksum.cc ${INNOBASE_SOURCES}) + TARGET_LINK_LIBRARIES(innochecksum mysys mysys_ssl) + ADD_DEPENDENCIES(innochecksum GenError) + diff --git a/extra/innochecksum.c b/extra/innochecksum.c deleted file mode 100644 index ed4dfc48789..00000000000 --- a/extra/innochecksum.c +++ /dev/null @@ -1,325 +0,0 @@ -/* - Copyright (c) 2005, 2011, Oracle and/or its affiliates - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; version 2 of the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -*/ - -/* - InnoDB offline file checksum utility. 85% of the code in this file - was taken wholesale fron the InnoDB codebase. - - The final 15% was originally written by Mark Smith of Danga - Interactive, Inc. - - Published with a permission. -*/ - -#include -#include -#include -#include -#include -#include -#include - -/* all of these ripped from InnoDB code from MySQL 4.0.22 */ -#define UT_HASH_RANDOM_MASK 1463735687 -#define UT_HASH_RANDOM_MASK2 1653893711 -#define FIL_PAGE_LSN 16 -#define FIL_PAGE_FILE_FLUSH_LSN 26 -#define FIL_PAGE_OFFSET 4 -#define FIL_PAGE_DATA 38 -#define FIL_PAGE_END_LSN_OLD_CHKSUM 8 -#define FIL_PAGE_SPACE_OR_CHKSUM 0 -#define UNIV_PAGE_SIZE (2 * 8192) - -/* command line argument to do page checks (that's it) */ -/* another argument to specify page ranges... seek to right spot and go from there */ - -typedef unsigned long int ulint; - -/* innodb function in name; modified slightly to not have the ASM version (lots of #ifs that didn't apply) */ -ulint mach_read_from_4(uchar *b) -{ - return( ((ulint)(b[0]) << 24) - + ((ulint)(b[1]) << 16) - + ((ulint)(b[2]) << 8) - + (ulint)(b[3]) - ); -} - -ulint -ut_fold_ulint_pair( -/*===============*/ - /* out: folded value */ - ulint n1, /* in: ulint */ - ulint n2) /* in: ulint */ -{ - return(((((n1 ^ n2 ^ UT_HASH_RANDOM_MASK2) << 8) + n1) - ^ UT_HASH_RANDOM_MASK) + n2); -} - -ulint -ut_fold_binary( -/*===========*/ - /* out: folded value */ - uchar* str, /* in: string of bytes */ - ulint len) /* in: length */ -{ - ulint i; - ulint fold= 0; - - for (i= 0; i < len; i++) - { - fold= ut_fold_ulint_pair(fold, (ulint)(*str)); - - str++; - } - - return(fold); -} - -ulint -buf_calc_page_new_checksum( -/*=======================*/ - /* out: checksum */ - uchar* page) /* in: buffer page */ -{ - ulint checksum; - - /* Since the fields FIL_PAGE_FILE_FLUSH_LSN and ..._ARCH_LOG_NO - are written outside the buffer pool to the first pages of data - files, we have to skip them in the page checksum calculation. - We must also skip the field FIL_PAGE_SPACE_OR_CHKSUM where the - checksum is stored, and also the last 8 bytes of page because - there we store the old formula checksum. */ - - checksum= ut_fold_binary(page + FIL_PAGE_OFFSET, - FIL_PAGE_FILE_FLUSH_LSN - FIL_PAGE_OFFSET) - + ut_fold_binary(page + FIL_PAGE_DATA, - UNIV_PAGE_SIZE - FIL_PAGE_DATA - - FIL_PAGE_END_LSN_OLD_CHKSUM); - checksum= checksum & 0xFFFFFFFF; - - return(checksum); -} - -ulint -buf_calc_page_old_checksum( -/*=======================*/ - /* out: checksum */ - uchar* page) /* in: buffer page */ -{ - ulint checksum; - - checksum= ut_fold_binary(page, FIL_PAGE_FILE_FLUSH_LSN); - - checksum= checksum & 0xFFFFFFFF; - - return(checksum); -} - - -int main(int argc, char **argv) -{ - FILE *f; /* our input file */ - uchar *p; /* storage of pages read */ - int bytes; /* bytes read count */ - ulint ct; /* current page number (0 based) */ - int now; /* current time */ - int lastt; /* last time */ - ulint oldcsum, oldcsumfield, csum, csumfield, logseq, logseqfield; /* ulints for checksum storage */ - struct stat st; /* for stat, if you couldn't guess */ - unsigned long long int size; /* size of file (has to be 64 bits) */ - ulint pages; /* number of pages in file */ - ulint start_page= 0, end_page= 0, use_end_page= 0; /* for starting and ending at certain pages */ - off_t offset= 0; - int just_count= 0; /* if true, just print page count */ - int verbose= 0; - int debug= 0; - int c; - int fd; - - /* remove arguments */ - while ((c= getopt(argc, argv, "cvds:e:p:")) != -1) - { - switch (c) - { - case 'v': - verbose= 1; - break; - case 'c': - just_count= 1; - break; - case 's': - start_page= atoi(optarg); - break; - case 'e': - end_page= atoi(optarg); - use_end_page= 1; - break; - case 'p': - start_page= atoi(optarg); - end_page= atoi(optarg); - use_end_page= 1; - break; - case 'd': - debug= 1; - break; - case ':': - fprintf(stderr, "option -%c requires an argument\n", optopt); - return 1; - break; - case '?': - fprintf(stderr, "unrecognized option: -%c\n", optopt); - return 1; - break; - } - } - - /* debug implies verbose... */ - if (debug) verbose= 1; - - /* make sure we have the right arguments */ - if (optind >= argc) - { - printf("InnoDB offline file checksum utility.\n"); - printf("usage: %s [-c] [-s ] [-e ] [-p ] [-v] [-d] \n", argv[0]); - printf("\t-c\tprint the count of pages in the file\n"); - printf("\t-s n\tstart on this page number (0 based)\n"); - printf("\t-e n\tend at this page number (0 based)\n"); - printf("\t-p n\tcheck only this page (0 based)\n"); - printf("\t-v\tverbose (prints progress every 5 seconds)\n"); - printf("\t-d\tdebug mode (prints checksums for each page)\n"); - return 1; - } - - /* stat the file to get size and page count */ - if (stat(argv[optind], &st)) - { - perror("error statting file"); - return 1; - } - size= st.st_size; - pages= size / UNIV_PAGE_SIZE; - if (just_count) - { - printf("%lu\n", pages); - return 0; - } - else if (verbose) - { - printf("file %s = %llu bytes (%lu pages)...\n", argv[optind], size, pages); - printf("checking pages in range %lu to %lu\n", start_page, use_end_page ? end_page : (pages - 1)); - } - - /* open the file for reading */ - f= fopen(argv[optind], "r"); - if (!f) - { - perror("error opening file"); - return 1; - } - - /* seek to the necessary position */ - if (start_page) - { - fd= fileno(f); - if (!fd) - { - perror("unable to obtain file descriptor number"); - return 1; - } - - offset= (off_t)start_page * (off_t)UNIV_PAGE_SIZE; - - if (lseek(fd, offset, SEEK_SET) != offset) - { - perror("unable to seek to necessary offset"); - return 1; - } - } - - /* allocate buffer for reading (so we don't realloc every time) */ - p= (uchar *)malloc(UNIV_PAGE_SIZE); - - /* main checksumming loop */ - ct= start_page; - lastt= 0; - while (!feof(f)) - { - bytes= fread(p, 1, UNIV_PAGE_SIZE, f); - if (!bytes && feof(f)) return 0; - if (bytes != UNIV_PAGE_SIZE) - { - fprintf(stderr, "bytes read (%d) doesn't match universal page size (%d)\n", bytes, UNIV_PAGE_SIZE); - return 1; - } - - /* check the "stored log sequence numbers" */ - logseq= mach_read_from_4(p + FIL_PAGE_LSN + 4); - logseqfield= mach_read_from_4(p + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM + 4); - if (debug) - printf("page %lu: log sequence number: first = %lu; second = %lu\n", ct, logseq, logseqfield); - if (logseq != logseqfield) - { - fprintf(stderr, "page %lu invalid (fails log sequence number check)\n", ct); - return 1; - } - - /* check old method of checksumming */ - oldcsum= buf_calc_page_old_checksum(p); - oldcsumfield= mach_read_from_4(p + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM); - if (debug) - printf("page %lu: old style: calculated = %lu; recorded = %lu\n", ct, oldcsum, oldcsumfield); - if (oldcsumfield != mach_read_from_4(p + FIL_PAGE_LSN) && oldcsumfield != oldcsum) - { - fprintf(stderr, "page %lu invalid (fails old style checksum)\n", ct); - return 1; - } - - /* now check the new method */ - csum= buf_calc_page_new_checksum(p); - csumfield= mach_read_from_4(p + FIL_PAGE_SPACE_OR_CHKSUM); - if (debug) - printf("page %lu: new style: calculated = %lu; recorded = %lu\n", ct, csum, csumfield); - if (csumfield != 0 && csum != csumfield) - { - fprintf(stderr, "page %lu invalid (fails new style checksum)\n", ct); - return 1; - } - - /* end if this was the last page we were supposed to check */ - if (use_end_page && (ct >= end_page)) - return 0; - - /* do counter increase and progress printing */ - ct++; - if (verbose) - { - if (ct % 64 == 0) - { - now= time(0); - if (!lastt) lastt= now; - if (now - lastt >= 1) - { - printf("page %lu okay: %.3f%% done\n", (ct - 1), (float) ct / pages * 100); - lastt= now; - } - } - } - } - return 0; -} - diff --git a/extra/innochecksum.cc b/extra/innochecksum.cc new file mode 100644 index 00000000000..5a0f7c630d3 --- /dev/null +++ b/extra/innochecksum.cc @@ -0,0 +1,470 @@ +/* + Copyright (c) 2005, 2012, Oracle and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +/* + InnoDB offline file checksum utility. 85% of the code in this utility + is included from the InnoDB codebase. + + The final 15% was originally written by Mark Smith of Danga + Interactive, Inc. + + Published with a permission. +*/ + +#include +#include +#include +#include +#include +#include +#ifndef __WIN__ +# include +#endif +#include +#include +#include /* ORACLE_WELCOME_COPYRIGHT_NOTICE */ +#include + +/* Only parts of these files are included from the InnoDB codebase. +The parts not included are excluded by #ifndef UNIV_INNOCHECKSUM. */ + +#include "univ.i" /* include all of this */ + +#define FLST_NODE_SIZE (2 * FIL_ADDR_SIZE) +#define FSEG_PAGE_DATA FIL_PAGE_DATA + +#include "ut0ut.h" +#include "ut0byte.h" +#include "mach0data.h" +#include "fsp0types.h" +#include "rem0rec.h" +#include "buf0checksum.h" /* buf_calc_page_*() */ +#include "fil0fil.h" /* FIL_* */ +#include "page0page.h" /* PAGE_* */ +#include "page0zip.h" /* page_zip_*() */ +#include "trx0undo.h" /* TRX_* */ +#include "fsp0fsp.h" /* fsp_flags_get_page_size() & + fsp_flags_get_zip_size() */ +#include "mach0data.h" /* mach_read_from_4() */ +#include "ut0crc32.h" /* ut_crc32_init() */ + +#ifdef UNIV_NONINL +# include "fsp0fsp.ic" +# include "mach0data.ic" +# include "ut0rnd.ic" +#endif + +/* Global variables */ +static my_bool verbose; +static my_bool debug; +static my_bool skip_corrupt; +static my_bool just_count; +static ulong start_page; +static ulong end_page; +static ulong do_page; +static my_bool use_end_page; +static my_bool do_one_page; +ulong srv_page_size; /* replaces declaration in srv0srv.c */ +static ulong physical_page_size; /* Page size in bytes on disk. */ +static ulong logical_page_size; /* Page size when uncompressed. */ +static bool compressed= false; /* Is tablespace compressed */ + +/* Get the page size of the filespace from the filespace header. */ +static +my_bool +get_page_size( +/*==========*/ + FILE* f, /*!< in: file pointer, must be open + and set to start of file */ + byte* buf, /*!< in: buffer used to read the page */ + ulong* logical_page_size, /*!< out: Logical/Uncompressed page size */ + ulong* physical_page_size) /*!< out: Physical/Commpressed page size */ +{ + ulong flags; + + int bytes= fread(buf, 1, UNIV_PAGE_SIZE_MIN, f); + + if (ferror(f)) + { + perror("Error reading file header"); + return FALSE; + } + + if (bytes != UNIV_PAGE_SIZE_MIN) + { + fprintf(stderr, "Error; Was not able to read the minimum page size "); + fprintf(stderr, "of %d bytes. Bytes read was %d\n", UNIV_PAGE_SIZE_MIN, bytes); + return FALSE; + } + + rewind(f); + + flags = mach_read_from_4(buf + FIL_PAGE_DATA + FSP_SPACE_FLAGS); + + /* srv_page_size is used by InnoDB code as UNIV_PAGE_SIZE */ + srv_page_size = *logical_page_size = fsp_flags_get_page_size(flags); + + /* fsp_flags_get_zip_size() will return zero if not compressed. */ + *physical_page_size = fsp_flags_get_zip_size(flags); + if (*physical_page_size == 0) + { + *physical_page_size= *logical_page_size; + } + else + { + compressed= true; + } + return TRUE; +} + + +/* command line argument to do page checks (that's it) */ +/* another argument to specify page ranges... seek to right spot and go from there */ + +static struct my_option innochecksum_options[] = +{ + {"help", '?', "Displays this help and exits.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"info", 'I', "Synonym for --help.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"version", 'V', "Displays version information and exits.", + 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"verbose", 'v', "Verbose (prints progress every 5 seconds).", + &verbose, &verbose, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"debug", 'd', "Debug mode (prints checksums for each page, implies verbose).", + &debug, &debug, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"skip_corrupt", 'u', "Skip corrupt pages.", + &skip_corrupt, &skip_corrupt, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"count", 'c', "Print the count of pages in the file.", + &just_count, &just_count, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"start_page", 's', "Start on this page number (0 based).", + &start_page, &start_page, 0, GET_ULONG, REQUIRED_ARG, + 0, 0, (longlong) 2L*1024L*1024L*1024L, 0, 1, 0}, + {"end_page", 'e', "End at this page number (0 based).", + &end_page, &end_page, 0, GET_ULONG, REQUIRED_ARG, + 0, 0, (longlong) 2L*1024L*1024L*1024L, 0, 1, 0}, + {"page", 'p', "Check only this page (0 based).", + &do_page, &do_page, 0, GET_ULONG, REQUIRED_ARG, + 0, 0, (longlong) 2L*1024L*1024L*1024L, 0, 1, 0}, + {0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0} +}; + +static void print_version(void) +{ + printf("%s Ver %s, for %s (%s)\n", + my_progname, INNODB_VERSION_STR, + SYSTEM_TYPE, MACHINE_TYPE); +} + +static void usage(void) +{ + print_version(); + puts(ORACLE_WELCOME_COPYRIGHT_NOTICE("2000")); + printf("InnoDB offline file checksum utility.\n"); + printf("Usage: %s [-c] [-s ] [-e ] [-p ] [-v] [-d] \n", my_progname); + my_print_help(innochecksum_options); + my_print_variables(innochecksum_options); +} + +extern "C" my_bool +innochecksum_get_one_option( +/*========================*/ + int optid, + const struct my_option *opt __attribute__((unused)), + char *argument __attribute__((unused))) +{ + switch (optid) { + case 'd': + verbose=1; /* debug implies verbose... */ + break; + case 'e': + use_end_page= 1; + break; + case 'p': + end_page= start_page= do_page; + use_end_page= 1; + do_one_page= 1; + break; + case 'V': + print_version(); + exit(0); + break; + case 'I': + case '?': + usage(); + exit(0); + break; + } + return 0; +} + +static int get_options( +/*===================*/ + int *argc, + char ***argv) +{ + int ho_error; + + if ((ho_error=handle_options(argc, argv, innochecksum_options, innochecksum_get_one_option))) + exit(ho_error); + + /* The next arg must be the filename */ + if (!*argc) + { + usage(); + return 1; + } + return 0; +} /* get_options */ + +int main(int argc, char **argv) +{ + FILE* f; /* our input file */ + char* filename; /* our input filename. */ + unsigned char *big_buf, *buf; + + ulong bytes; /* bytes read count */ + ulint ct; /* current page number (0 based) */ + time_t now; /* current time */ + time_t lastt; /* last time */ + ulint oldcsum, oldcsumfield, csum, csumfield, crc32, logseq, logseqfield; + /* ulints for checksum storage */ + struct stat st; /* for stat, if you couldn't guess */ + unsigned long long int size; /* size of file (has to be 64 bits) */ + ulint pages; /* number of pages in file */ + off_t offset= 0; + int fd; + + printf("InnoDB offline file checksum utility.\n"); + + ut_crc32_init(); + + MY_INIT(argv[0]); + + if (get_options(&argc,&argv)) + exit(1); + + if (verbose) + my_print_variables(innochecksum_options); + + /* The file name is not optional */ + filename = *argv; + if (*filename == '\0') + { + fprintf(stderr, "Error; File name missing\n"); + return 1; + } + + /* stat the file to get size and page count */ + if (stat(filename, &st)) + { + fprintf(stderr, "Error; %s cannot be found\n", filename); + return 1; + } + size= st.st_size; + + /* Open the file for reading */ + f= fopen(filename, "rb"); + if (f == NULL) + { + fprintf(stderr, "Error; %s cannot be opened", filename); + perror(" "); + return 1; + } + + big_buf = (unsigned char *)malloc(2 * UNIV_PAGE_SIZE_MAX); + if (big_buf == NULL) + { + fprintf(stderr, "Error; failed to allocate memory\n"); + perror(""); + return 1; + } + + /* Make sure the page is aligned */ + buf = (unsigned char*)ut_align_down(big_buf + + UNIV_PAGE_SIZE_MAX, UNIV_PAGE_SIZE_MAX); + + if (!get_page_size(f, buf, &logical_page_size, &physical_page_size)) + { + free(big_buf); + return 1; + } + + if (compressed) + { + printf("Table is compressed\n"); + printf("Key block size is %lu\n", physical_page_size); + } + else + { + printf("Table is uncompressed\n"); + printf("Page size is %lu\n", physical_page_size); + } + + pages= (ulint) (size / physical_page_size); + + if (just_count) + { + if (verbose) + printf("Number of pages: "); + printf("%lu\n", pages); + free(big_buf); + return 0; + } + else if (verbose) + { + printf("file %s = %llu bytes (%lu pages)...\n", filename, size, pages); + if (do_one_page) + printf("InnoChecksum; checking page %lu\n", do_page); + else + printf("InnoChecksum; checking pages in range %lu to %lu\n", start_page, use_end_page ? end_page : (pages - 1)); + } + +#ifdef UNIV_LINUX + if (posix_fadvise(fileno(f), 0, 0, POSIX_FADV_SEQUENTIAL) || + posix_fadvise(fileno(f), 0, 0, POSIX_FADV_NOREUSE)) + { + perror("posix_fadvise failed"); + } +#endif + + /* seek to the necessary position */ + if (start_page) + { + fd= fileno(f); + if (!fd) + { + perror("Error; Unable to obtain file descriptor number"); + free(big_buf); + return 1; + } + + offset= (off_t)start_page * (off_t)physical_page_size; + + if (lseek(fd, offset, SEEK_SET) != offset) + { + perror("Error; Unable to seek to necessary offset"); + free(big_buf); + return 1; + } + } + + /* main checksumming loop */ + ct= start_page; + lastt= 0; + while (!feof(f)) + { + bytes= fread(buf, 1, physical_page_size, f); + if (!bytes && feof(f)) + { + free(big_buf); + return 0; + } + + if (ferror(f)) + { + fprintf(stderr, "Error reading %lu bytes", physical_page_size); + perror(" "); + free(big_buf); + return 1; + } + + if (compressed) { + /* compressed pages */ + if (!page_zip_verify_checksum(buf, physical_page_size)) { + fprintf(stderr, "Fail; page %lu invalid (fails compressed page checksum).\n", ct); + if (!skip_corrupt) + { + free(big_buf); + return 1; + } + } + } else { + + /* check the "stored log sequence numbers" */ + logseq= mach_read_from_4(buf + FIL_PAGE_LSN + 4); + logseqfield= mach_read_from_4(buf + logical_page_size - FIL_PAGE_END_LSN_OLD_CHKSUM + 4); + if (debug) + printf("page %lu: log sequence number: first = %lu; second = %lu\n", ct, logseq, logseqfield); + if (logseq != logseqfield) + { + fprintf(stderr, "Fail; page %lu invalid (fails log sequence number check)\n", ct); + if (!skip_corrupt) + { + free(big_buf); + return 1; + } + } + + /* check old method of checksumming */ + oldcsum= buf_calc_page_old_checksum(buf); + oldcsumfield= mach_read_from_4(buf + logical_page_size - FIL_PAGE_END_LSN_OLD_CHKSUM); + if (debug) + printf("page %lu: old style: calculated = %lu; recorded = %lu\n", ct, oldcsum, oldcsumfield); + if (oldcsumfield != mach_read_from_4(buf + FIL_PAGE_LSN) && oldcsumfield != oldcsum) + { + fprintf(stderr, "Fail; page %lu invalid (fails old style checksum)\n", ct); + if (!skip_corrupt) + { + free(big_buf); + return 1; + } + } + + /* now check the new method */ + csum= buf_calc_page_new_checksum(buf); + crc32= buf_calc_page_crc32(buf); + csumfield= mach_read_from_4(buf + FIL_PAGE_SPACE_OR_CHKSUM); + if (debug) + printf("page %lu: new style: calculated = %lu; crc32 = %lu; recorded = %lu\n", + ct, csum, crc32, csumfield); + if (csumfield != 0 && crc32 != csumfield && csum != csumfield) + { + fprintf(stderr, "Fail; page %lu invalid (fails innodb and crc32 checksum)\n", ct); + if (!skip_corrupt) + { + free(big_buf); + return 1; + } + } + } + /* end if this was the last page we were supposed to check */ + if (use_end_page && (ct >= end_page)) + { + free(big_buf); + return 0; + } + + /* do counter increase and progress printing */ + ct++; + if (verbose) + { + if (ct % 64 == 0) + { + now= time(0); + if (!lastt) lastt= now; + if (now - lastt >= 1) + { + printf("page %lu okay: %.3f%% done\n", (ct - 1), (float) ct / pages * 100); + lastt= now; + } + } + } + } + free(big_buf); + return 0; +} diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl index d60ff9000a4..a1cacb2099b 100755 --- a/mysql-test/mysql-test-run.pl +++ b/mysql-test/mysql-test-run.pl @@ -2491,6 +2491,17 @@ sub environment_setup { "$bindir/sql$opt_vs_config/mysql_tzinfo_to_sql"); $ENV{'MYSQL_TZINFO_TO_SQL'}= native_path($exe_mysql_tzinfo_to_sql); + # ---------------------------------------------------- + # innochecksum + # ---------------------------------------------------- + my $exe_innochecksum= + mtr_exe_maybe_exists("$bindir/extra$opt_vs_config/innochecksum", + "$path_client_bindir/innochecksum"); + if ($exe_innochecksum) + { + $ENV{'INNOCHECKSUM'}= native_path($exe_innochecksum); + } + # Create an environment variable to make it possible # to detect that valgrind is being used from test cases $ENV{'VALGRIND_TEST'}= $opt_valgrind; diff --git a/mysql-test/suite/innodb/r/innochecksum.result b/mysql-test/suite/innodb/r/innochecksum.result new file mode 100644 index 00000000000..c75e83e5ed7 --- /dev/null +++ b/mysql-test/suite/innodb/r/innochecksum.result @@ -0,0 +1,31 @@ +# Create and populate a table +CREATE TABLE t1 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB; +INSERT INTO t1 (b) VALUES ('corrupt me'); +INSERT INTO t1 (b) VALUES ('corrupt me'); +CREATE TABLE t2 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) +ROW_FORMAT=COMPRESSED ENGINE=InnoDB ; +INSERT INTO t2(b) SELECT b from t1; +CREATE TABLE t3 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) +ROW_FORMAT=COMPRESSED ENGINE=InnoDB KEY_BLOCK_SIZE=16; +INSERT INTO t3(b) SELECT b from t1; +# Write file to make mysql-test-run.pl expect the "crash", but don't +# start it until it's told to +# We give 30 seconds to do a clean shutdown because we do not want +# to redo apply the pages of t1.ibd at the time of recovery. +# We want SQL to initiate the first access to t1.ibd. +# Wait until disconnected. +# Run innochecksum on t1 +InnoDB offline file checksum utility. +Table is uncompressed +Page size is 16384 +# Run innochecksum on t2 +InnoDB offline file checksum utility. +Table is compressed +Key block size is 8192 +# Run innochecksum on t3 +InnoDB offline file checksum utility. +Table is compressed +Key block size is 16384 +# Write file to make mysql-test-run.pl start up the server again +# Cleanup +DROP TABLE t1, t2, t3; diff --git a/mysql-test/suite/innodb/t/innochecksum.opt b/mysql-test/suite/innodb/t/innochecksum.opt new file mode 100644 index 00000000000..cc738d97434 --- /dev/null +++ b/mysql-test/suite/innodb/t/innochecksum.opt @@ -0,0 +1,2 @@ +--innodb_file_per_table=1 +--innodb_file_format=Barracuda diff --git a/mysql-test/suite/innodb/t/innochecksum.test b/mysql-test/suite/innodb/t/innochecksum.test new file mode 100644 index 00000000000..34df2801880 --- /dev/null +++ b/mysql-test/suite/innodb/t/innochecksum.test @@ -0,0 +1,70 @@ +# +# Test innochecksum +# + +# Don't test under embedded +source include/not_embedded.inc; +# Require InnoDB +source include/have_innodb.inc; + +if (!$INNOCHECKSUM) { + --echo Need innochecksum binary + --die Need innochecksum binary +} + +--echo # Create and populate a table +CREATE TABLE t1 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB; +INSERT INTO t1 (b) VALUES ('corrupt me'); +--disable_query_log +--let $i = 1000 +while ($i) +{ + INSERT INTO t1 (b) VALUES (REPEAT('abcdefghijklmnopqrstuvwxyz', 100)); + dec $i; +} +--enable_query_log +INSERT INTO t1 (b) VALUES ('corrupt me'); + +CREATE TABLE t2 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) +ROW_FORMAT=COMPRESSED ENGINE=InnoDB ; + +INSERT INTO t2(b) SELECT b from t1; + +CREATE TABLE t3 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) +ROW_FORMAT=COMPRESSED ENGINE=InnoDB KEY_BLOCK_SIZE=16; + +INSERT INTO t3(b) SELECT b from t1; + +let $MYSQLD_DATADIR=`select @@datadir`; +let t1_IBD = $MYSQLD_DATADIR/test/t1.ibd; +let t2_IBD = $MYSQLD_DATADIR/test/t2.ibd; +let t3_IBD = $MYSQLD_DATADIR/test/t3.ibd; + +--echo # Write file to make mysql-test-run.pl expect the "crash", but don't +--echo # start it until it's told to +--exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect + +--echo # We give 30 seconds to do a clean shutdown because we do not want +--echo # to redo apply the pages of t1.ibd at the time of recovery. +--echo # We want SQL to initiate the first access to t1.ibd. +shutdown_server 30; + +--echo # Wait until disconnected. +--source include/wait_until_disconnected.inc + +--echo # Run innochecksum on t1 +--exec $INNOCHECKSUM $t1_IBD + +--echo # Run innochecksum on t2 +--exec $INNOCHECKSUM $t2_IBD + +--echo # Run innochecksum on t3 +--exec $INNOCHECKSUM $t3_IBD + +--echo # Write file to make mysql-test-run.pl start up the server again +--exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--enable_reconnect +--source include/wait_until_connected_again.inc + +--echo # Cleanup +DROP TABLE t1, t2, t3; diff --git a/storage/innobase/buf/buf0checksum.cc b/storage/innobase/buf/buf0checksum.cc index ec79bbe6be9..4ba65d6f2d0 100644 --- a/storage/innobase/buf/buf0checksum.cc +++ b/storage/innobase/buf/buf0checksum.cc @@ -27,20 +27,21 @@ Created Aug 11, 2011 Vasil Dimov #include "fil0fil.h" /* FIL_* */ #include "ut0crc32.h" /* ut_crc32() */ #include "ut0rnd.h" /* ut_fold_binary() */ +#include "buf0checksum.h" #ifndef UNIV_INNOCHECKSUM #include "srv0srv.h" /* SRV_CHECKSUM_* */ #include "buf0types.h" +#endif /* !UNIV_INNOCHECKSUM */ + /** the macro MYSQL_SYSVAR_ENUM() requires "long unsigned int" and if we use srv_checksum_algorithm_t here then we get a compiler error: ha_innodb.cc:12251: error: cannot convert 'srv_checksum_algorithm_t*' to 'long unsigned int*' in initialization */ UNIV_INTERN ulong srv_checksum_algorithm = SRV_CHECKSUM_ALGORITHM_INNODB; -#endif /* !UNIV_INNOCHECKSUM */ - /********************************************************************//** Calculates a page CRC32 which is stored to the page when it is written to a file. Note that we must be careful to calculate the same value on diff --git a/storage/innobase/include/buf0buf.h b/storage/innobase/include/buf0buf.h index 31ec6b9ef8b..3097015999c 100644 --- a/storage/innobase/include/buf0buf.h +++ b/storage/innobase/include/buf0buf.h @@ -96,9 +96,6 @@ extern buf_block_t* back_block1; /*!< first block, for --apply-log */ extern buf_block_t* back_block2; /*!< second block, for page reorganize */ #endif /* !UNIV_HOTBACKUP */ -/** Magic value to use instead of checksums when they are disabled */ -#define BUF_NO_CHECKSUM_MAGIC 0xDEADBEEFUL - /** @brief States of a control block @see buf_page_t diff --git a/storage/innobase/include/buf0checksum.h b/storage/innobase/include/buf0checksum.h index cd21781dc6e..6818345f965 100644 --- a/storage/innobase/include/buf0checksum.h +++ b/storage/innobase/include/buf0checksum.h @@ -28,11 +28,10 @@ Created Aug 11, 2011 Vasil Dimov #include "univ.i" -#ifndef UNIV_INNOCHECKSUM - #include "buf0types.h" -#endif /* !UNIV_INNOCHECKSUM */ +/** Magic value to use instead of checksums when they are disabled */ +#define BUF_NO_CHECKSUM_MAGIC 0xDEADBEEFUL /********************************************************************//** Calculates a page CRC32 which is stored to the page when it is written @@ -70,8 +69,6 @@ buf_calc_page_old_checksum( /*=======================*/ const byte* page); /*!< in: buffer page */ -#ifndef UNIV_INNOCHECKSUM - /********************************************************************//** Return a printable string describing the checksum algorithm. @return algorithm name */ @@ -83,6 +80,4 @@ buf_checksum_algorithm_name( extern ulong srv_checksum_algorithm; -#endif /* !UNIV_INNOCHECKSUM */ - #endif /* buf0checksum_h */ diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index 798423eeddd..55559bdd999 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -65,11 +65,16 @@ of the address is FIL_NULL, the address is considered undefined. */ typedef byte fil_faddr_t; /*!< 'type' definition in C: an address stored in a file page is a string of bytes */ + +#endif /* !UNIV_INNOCHECKSUM */ + #define FIL_ADDR_PAGE 0 /* first in address is the page offset */ #define FIL_ADDR_BYTE 4 /* then comes 2-byte byte offset within page*/ #define FIL_ADDR_SIZE 6 /* address size is 6 bytes */ +#ifndef UNIV_INNOCHECKSUM + /** File space address */ struct fil_addr_t{ ulint page; /*!< page number within a space */ @@ -140,8 +145,6 @@ extern fil_addr_t fil_addr_null; #define FIL_PAGE_DATA_END 8 /*!< size of the page trailer */ /* @} */ -#ifndef UNIV_INNOCHECKSUM - /** File page types (values of FIL_PAGE_TYPE) @{ */ #define FIL_PAGE_INDEX 17855 /*!< B-tree node */ #define FIL_PAGE_UNDO_LOG 2 /*!< Undo log page */ @@ -166,6 +169,8 @@ extern fil_addr_t fil_addr_null; #define FIL_LOG 502 /*!< redo log */ /* @} */ +#ifndef UNIV_INNOCHECKSUM + /** The number of fsyncs done to the log */ extern ulint fil_n_log_flushes; diff --git a/storage/innobase/include/mach0data.ic b/storage/innobase/include/mach0data.ic index 7449d2da2b8..c46fcec107e 100644 --- a/storage/innobase/include/mach0data.ic +++ b/storage/innobase/include/mach0data.ic @@ -73,6 +73,8 @@ mach_write_to_2( b[1] = (byte)(n); } +#endif /* !UNIV_INNOCHECKSUM */ + /********************************************************//** The following function is used to fetch data from 2 consecutive bytes. The most significant byte is at the lowest address. @@ -86,6 +88,8 @@ mach_read_from_2( return(((ulint)(b[0]) << 8) | (ulint)(b[1])); } +#ifndef UNIV_INNOCHECKSUM + /********************************************************//** The following function is used to convert a 16-bit data item to the canonical format, for fast bytewise equality test @@ -295,6 +299,8 @@ mach_write_to_8( mach_write_to_4(static_cast(b) + 4, (ulint) n); } +#endif /* !UNIV_INNOCHECKSUM */ + /********************************************************//** The following function is used to fetch data from 8 consecutive bytes. The most significant byte is at the lowest address. @@ -313,6 +319,8 @@ mach_read_from_8( return(ull); } +#ifndef UNIV_INNOCHECKSUM + /*******************************************************//** The following function is used to store data in 7 consecutive bytes. We store the most significant byte to the lowest address. */ diff --git a/storage/innobase/include/page0page.h b/storage/innobase/include/page0page.h index b572f7abb49..cb6633bb941 100644 --- a/storage/innobase/include/page0page.h +++ b/storage/innobase/include/page0page.h @@ -29,6 +29,7 @@ Created 2/2/1994 Heikki Tuuri #include "univ.i" #include "page0types.h" +#ifndef UNIV_INNOCHECKSUM #include "fil0fil.h" #include "buf0buf.h" #include "data0data.h" @@ -42,6 +43,8 @@ Created 2/2/1994 Heikki Tuuri #define UNIV_INLINE #endif +#endif /* !UNIV_INNOCHECKSUM */ + /* PAGE HEADER =========== @@ -117,6 +120,8 @@ typedef byte page_header_t; a new-style compact page */ /*-----------------------------*/ +#ifndef UNIV_INNOCHECKSUM + /* Heap numbers */ #define PAGE_HEAP_NO_INFIMUM 0 /* page infimum */ #define PAGE_HEAP_NO_SUPREMUM 1 /* page supremum */ @@ -343,6 +348,7 @@ page_cmp_dtuple_rec_with_match( matched; when function returns contains the value for current comparison */ #endif /* !UNIV_HOTBACKUP */ +#endif /* !UNIV_INNOCHECKSUM */ /*************************************************************//** Gets the page number. @return page number */ @@ -351,6 +357,7 @@ ulint page_get_page_no( /*=============*/ const page_t* page); /*!< in: page */ +#ifndef UNIV_INNOCHECKSUM /*************************************************************//** Gets the tablespace identifier. @return space id */ @@ -359,6 +366,7 @@ ulint page_get_space_id( /*==============*/ const page_t* page); /*!< in: page */ +#endif /* !UNIV_INNOCHECKSUM */ /*************************************************************//** Gets the number of user records on page (the infimum and supremum records are not user records). @@ -368,6 +376,7 @@ ulint page_get_n_recs( /*============*/ const page_t* page); /*!< in: index page */ +#ifndef UNIV_INNOCHECKSUM /***************************************************************//** Returns the number of records before the given record in chain. The number includes infimum and supremum records. @@ -516,6 +525,7 @@ ulint page_rec_get_heap_no( /*=================*/ const rec_t* rec); /*!< in: the physical record */ +#endif /* !UNIV_INNOCHECKSUM */ /************************************************************//** Determine whether the page is a B-tree leaf. @return true if the page is a B-tree leaf (PAGE_LEVEL = 0) */ @@ -525,6 +535,7 @@ page_is_leaf( /*=========*/ const page_t* page) /*!< in: page */ __attribute__((nonnull, pure)); +#ifndef UNIV_INNOCHECKSUM /************************************************************//** Determine whether the page is empty. @return true if the page is empty (PAGE_N_RECS = 0) */ @@ -1115,8 +1126,11 @@ page_find_rec_max_not_deleted( #define UNIV_INLINE UNIV_INLINE_ORIGINAL #endif +#endif /* !UNIV_INNOCHECKSUM */ + #ifndef UNIV_NONINL #include "page0page.ic" #endif + #endif diff --git a/storage/innobase/include/page0page.ic b/storage/innobase/include/page0page.ic index 9b81156708f..99e17001c0a 100644 --- a/storage/innobase/include/page0page.ic +++ b/storage/innobase/include/page0page.ic @@ -23,6 +23,8 @@ Index page routines Created 2/2/1994 Heikki Tuuri *******************************************************/ +#ifndef UNIV_INNOCHECKSUM + #include "mach0data.h" #ifdef UNIV_DEBUG # include "log0recv.h" @@ -38,6 +40,7 @@ Created 2/2/1994 Heikki Tuuri #define UNIV_INLINE #endif +#endif /* !UNIV_INNOCHECKSUM */ /************************************************************//** Gets the start of a page. @return start of the page */ @@ -49,6 +52,7 @@ page_align( { return((page_t*) ut_align_down(ptr, UNIV_PAGE_SIZE)); } +#ifndef UNIV_INNOCHECKSUM /************************************************************//** Gets the offset within a page. @return offset from the start of the page */ @@ -103,6 +107,7 @@ page_update_max_trx_id( } } +#endif /* !UNIV_INNOCHECKSUM */ /*************************************************************//** Reads the given header field. */ UNIV_INLINE @@ -118,6 +123,7 @@ page_header_get_field( return(mach_read_from_2(page + PAGE_HEADER + field)); } +#ifndef UNIV_INNOCHECKSUM /*************************************************************//** Sets the given header field. */ UNIV_INLINE @@ -223,6 +229,7 @@ page_header_reset_last_insert( } #endif /* !UNIV_HOTBACKUP */ +#endif /* !UNIV_INNOCHECKSUM */ /************************************************************//** Determine whether the page is in new-style compact format. @return nonzero if the page is in compact format, zero if it is in @@ -236,6 +243,7 @@ page_is_comp( return(page_header_get_field(page, PAGE_N_HEAP) & 0x8000); } +#ifndef UNIV_INNOCHECKSUM /************************************************************//** TRUE if the record is on a page in compact format. @return nonzero if in compact format */ @@ -264,6 +272,7 @@ page_rec_get_heap_no( } } +#endif /* !UNIV_INNOCHECKSUM */ /************************************************************//** Determine whether the page is a B-tree leaf. @return true if the page is a B-tree leaf (PAGE_LEVEL = 0) */ @@ -276,6 +285,7 @@ page_is_leaf( return(!*(const uint16*) (page + (PAGE_HEADER + PAGE_LEVEL))); } +#ifndef UNIV_INNOCHECKSUM /************************************************************//** Determine whether the page is empty. @return true if the page is empty (PAGE_N_RECS = 0) */ @@ -529,6 +539,7 @@ page_cmp_dtuple_rec_with_match( } #endif /* !UNIV_HOTBACKUP */ +#endif /* !UNIV_INNOCHECKSUM */ /*************************************************************//** Gets the page number. @return page number */ @@ -542,6 +553,7 @@ page_get_page_no( return(mach_read_from_4(page + FIL_PAGE_OFFSET)); } +#ifndef UNIV_INNOCHECKSUM /*************************************************************//** Gets the tablespace identifier. @return space id */ @@ -555,6 +567,7 @@ page_get_space_id( return(mach_read_from_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID)); } +#endif /* !UNIV_INNOCHECKSUM */ /*************************************************************//** Gets the number of user records on page (infimum and supremum records are not user records). @@ -568,6 +581,7 @@ page_get_n_recs( return(page_header_get_field(page, PAGE_N_RECS)); } +#ifndef UNIV_INNOCHECKSUM /*************************************************************//** Gets the number of dir slots in directory. @return number of slots */ @@ -958,6 +972,7 @@ page_rec_get_base_extra_size( return(REC_N_NEW_EXTRA_BYTES + (ulint) !page_rec_is_comp(rec)); } +#endif /* !UNIV_INNOCHECKSUM */ /************************************************************//** Returns the sum of the sizes of the records in the record list, excluding the infimum and supremum records. @@ -981,7 +996,7 @@ page_get_data_size( return(ret); } - +#ifndef UNIV_INNOCHECKSUM /************************************************************//** Allocates a block of memory from the free list of an index page. */ UNIV_INLINE @@ -1170,6 +1185,8 @@ page_mem_free( } } +#endif /* !UNIV_INNOCHECKSUM */ + #ifdef UNIV_MATERIALIZE #undef UNIV_INLINE #define UNIV_INLINE UNIV_INLINE_ORIGINAL diff --git a/storage/innobase/include/page0types.h b/storage/innobase/include/page0types.h index 95143a4bb44..fb9250a5a3b 100644 --- a/storage/innobase/include/page0types.h +++ b/storage/innobase/include/page0types.h @@ -33,6 +33,8 @@ using namespace std; #include "univ.i" #include "dict0types.h" #include "mtr0types.h" +#include "sync0types.h" +#include "os0thread.h" /** Eliminates a name collision on HP-UX */ #define page_t ib_page_t diff --git a/storage/innobase/include/page0zip.h b/storage/innobase/include/page0zip.h index 9d3b78ed2fc..6fe6934e35c 100644 --- a/storage/innobase/include/page0zip.h +++ b/storage/innobase/include/page0zip.h @@ -1,3 +1,4 @@ + /***************************************************************************** Copyright (c) 2005, 2013, Oracle and/or its affiliates. All Rights Reserved. @@ -32,13 +33,15 @@ Created June 2005 by Marko Makela # define UNIV_INLINE #endif -#include "mtr0types.h" #include "page0types.h" #include "buf0types.h" +#ifndef UNIV_INNOCHECKSUM +#include "mtr0types.h" #include "dict0types.h" #include "srv0srv.h" #include "trx0types.h" #include "mem0mem.h" +#endif /* !UNIV_INNOCHECKSUM */ /* Compression level to be used by zlib. Settable by user. */ extern uint page_zip_level; @@ -50,6 +53,7 @@ extern uint page_zip_level; compression algorithm changes in zlib. */ extern my_bool page_zip_log_pages; +#ifndef UNIV_INNOCHECKSUM /**********************************************************************//** Determine the size of a compressed page in bytes. @return size in bytes */ @@ -112,6 +116,7 @@ page_zip_set_alloc( /*===============*/ void* stream, /*!< in/out: zlib stream */ mem_heap_t* heap); /*!< in: memory heap to use */ +#endif /* !UNIV_INNOCHECKSUM */ /**********************************************************************//** Compress a page. @@ -147,6 +152,7 @@ page_zip_decompress( after page creation */ __attribute__((nonnull(1,2))); +#ifndef UNIV_INNOCHECKSUM #ifdef UNIV_DEBUG /**********************************************************************//** Validate a compressed page descriptor. @@ -158,6 +164,7 @@ page_zip_simple_validate( const page_zip_des_t* page_zip); /*!< in: compressed page descriptor */ #endif /* UNIV_DEBUG */ +#endif /* !UNIV_INNOCHECKSUM */ #ifdef UNIV_ZIP_DEBUG /**********************************************************************//** @@ -185,6 +192,7 @@ page_zip_validate( __attribute__((nonnull(1,2))); #endif /* UNIV_ZIP_DEBUG */ +#ifndef UNIV_INNOCHECKSUM /**********************************************************************//** Determine how big record can be inserted without recompressing the page. @return a positive number indicating the maximum size of a record @@ -418,6 +426,8 @@ page_zip_reorganize( dict_index_t* index, /*!< in: index of the B-tree node */ mtr_t* mtr) /*!< in: mini-transaction */ __attribute__((nonnull)); +#endif /* !UNIV_INNOCHECKSUM */ + #ifndef UNIV_HOTBACKUP /**********************************************************************//** Copy the records of a page byte for byte. Do not copy the page header @@ -474,6 +484,8 @@ page_zip_verify_checksum( /*=====================*/ const void* data, /*!< in: compressed page */ ulint size); /*!< in: size of compressed page */ + +#ifndef UNIV_INNOCHECKSUM /**********************************************************************//** Write a log record of compressing an index page without the data on the page. */ UNIV_INLINE @@ -506,6 +518,8 @@ void page_zip_reset_stat_per_index(); /*===========================*/ +#endif /* !UNIV_INNOCHECKSUM */ + #ifndef UNIV_HOTBACKUP /** Check if a pointer to an uncompressed page matches a compressed page. When we IMPORT a tablespace the blocks and accompanying frames are allocted @@ -531,8 +545,10 @@ from outside the buffer pool. # define UNIV_INLINE UNIV_INLINE_ORIGINAL #endif +#ifndef UNIV_INNOCHECKSUM #ifndef UNIV_NONINL # include "page0zip.ic" #endif +#endif /* !UNIV_INNOCHECKSUM */ #endif /* page0zip_h */ diff --git a/storage/innobase/include/rem0rec.h b/storage/innobase/include/rem0rec.h index 8e7d5ff2d48..cfd0f10642a 100644 --- a/storage/innobase/include/rem0rec.h +++ b/storage/innobase/include/rem0rec.h @@ -26,11 +26,13 @@ Created 5/30/1994 Heikki Tuuri #ifndef rem0rec_h #define rem0rec_h +#ifndef UNIV_INNOCHECKSUM #include "univ.i" #include "data0data.h" #include "rem0types.h" #include "mtr0types.h" #include "page0types.h" +#endif /* !UNIV_INNOCHECKSUM */ /* Info bit denoting the predefined minimum record: this bit is set if and only if the record is the first user record on a non-leaf @@ -88,6 +90,7 @@ offsets[] array, first passed to rec_get_offsets() */ #define REC_OFFS_NORMAL_SIZE 100 #define REC_OFFS_SMALL_SIZE 10 +#ifndef UNIV_INNOCHECKSUM /******************************************************//** The following function is used to get the pointer of the next chained record on the same page. @@ -985,4 +988,5 @@ two upmost bits in a two byte offset for special purposes */ #include "rem0rec.ic" #endif +#endif /* !UNIV_INNOCHECKSUM */ #endif diff --git a/storage/innobase/include/srv0srv.h b/storage/innobase/include/srv0srv.h index a3e6a17a6e2..d06a14a9153 100644 --- a/storage/innobase/include/srv0srv.h +++ b/storage/innobase/include/srv0srv.h @@ -361,7 +361,6 @@ extern my_bool srv_stats_sample_traditional; extern ibool srv_use_doublewrite_buf; extern ulong srv_doublewrite_batch_size; -extern ulong srv_checksum_algorithm; extern ibool srv_use_atomic_writes; #ifdef HAVE_POSIX_FALLOCATE diff --git a/storage/innobase/include/trx0undo.h b/storage/innobase/include/trx0undo.h index 660551961a6..45733921212 100644 --- a/storage/innobase/include/trx0undo.h +++ b/storage/innobase/include/trx0undo.h @@ -26,6 +26,8 @@ Created 3/26/1996 Heikki Tuuri #ifndef trx0undo_h #define trx0undo_h +#ifndef UNIV_INNOCHECKSUM + #include "univ.i" #include "trx0types.h" #include "mtr0mtr.h" @@ -385,6 +387,8 @@ trx_undo_mem_free( /*==============*/ trx_undo_t* undo); /* in: the undo object to be freed */ +#endif /* !UNIV_INNOCHECKSUM */ + /* Types of an undo log segment */ #define TRX_UNDO_INSERT 1 /* contains undo entries for inserts */ #define TRX_UNDO_UPDATE 2 /* contains undo entries for updates @@ -403,6 +407,7 @@ trx_undo_mem_free( prepared transaction */ #ifndef UNIV_HOTBACKUP +#ifndef UNIV_INNOCHECKSUM /** Transaction undo log memory object; this is protected by the undo_mutex in the corresponding transaction object */ @@ -461,6 +466,7 @@ struct trx_undo_t{ /*!< undo log objects in the rollback segment are chained into lists */ }; +#endif /* !UNIV_INNOCHECKSUM */ #endif /* !UNIV_HOTBACKUP */ /** The offset of the undo log page header on pages of the undo log */ @@ -588,8 +594,10 @@ quite a large overhead. */ with the XA XID */ /* @} */ +#ifndef UNIV_INNOCHECKSUM #ifndef UNIV_NONINL #include "trx0undo.ic" #endif +#endif /* !UNIV_INNOCHECKSUM */ #endif diff --git a/storage/innobase/include/univ.i b/storage/innobase/include/univ.i index eeeaca166a8..cb2e6613171 100644 --- a/storage/innobase/include/univ.i +++ b/storage/innobase/include/univ.i @@ -105,7 +105,7 @@ if we are compiling on Windows. */ /* Include the header file generated by GNU autoconf */ # ifndef __WIN__ # ifndef UNIV_HOTBACKUP -# include "config.h" +# include "my_config.h" # endif /* UNIV_HOTBACKUP */ # endif diff --git a/storage/innobase/include/ut0ut.h b/storage/innobase/include/ut0ut.h index 7d1c3cd4f0b..939ccee6e3e 100644 --- a/storage/innobase/include/ut0ut.h +++ b/storage/innobase/include/ut0ut.h @@ -36,6 +36,8 @@ Created 1/20/1994 Heikki Tuuri # include "os0sync.h" /* for HAVE_ATOMIC_BUILTINS */ #endif /* UNIV_HOTBACKUP */ +#endif /* !UNIV_INNOCHECKSUM */ + #include #ifndef MYSQL_SERVER #include @@ -64,6 +66,7 @@ private: F& f; }; +#ifndef UNIV_INNOCHECKSUM #ifndef UNIV_HOTBACKUP # if defined(HAVE_PAUSE_INSTRUCTION) /* According to the gcc info page, asm volatile means that the @@ -162,6 +165,7 @@ ut_pair_cmp( ulint a2, /*!< in: less significant part of first pair */ ulint b1, /*!< in: more significant part of second pair */ ulint b2); /*!< in: less significant part of second pair */ +#endif /* !UNIV_INNOCHECKSUM */ /*************************************************************//** Determines if a number is zero or a power of two. @param n in: number @@ -192,6 +196,7 @@ when m is a power of two. In other words, rounds n up to m * k. @param m in: alignment, must be a power of two @return n rounded up to the smallest possible integer multiple of m */ #define ut_calc_align(n, m) (((n) + ((m) - 1)) & ~((m) - 1)) +#ifndef UNIV_INNOCHECKSUM /*************************************************************//** Calculates fast the 2-logarithm of a number, rounded upward to an integer. diff --git a/storage/innobase/page/page0zip.cc b/storage/innobase/page/page0zip.cc index 6989953cb0c..85ee661a746 100644 --- a/storage/innobase/page/page0zip.cc +++ b/storage/innobase/page/page0zip.cc @@ -36,6 +36,10 @@ using namespace std; # include "page0zip.ic" #endif #undef THIS_MODULE +#include "fil0fil.h" +#include "buf0checksum.h" +#include "mach0data.h" +#ifndef UNIV_INNOCHECKSUM #include "page0page.h" #include "mtr0log.h" #include "ut0sort.h" @@ -43,15 +47,18 @@ using namespace std; #include "btr0cur.h" #include "page0types.h" #include "log0recv.h" +#endif /* !UNIV_INNOCHECKSUM */ #include "zlib.h" #ifndef UNIV_HOTBACKUP +#ifndef UNIV_INNOCHECKSUM # include "buf0buf.h" -# include "buf0lru.h" # include "btr0sea.h" # include "dict0boot.h" # include "lock0lock.h" -# include "srv0mon.h" # include "srv0srv.h" +#endif /* !UNIV_INNOCHECKSUM */ +# include "buf0lru.h" +# include "srv0mon.h" # include "ut0crc32.h" #else /* !UNIV_HOTBACKUP */ # include "buf0checksum.h" @@ -60,6 +67,7 @@ using namespace std; #endif /* !UNIV_HOTBACKUP */ #ifndef UNIV_HOTBACKUP +#ifndef UNIV_INNOCHECKSUM /** Statistics on compression, indexed by page_zip_des_t::ssize - 1 */ UNIV_INTERN page_zip_stat_t page_zip_stat[PAGE_ZIP_SSIZE_MAX]; /** Statistics on compression, indexed by index->id */ @@ -69,6 +77,7 @@ UNIV_INTERN ib_mutex_t page_zip_stat_per_index_mutex; #ifdef HAVE_PSI_INTERFACE UNIV_INTERN mysql_pfs_key_t page_zip_stat_per_index_mutex_key; #endif /* HAVE_PSI_INTERFACE */ +#endif /* !UNIV_INNOCHECKSUM */ #endif /* !UNIV_HOTBACKUP */ /* Compression level to be used by zlib. Settable by user. */ @@ -117,6 +126,7 @@ Compare at most sizeof(field_ref_zero) bytes. /* Enable some extra debugging output. This code can be enabled independently of any UNIV_ debugging conditions. */ +#ifndef UNIV_INNOCHECKSUM #if defined UNIV_DEBUG || defined UNIV_ZIP_DEBUG # include __attribute__((format (printf, 1, 2))) @@ -149,7 +159,9 @@ page_zip_fail_func( @param fmt_args ignored: printf(3) format string and arguments */ # define page_zip_fail(fmt_args) /* empty */ #endif /* UNIV_DEBUG || UNIV_ZIP_DEBUG */ +#endif /* !UNIV_INNOCHECKSUM */ +#ifndef UNIV_INNOCHECKSUM #ifndef UNIV_HOTBACKUP /**********************************************************************//** Determine the guaranteed free space on an empty page. @@ -4838,6 +4850,7 @@ corrupt: return(ptr + 8 + size + trailer_size); } +#endif /* !UNIV_INNOCHECKSUM */ /**********************************************************************//** Calculate the compressed page checksum. @@ -4913,6 +4926,10 @@ page_zip_verify_checksum( stored = static_cast(mach_read_from_4( static_cast(data) + FIL_PAGE_SPACE_OR_CHKSUM)); +#ifndef UNIV_INNOCHECKSUM + /* innochecksum doesn't compile with ut_d. Since we don't + need to check for empty pages when running innochecksum, + just don't include this code. */ /* declare empty pages non-corrupted */ if (stored == 0) { /* make sure that the page is really empty */ @@ -4925,6 +4942,7 @@ page_zip_verify_checksum( return(TRUE); } +#endif calc = static_cast(page_zip_calc_checksum( data, size, static_cast(