From 6a4fe9923da777cbefe706d7e91c52cdd7563d56 Mon Sep 17 00:00:00 2001 From: Monty Date: Sat, 12 Jul 2025 15:45:02 +0300 Subject: [PATCH] Improvements for myisamchk These changes was done as part of fixing MDEV-36858 MariaDB MyISAM secondary indexes silently break for tables > 10B rows Changes done in myisamchk: - Tables that are checked are opened in readonly mode if --force is not used. - *.MYD files will be opened in readonly mode for repair if --quick is used. - Added information about check progress if --verbose is used. - Output information about repaired/checked rows every 10000 rows instead of every 1000 rows. Note that this also affects aria_chk - Store open file mode in share->index_mode and share->data_mode instead of in share->mode. - Added new option --keys-active= as a simpler version of keys-used. - Changed output for "myisamchk -dvv" to get nicer output for tables with 10 billion rows. --- include/my_base.h | 7 ++ include/my_global.h | 2 +- include/myisamchk.h | 2 +- mysql-test/main/long_unique.result | 16 ++-- mysql-test/main/myisam.result | 8 +- .../suite/vcol/r/vcol_keys_myisam.result | 4 +- storage/myisam/mi_check.c | 18 ++++- storage/myisam/mi_close.c | 2 +- storage/myisam/mi_dynrec.c | 2 +- storage/myisam/mi_open.c | 38 ++++++--- storage/myisam/myisamchk.c | 79 +++++++++++++++---- storage/myisam/myisamdef.h | 3 +- 12 files changed, 135 insertions(+), 46 deletions(-) diff --git a/include/my_base.h b/include/my_base.h index 0e4dab9da5a..2da0d2ef651 100644 --- a/include/my_base.h +++ b/include/my_base.h @@ -50,6 +50,13 @@ #define HA_OPEN_FOR_CREATE 4096U #define HA_OPEN_FOR_DROP (1U << 13) /* Open part of drop */ #define HA_OPEN_GLOBAL_TMP_TABLE (1U << 14) /* TMP table used by repliction */ +/* + This is to signal that the table will not be cached by the caller + and the table should be open in read-only mode if the tool requests + that +*/ +#define HA_OPEN_FORCE_MODE (1U << 15) /* Force open mode */ +#define HA_OPEN_DATA_READONLY (1U << 16) /* Use readonly for data */ /* Allow opening even if table is incompatible as this is for ALTER TABLE which diff --git a/include/my_global.h b/include/my_global.h index 82b6f21f9a5..fe970ad1304 100644 --- a/include/my_global.h +++ b/include/my_global.h @@ -1027,7 +1027,7 @@ typedef ulong myf; /* Type of MyFlags in my_funcs */ #define YESNO(X) ((X) ? "yes" : "no") #define MY_HOW_OFTEN_TO_ALARM 2 /* How often we want info on screen */ -#define MY_HOW_OFTEN_TO_WRITE 10000 /* How often we want info on screen */ +#define MY_HOW_OFTEN_TO_WRITE 100000 /* How often we want info on screen */ #include diff --git a/include/myisamchk.h b/include/myisamchk.h index 85f7c0a56f7..00ac234b5f2 100644 --- a/include/myisamchk.h +++ b/include/myisamchk.h @@ -116,7 +116,7 @@ typedef struct st_handler_check_param int (*fix_record)(struct st_myisam_info *info, uchar *record, int keynum); mysql_mutex_t print_msg_mutex; - my_bool need_print_msg_lock; + my_bool need_print_msg_lock, status_reporting; myf malloc_flags; } HA_CHECK; diff --git a/mysql-test/main/long_unique.result b/mysql-test/main/long_unique.result index b1b537a29b0..03a8b14752a 100644 --- a/mysql-test/main/long_unique.result +++ b/mysql-test/main/long_unique.result @@ -42,8 +42,8 @@ Ignored NO MyISAM file: DATADIR/test/t1 Record format: Packed Character set: latin1_swedish_ci (8) -Data records: 10 Deleted blocks: 0 -Recordlength: 12 +Data records: 10 Deleted blocks: 0 +Recordlength: 12 table description: Key Start Len Index Type @@ -130,8 +130,8 @@ insert into t1 values(1),(2),(3),(4),(5),(8),(7); MyISAM file: DATADIR/test/t1 Record format: Packed Character set: latin1_swedish_ci (8) -Data records: 7 Deleted blocks: 0 -Recordlength: 12 +Data records: 7 Deleted blocks: 0 +Recordlength: 12 table description: Key Start Len Index Type @@ -365,8 +365,8 @@ t1 0 e 1 e A NULL NULL NULL YES HASH NO MyISAM file: DATADIR/test/t1 Record format: Packed Character set: latin1_swedish_ci (8) -Data records: 8 Deleted blocks: 0 -Recordlength: 3040 +Data records: 8 Deleted blocks: 0 +Recordlength: 3040 table description: Key Start Len Index Type @@ -722,8 +722,8 @@ t1 0 b 4 h A NULL NULL NULL YES HASH NO MyISAM file: DATADIR/test/t1 Record format: Packed Character set: latin1_swedish_ci (8) -Data records: 9 Deleted blocks: 0 -Recordlength: 5059 +Data records: 9 Deleted blocks: 0 +Recordlength: 5059 table description: Key Start Len Index Type diff --git a/mysql-test/main/myisam.result b/mysql-test/main/myisam.result index 7d930f1aa0d..a90a2b6d6f1 100644 --- a/mysql-test/main/myisam.result +++ b/mysql-test/main/myisam.result @@ -2402,8 +2402,8 @@ KEY (c2) MyISAM file: MYSQLD_DATADIR/test/t1 Record format: Packed Character set: utf8mb3_general_ci (33) -Data records: 0 Deleted blocks: 0 -Recordlength: 94 +Data records: 0 Deleted blocks: 0 +Recordlength: 94 table description: Key Start Len Index Type @@ -2690,8 +2690,8 @@ KEY (d) MyISAM file: MYSQLD_DATADIR/test/t1 Record format: Fixed length Character set: latin1_swedish_ci (8) -Data records: 0 Deleted blocks: 0 -Recordlength: 13 +Data records: 0 Deleted blocks: 0 +Recordlength: 13 table description: Key Start Len Index Type diff --git a/mysql-test/suite/vcol/r/vcol_keys_myisam.result b/mysql-test/suite/vcol/r/vcol_keys_myisam.result index eb6908041f1..c29bccfb6db 100644 --- a/mysql-test/suite/vcol/r/vcol_keys_myisam.result +++ b/mysql-test/suite/vcol/r/vcol_keys_myisam.result @@ -244,8 +244,8 @@ a b c MyISAM file: datadir/test/t1 Record format: Fixed length Character set: latin1_swedish_ci (8) -Data records: 1001 Deleted blocks: 0 -Recordlength: 9 +Data records: 1001 Deleted blocks: 0 +Recordlength: 9 table description: Key Start Len Index Type diff --git a/storage/myisam/mi_check.c b/storage/myisam/mi_check.c index dff5fcdbcb6..7c11903abda 100644 --- a/storage/myisam/mi_check.c +++ b/storage/myisam/mi_check.c @@ -469,6 +469,11 @@ int chk_key(HA_CHECK *param, register MI_INFO *info) if (chk_index(param,info,keyinfo,share->state.key_root[key],info->buff, &keys, param->key_crc+key,1)) DBUG_RETURN(-1); + if ((param->testflag & T_WRITE_LOOP) && param->verbose) + { + puts(" \r"); + fflush(stdout); + } if(!(keyinfo->flag & (HA_FULLTEXT | HA_SPATIAL))) { if (keys != info->state->records) @@ -565,7 +570,8 @@ do_stat: puts(""); } if (param->key_file_blocks != info->state->key_file_length && - param->keys_in_use != ~(ulonglong) 0) + mi_is_all_keys_active(share->state.key_map, share->base.keys) && + !full_text_keys) mi_check_print_warning(param, "Some data are unreferenced in keyfile"); if (found_keys != full_text_keys) param->record_checksum=old_record_checksum-init_checksum; /* Remove delete links */ @@ -870,6 +876,15 @@ static int chk_index(HA_CHECK *param, MI_INFO *info, MI_KEYDEF *keyinfo, goto err; } param->record_checksum+=(ha_checksum) record; + if ((param->testflag & T_WRITE_LOOP) && param->verbose && + (*keys % WRITE_COUNT) == 0) + { + char llbuff[22]; + ulonglong records= info->state->records; + printf("%15s (%3.4f%%)\r", llstr(*keys, llbuff), + ((double) *keys / (records > *keys ? records : *keys)) *100); + fflush(stdout); + } } if (keypos != endpos) { @@ -3878,6 +3893,7 @@ static int sort_key_write(MI_SORT_PARAM *sort_param, const void *a) { mi_check_print_error(param, "Internal error: Keys are not in order from sort"); + DBUG_ASSERT(0); return(1); } #endif diff --git a/storage/myisam/mi_close.c b/storage/myisam/mi_close.c index 56197729251..a5cd2284227 100644 --- a/storage/myisam/mi_close.c +++ b/storage/myisam/mi_close.c @@ -82,7 +82,7 @@ int mi_close(register MI_INFO *info) (the server might want to reopen it, and mi_lock_database() only writes the state for non-temp ones) */ - if (share->mode != O_RDONLY && + if (share->index_mode != O_RDONLY && (mi_is_crashed(info) || (share->temporary && !share->deleting))) mi_state_info_write(share->kfile, &share->state, 1); /* Decrement open count must be last I/O on this file. */ diff --git a/storage/myisam/mi_dynrec.c b/storage/myisam/mi_dynrec.c index 09c10040f9c..17d4e248842 100644 --- a/storage/myisam/mi_dynrec.c +++ b/storage/myisam/mi_dynrec.c @@ -83,7 +83,7 @@ my_bool mi_dynmap_file(MI_INFO *info, my_off_t size) */ info->s->file_map= (uchar*) my_mmap(0, (size_t) size, - info->s->mode==O_RDONLY ? PROT_READ : + info->s->index_mode == O_RDONLY ? PROT_READ : PROT_READ | PROT_WRITE, MAP_SHARED | MAP_NORESERVE, info->dfile, 0L); diff --git a/storage/myisam/mi_open.c b/storage/myisam/mi_open.c index 414e483acbc..9e3a8f43944 100644 --- a/storage/myisam/mi_open.c +++ b/storage/myisam/mi_open.c @@ -82,7 +82,8 @@ MI_INFO *test_if_reopen(char *filename) MI_INFO *mi_open(const char *name, int mode, uint open_flags) { - int lock_error,kfile,open_mode,save_errno,have_rtree=0, realpath_err; + int lock_error,kfile,save_errno,have_rtree=0, realpath_err; + int open_mode, try_open_mode; uint i,j,len,errpos,head_length,base_pos,offset,info_length,keys, key_parts,unique_key_parts,base_key_parts,fulltext_keys,uniques; uint internal_table= open_flags & HA_OPEN_INTERNAL_TABLE; @@ -138,18 +139,31 @@ MI_INFO *mi_open(const char *name, int mode, uint open_flags) }); DEBUG_SYNC_C("mi_open_kfile"); + + /* + We first try to open the file on read-write mode to ensure that + the table is usable for future read and write queries in + MariaDB. Only if the read-write mode fails we try to readonly. + */ + try_open_mode= (open_flags & HA_OPEN_FORCE_MODE) ? mode : O_RDWR; + if ((kfile= mysql_file_open(mi_key_file_kfile, name_buff, - (open_mode= O_RDWR) | O_SHARE | O_NOFOLLOW | O_CLOEXEC, + (open_mode= try_open_mode) | O_SHARE | + O_NOFOLLOW | O_CLOEXEC, MYF(MY_NOSYMLINKS))) < 0) { - if ((errno != EROFS && errno != EACCES) || + if ((errno != EROFS && errno != EACCES) || open_mode == O_RDONLY || mode != O_RDONLY || (kfile= mysql_file_open(mi_key_file_kfile, name_buff, - (open_mode= O_RDONLY) | O_SHARE| O_NOFOLLOW | O_CLOEXEC, + (open_mode= O_RDONLY) | O_SHARE | + O_NOFOLLOW | O_CLOEXEC, MYF(MY_NOSYMLINKS))) < 0) goto err; } - share->mode=open_mode; + share->index_mode= share->data_mode= open_mode; + if (open_flags & HA_OPEN_DATA_READONLY) + share->data_mode= O_RDONLY; + errpos=1; if (mysql_file_read(kfile, (uchar*)&share->state.header, head_length, MYF(MY_NABP))) @@ -200,7 +214,9 @@ MI_INFO *mi_open(const char *name, int mode, uint open_flags) my_errno= HA_WRONG_CREATE_OPTION; goto err; } - share->mode|= O_NOFOLLOW; /* all symlinks are resolved by realpath() */ + /* all symlinks are resolved by realpath() */ + share->index_mode|= O_NOFOLLOW; + share->data_mode|= O_NOFOLLOW; } info_length=mi_uint2korr(share->state.header.header_length); @@ -588,7 +604,7 @@ MI_INFO *mi_open(const char *name, int mode, uint open_flags) else { share= old_info->s; - if (mode == O_RDWR && share->mode == O_RDONLY) + if (mode == O_RDWR && share->index_mode == O_RDONLY) { my_errno=EACCES; /* Can't open in write mode */ goto err; @@ -1279,10 +1295,11 @@ active seek-positions. int mi_open_datafile(MI_INFO *info, MYISAM_SHARE *share) { - myf flags= MY_WME | (share->mode & O_NOFOLLOW ? MY_NOSYMLINKS: 0); + myf flags= MY_WME | (share->data_mode & O_NOFOLLOW ? MY_NOSYMLINKS: 0); DEBUG_SYNC_C("mi_open_datafile"); info->dfile= mysql_file_open(mi_key_file_dfile, share->data_file_name, - share->mode | O_SHARE | O_CLOEXEC, MYF(flags)); + share->data_mode | O_SHARE | O_CLOEXEC, + MYF(flags)); return info->dfile >= 0 ? 0 : 1; } @@ -1291,7 +1308,8 @@ int mi_open_keyfile(MYISAM_SHARE *share) { if ((share->kfile= mysql_file_open(mi_key_file_kfile, share->unique_file_name, - share->mode | O_SHARE | O_NOFOLLOW | O_CLOEXEC, + share->index_mode | O_SHARE | O_NOFOLLOW + | O_CLOEXEC, MYF(MY_NOSYMLINKS | MY_WME))) < 0) return 1; return 0; diff --git a/storage/myisam/myisamchk.c b/storage/myisam/myisamchk.c index b9b04582a89..a9ecb46c30b 100644 --- a/storage/myisam/myisamchk.c +++ b/storage/myisam/myisamchk.c @@ -144,7 +144,7 @@ enum options_mc { OPT_READ_BUFFER_SIZE, OPT_WRITE_BUFFER_SIZE, OPT_SORT_BUFFER_SIZE, OPT_SORT_KEY_BLOCKS, OPT_DECODE_BITS, OPT_FT_MIN_WORD_LEN, OPT_FT_MAX_WORD_LEN, OPT_FT_STOPWORD_FILE, - OPT_MAX_RECORD_LENGTH, OPT_STATS_METHOD + OPT_MAX_RECORD_LENGTH, OPT_STATS_METHOD, OPT_ACTIVE_KEYS }; static struct my_option my_long_options[] = @@ -207,10 +207,16 @@ static struct my_option my_long_options[] = "Print statistics information about table that is checked.", 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, {"keys-used", 'k', - "Tell MyISAM to update only some specific keys. # is a bit mask of which keys to use. This can be used to get faster inserts.", + "Tell MyISAM to update only some specific keys. # is a bit mask of which keys to use. This can be used to get faster inserts. See also keys-active", &check_param.keys_in_use, &check_param.keys_in_use, 0, GET_ULL, REQUIRED_ARG, -1, 0, 0, 0, 0, 0}, + {"keys-active", OPT_ACTIVE_KEYS, + "Treat all not listed keys as disabled. If used with repair, the keys " + "will be disabled permanently. The argument is a list of key numbers, " + "starting from 1, separated by ','. " + "keys-active and keys-used are two ways to do the same thing", + 0, 0, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"max-record-length", OPT_MAX_RECORD_LENGTH, "Skip rows bigger than this if myisamchk can't allocate memory to hold it", &check_param.max_record_length, @@ -331,7 +337,7 @@ static struct my_option my_long_options[] = REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"stats_method", OPT_STATS_METHOD, "Specifies how index statistics collection code should treat NULLs. " - "Possible values of name are \"nulls_unequal\" (default behavior for 4.1/5.0), " + "Possible values of name are \"nulls_unequal\" (default behavior for MySQL 4.1/5.0), " "\"nulls_equal\" (emulate 4.0 behavior), and \"nulls_ignored\".", (char**) &myisam_stats_method_str, (char**) &myisam_stats_method_str, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, @@ -341,7 +347,7 @@ static struct my_option my_long_options[] = static void print_version(void) { - printf("%s Ver 2.7 for %s at %s\n", my_progname, SYSTEM_TYPE, + printf("%s Ver 2.8 for %s at %s\n", my_progname, SYSTEM_TYPE, MACHINE_TYPE); } @@ -413,6 +419,11 @@ static void usage(void) -k, --keys-used=# Tell MyISAM to update only some specific keys. # is a\n\ bit mask of which keys to use. This can be used to\n\ get faster inserts.\n\ + --keys-active\n\ + Threat all not listed keys as disabled. If used with repair, the keys\n\ + will be disabled permanently. The argument is a list of key numbers,\n\ + starting from 1, separated by ','\n\ + keys-active and keys-used are two ways to do the same thing\n\ --create-missing-keys\n\ Create missing keys. This assumes that the data\n\ file is correct and that the the number of rows stored\n\ @@ -578,6 +589,32 @@ get_one_option(const struct my_option *opt, case 'k': check_param.keys_in_use= (ulonglong) strtoll(argument, NULL, 10); break; + case OPT_ACTIVE_KEYS: + if (argument == disabled_my_option) + check_param.keys_in_use= ~0LL; + else + { + const char *start; + char *end, *str_end= strend(argument); + check_param.keys_in_use= 0; + for (start= argument; *start ; start= end) + { + int error; + longlong key; + end= str_end; + key= my_strtoll10(start, &end, &error); + if (error || key > 64 || (*end && *end != ',')) + { + fprintf(stderr, "Wrong argument to active-keys. Expected a list of " + "numbers like 1,2,3,4\n"); + exit(1); /* Change to my_exit after merge */ + } + check_param.keys_in_use|= 1LL << (key-1); + if (*end == ',') + end++; + } + } + break; case 'm': if (argument == disabled_my_option) check_param.testflag&= ~T_MEDIUM; @@ -838,7 +875,10 @@ static int myisamchk(HA_CHECK *param, char * filename) open_flags|= HA_OPEN_IGNORE_IF_LOCKED | HA_OPEN_FROM_SQL_LAYER; else open_flags|= HA_OPEN_ABORT_IF_LOCKED; - if (!(info=mi_open(filename, open_mode, open_flags))) + if (param->testflag & T_QUICK) + open_flags|= HA_OPEN_DATA_READONLY; + + if (!(info=mi_open(filename, open_mode, open_flags | HA_OPEN_FORCE_MODE))) { /* Avoid twice printing of isam file name */ param->error_printed=1; @@ -926,6 +966,13 @@ static int myisamchk(HA_CHECK *param, char * filename) DBUG_RETURN(0); } } + + /* Don't allow disable of active auto_increment keys for repair */ + if (share->base.auto_key && + (share->state.key_map & (1LL << share->base.auto_key)) && + param->testflag & (T_REP_ANY | T_SORT_RECORDS | T_SORT_INDEX)) + check_param.keys_in_use|= (1LL << share->base.auto_key); + if ((param->testflag & (T_REP_ANY | T_STATISTICS | T_SORT_RECORDS | T_SORT_INDEX)) && (((param->testflag & T_UNPACK) && @@ -1279,18 +1326,18 @@ static void descript(HA_CHECK *param, register MI_INFO *info, char * name) printf("Status: %s\n",buff); if (share->base.auto_key) { - printf("Auto increment key: %13d Last value: %13s\n", + printf("Auto increment key: %15d Last value: %13s\n", share->base.auto_key, llstr(share->state.auto_increment,llbuff)); } if (share->options & (HA_OPTION_CHECKSUM | HA_OPTION_COMPRESS_RECORD)) - printf("Checksum: %23s\n",llstr(info->state->checksum,llbuff)); + printf("Checksum: %25s\n",llstr(info->state->checksum,llbuff)); if (share->options & HA_OPTION_DELAY_KEY_WRITE) printf("Keys are only flushed at close\n"); } - printf("Data records: %13s Deleted blocks: %13s\n", + printf("Data records: %15s Deleted blocks: %18s\n", llstr(info->state->records,llbuff),llstr(info->state->del,llbuff2)); if (param->testflag & T_SILENT) DBUG_VOID_RETURN; /* This is enough */ @@ -1298,14 +1345,14 @@ static void descript(HA_CHECK *param, register MI_INFO *info, char * name) if (param->testflag & T_VERBOSE) { #ifdef USE_RELOC - printf("Init-relocation: %13s\n",llstr(share->base.reloc,llbuff)); + printf("Init-relocation: %15s\n",llstr(share->base.reloc,llbuff)); #endif - printf("Datafile parts: %13s Deleted data: %13s\n", + printf("Datafile parts: %15s Deleted data: %18s\n", llstr(share->state.split,llbuff), llstr(info->state->empty,llbuff2)); - printf("Datafile pointer (bytes):%9d Keyfile pointer (bytes):%9d\n", + printf("Datafile pointer (bytes):%11d Keyfile pointer (bytes): %13d\n", share->rec_reflength,share->base.key_reflength); - printf("Datafile length: %13s Keyfile length: %13s\n", + printf("Datafile length: %17s Keyfile length: %18s\n", llstr(info->state->data_file_length,llbuff), llstr(info->state->key_file_length,llbuff2)); @@ -1315,13 +1362,13 @@ static void descript(HA_CHECK *param, register MI_INFO *info, char * name) { if (share->base.max_data_file_length != HA_OFFSET_ERROR || share->base.max_key_file_length != HA_OFFSET_ERROR) - printf("Max datafile length: %13s Max keyfile length: %13s\n", + printf("Max datafile length: %15s Max keyfile length: %18s\n", llstr(share->base.max_data_file_length-1,llbuff), ullstr(share->base.max_key_file_length - 1, llbuff2)); } } - printf("Recordlength: %13d\n",(int) share->base.pack_reclength); + printf("Recordlength: %15d\n",(int) share->base.pack_reclength); if (! mi_is_all_keys_active(share->state.key_map, share->base.keys)) { longlong2str(share->state.key_map,buff,2); @@ -1331,7 +1378,7 @@ static void descript(HA_CHECK *param, register MI_INFO *info, char * name) puts("\ntable description:"); printf("Key Start Len Index Type"); if (param->testflag & T_VERBOSE) - printf(" Rec/key Root Blocksize"); + printf(" Rec/key Root Blocksize"); (void) putchar('\n'); for (key=keyseg_nr=0, keyinfo= &share->keyinfo[0] ; @@ -1368,7 +1415,7 @@ static void descript(HA_CHECK *param, register MI_INFO *info, char * name) else buff[0]=0; if (param->testflag & T_VERBOSE) - printf("%11lu %12s %10d", + printf("%11lu %14s %10d", share->state.rec_per_key_part[keyseg_nr++], buff,keyinfo->block_length); (void) putchar('\n'); diff --git a/storage/myisam/myisamdef.h b/storage/myisam/myisamdef.h index 3a389802457..c9a6b550b76 100644 --- a/storage/myisam/myisamdef.h +++ b/storage/myisam/myisamdef.h @@ -207,7 +207,8 @@ typedef struct st_mi_isam_share uint32 ftkeys; /* Number of full-text keys + 1 */ File kfile; /* Shared keyfile */ File data_file; /* Shared data file */ - int mode; /* mode of file on open */ + int index_mode; /* mode on index file on open */ + int data_mode; /* mode of data file on open */ uint reopen; /* How many times reopened */ uint w_locks,r_locks,tot_locks; /* Number of read/write locks */ uint blocksize; /* blocksize of keyfile */