diff --git a/include/myisam.h b/include/myisam.h index f11b18824d5..4f8fc149ba1 100644 --- a/include/myisam.h +++ b/include/myisam.h @@ -360,31 +360,6 @@ typedef struct st_sort_info pthread_cond_t cond; } SORT_INFO; - -typedef struct st_mi_sort_param -{ - pthread_t thr; - IO_CACHE read_cache, tempfile, tempfile_for_exceptions; - DYNAMIC_ARRAY buffpek; - ulonglong unique[MI_MAX_KEY_SEG+1]; - my_off_t pos,max_pos,filepos,start_recpos; - uint key, key_length,real_key_length,sortbuff_size; - uint maxbuffers, keys, find_length, sort_keys_length; - my_bool fix_datafile, master; - MI_KEYDEF *keyinfo; - SORT_INFO *sort_info; - uchar **sort_keys; - byte *rec_buff; - void *wordlist, *wordptr; - char *record; - MY_TMPDIR *tmpdir; - int (*key_cmp)(struct st_mi_sort_param *, const void *, const void *); - int (*key_read)(struct st_mi_sort_param *,void *); - int (*key_write)(struct st_mi_sort_param *, const void *); - void (*lock_in_memory)(MI_CHECK *); -} MI_SORT_PARAM; - - /* functions in mi_check */ void myisamchk_init(MI_CHECK *param); int chk_status(MI_CHECK *param, MI_INFO *info); @@ -415,9 +390,7 @@ int filecopy(MI_CHECK *param, File to,File from,my_off_t start, my_off_t length, const char *type); int movepoint(MI_INFO *info,byte *record,my_off_t oldpos, my_off_t newpos, uint prot_key); -int sort_write_record(MI_SORT_PARAM *sort_param); int write_data_suffix(SORT_INFO *sort_info, my_bool fix_datafile); -int _create_index_by_sort(MI_SORT_PARAM *info,my_bool no_messages, ulong); int test_if_almost_full(MI_INFO *info); int recreate_table(MI_CHECK *param, MI_INFO **org_info, char *filename); void mi_disable_non_unique_index(MI_INFO *info, ha_rows rows); diff --git a/myisam/mi_check.c b/myisam/mi_check.c index ba90b2c5bcf..d3222a770a8 100644 --- a/myisam/mi_check.c +++ b/myisam/mi_check.c @@ -3682,10 +3682,7 @@ static my_bool mi_too_big_key_for_sort(MI_KEYDEF *key, ha_rows rows) key_maxlength+=ft_max_word_len_for_sort-HA_FT_MAXLEN; return (key->flag & (HA_BINARY_PACK_KEY | HA_VAR_LENGTH_KEY | HA_FULLTEXT) && ((ulonglong) rows * key_maxlength > - (ulonglong) myisam_max_temp_length || - (ulonglong) rows * (key_maxlength - key->minlength) / 2 > - myisam_max_extra_temp_length || - (rows == 0 && (key_maxlength / key->minlength) > 2))); + (ulonglong) myisam_max_temp_length)); } diff --git a/myisam/myisamdef.h b/myisam/myisamdef.h index c63f4a28562..90aaff46919 100644 --- a/myisam/myisamdef.h +++ b/myisam/myisamdef.h @@ -275,7 +275,41 @@ struct st_myisam_info { int rtree_recursion_depth; }; +typedef struct st_buffpek { + my_off_t file_pos; /* Where we are in the sort file */ + uchar *base,*key; /* Key pointers */ + ha_rows count; /* Number of rows in table */ + ulong mem_count; /* numbers of keys in memory */ + ulong max_keys; /* Max keys in buffert */ +} BUFFPEK; +typedef struct st_mi_sort_param +{ + pthread_t thr; + IO_CACHE read_cache, tempfile, tempfile_for_exceptions; + DYNAMIC_ARRAY buffpek; + ulonglong unique[MI_MAX_KEY_SEG+1]; + my_off_t pos,max_pos,filepos,start_recpos; + uint key, key_length,real_key_length,sortbuff_size; + uint maxbuffers, keys, find_length, sort_keys_length; + my_bool fix_datafile, master; + MI_KEYDEF *keyinfo; + SORT_INFO *sort_info; + uchar **sort_keys; + byte *rec_buff; + void *wordlist, *wordptr; + char *record; + MY_TMPDIR *tmpdir; + int (*key_cmp)(struct st_mi_sort_param *, const void *, const void *); + int (*key_read)(struct st_mi_sort_param *,void *); + int (*key_write)(struct st_mi_sort_param *, const void *); + void (*lock_in_memory)(MI_CHECK *); + NEAR int (*write_keys)(struct st_mi_sort_param *, register uchar **, + uint , struct st_buffpek *, IO_CACHE *); + NEAR uint (*read_to_buffer)(IO_CACHE *,struct st_buffpek *, uint); + NEAR int (*write_key)(struct st_mi_sort_param *, IO_CACHE *,char *, + uint, uint); +} MI_SORT_PARAM; /* Some defines used by isam-funktions */ #define USE_WHOLE_KEY MI_MAX_KEY_BUFF*2 /* Use whole key in _mi_search() */ @@ -662,6 +696,9 @@ int thr_write_keys(MI_SORT_PARAM *sort_param); pthread_handler_decl(thr_find_all_keys,arg); #endif +int sort_write_record(MI_SORT_PARAM *sort_param); +int _create_index_by_sort(MI_SORT_PARAM *info,my_bool no_messages, ulong); + #ifdef __cplusplus } #endif diff --git a/myisam/sort.c b/myisam/sort.c index 50618e43991..c9ba0f51d9c 100644 --- a/myisam/sort.c +++ b/myisam/sort.c @@ -39,13 +39,10 @@ #define MYF_RW MYF(MY_NABP | MY_WME | MY_WAIT_IF_FULL) #define DISK_BUFFER_SIZE (IO_SIZE*16) -typedef struct st_buffpek { - my_off_t file_pos; /* Where we are in the sort file */ - uchar *base,*key; /* Key pointers */ - ha_rows count; /* Number of rows in table */ - ulong mem_count; /* numbers of keys in memory */ - ulong max_keys; /* Max keys in buffert */ -} BUFFPEK; + +/* + Pointers of functions for store and read keys from temp file +*/ extern void print_error _VARARGS((const char *fmt,...)); @@ -56,7 +53,7 @@ static ha_rows NEAR_F find_all_keys(MI_SORT_PARAM *info,uint keys, DYNAMIC_ARRAY *buffpek,int *maxbuffer, IO_CACHE *tempfile, IO_CACHE *tempfile_for_exceptions); -static int NEAR_F write_keys(MI_SORT_PARAM *info,uchar * *sort_keys, +static int NEAR_F write_keys(MI_SORT_PARAM *info,uchar **sort_keys, uint count, BUFFPEK *buffpek,IO_CACHE *tempfile); static int NEAR_F write_key(MI_SORT_PARAM *info, uchar *key, IO_CACHE *tempfile); @@ -74,8 +71,17 @@ static int NEAR_F merge_buffers(MI_SORT_PARAM *info,uint keys, BUFFPEK *Fb, BUFFPEK *Tb); static int NEAR_F merge_index(MI_SORT_PARAM *,uint,uchar **,BUFFPEK *, int, IO_CACHE *); - - + +static int NEAR_F write_keys_varlen(MI_SORT_PARAM *info,uchar **sort_keys, + uint count, BUFFPEK *buffpek, + IO_CACHE *tempfile); +static uint NEAR_F read_to_buffer_varlen(IO_CACHE *fromfile,BUFFPEK *buffpek, + uint sort_length); +static int NEAR_F write_merge_key(MI_SORT_PARAM *info, IO_CACHE *to_file, + char* key, uint sort_length, uint count); +static int NEAR_F write_merge_key_varlen(MI_SORT_PARAM *info, IO_CACHE *to_file, + char* key, uint sort_length, uint count); +inline int my_var_write(MI_SORT_PARAM *info,IO_CACHE *to_file,char *bufs); /* Creates a index of sorted keys @@ -102,6 +108,19 @@ int _create_index_by_sort(MI_SORT_PARAM *info,my_bool no_messages, DBUG_ENTER("_create_index_by_sort"); DBUG_PRINT("enter",("sort_length: %d", info->key_length)); + if (info->keyinfo->flag && HA_VAR_LENGTH_KEY) + { + info->write_keys=write_keys_varlen; + info->read_to_buffer=read_to_buffer_varlen; + info->write_key=write_merge_key_varlen; + } + else + { + info->write_keys=write_keys; + info->read_to_buffer=read_to_buffer; + info->write_key=write_merge_key; + } + my_b_clear(&tempfile); my_b_clear(&tempfile_for_exceptions); bzero((char*) &buffpek,sizeof(buffpek)); @@ -249,7 +268,7 @@ static ha_rows NEAR_F find_all_keys(MI_SORT_PARAM *info, uint keys, if (++idx == keys) { - if (write_keys(info,sort_keys,idx-1,(BUFFPEK *)alloc_dynamic(buffpek), + if (info->write_keys(info,sort_keys,idx-1,(BUFFPEK *)alloc_dynamic(buffpek), tempfile)) DBUG_RETURN(HA_POS_ERROR); /* purecov: inspected */ @@ -263,7 +282,7 @@ static ha_rows NEAR_F find_all_keys(MI_SORT_PARAM *info, uint keys, DBUG_RETURN(HA_POS_ERROR); /* Aborted by get_key */ /* purecov: inspected */ if (buffpek->elements) { - if (write_keys(info,sort_keys,idx,(BUFFPEK *)alloc_dynamic(buffpek), + if (info->write_keys(info,sort_keys,idx,(BUFFPEK *)alloc_dynamic(buffpek), tempfile)) DBUG_RETURN(HA_POS_ERROR); /* purecov: inspected */ *maxbuffer=buffpek->elements-1; @@ -292,6 +311,19 @@ pthread_handler_decl(thr_find_all_keys,arg) if (info->sort_info->got_error) goto err; + if (info->keyinfo->flag && HA_VAR_LENGTH_KEY) + { + info->write_keys=write_keys_varlen; + info->read_to_buffer=read_to_buffer_varlen; + info->write_key=write_merge_key_varlen; + } + else + { + info->write_keys=write_keys; + info->read_to_buffer=read_to_buffer; + info->write_key=write_merge_key; + } + my_b_clear(&info->tempfile); my_b_clear(&info->tempfile_for_exceptions); bzero((char*) &info->buffpek,sizeof(info->buffpek)); @@ -365,7 +397,7 @@ pthread_handler_decl(thr_find_all_keys,arg) if (++idx == keys) { - if (write_keys(info,sort_keys,idx-1, + if (info->write_keys(info,sort_keys,idx-1, (BUFFPEK *)alloc_dynamic(&info->buffpek), &info->tempfile)) goto err; @@ -379,7 +411,7 @@ pthread_handler_decl(thr_find_all_keys,arg) goto err; if (info->buffpek.elements) { - if (write_keys(info,sort_keys, idx, + if (info->write_keys(info,sort_keys, idx, (BUFFPEK *) alloc_dynamic(&info->buffpek), &info->tempfile)) goto err; info->keys=(info->buffpek.elements-1)*(keys-1)+idx; @@ -464,6 +496,18 @@ int thr_write_keys(MI_SORT_PARAM *sort_param) { if (got_error) continue; + if (sinfo->keyinfo->flag && HA_VAR_LENGTH_KEY) + { + sinfo->write_keys=write_keys_varlen; + sinfo->read_to_buffer=read_to_buffer_varlen; + sinfo->write_key=write_merge_key_varlen; + } + else + { + sinfo->write_keys=write_keys; + sinfo->read_to_buffer=read_to_buffer; + sinfo->write_key=write_merge_key; + } if (sinfo->buffpek.elements) { uint maxbuffer=sinfo->buffpek.elements-1; @@ -568,6 +612,43 @@ static int NEAR_F write_keys(MI_SORT_PARAM *info, register uchar **sort_keys, DBUG_RETURN(0); } /* write_keys */ +inline int my_var_write(MI_SORT_PARAM *info,IO_CACHE *to_file,char *bufs) +{ + int err; + uint16 len = _mi_keylength(info->keyinfo,bufs); + + if (err=my_b_write(to_file,(byte*)&len,sizeof(len))) + return(err); + if (err=my_b_write(to_file,(byte*)bufs,(uint) len)) + return(err); + return(0); +} + + +static int NEAR_F write_keys_varlen(MI_SORT_PARAM *info, register uchar **sort_keys, + uint count, BUFFPEK *buffpek, IO_CACHE *tempfile) +{ + uchar **end; + int err; + DBUG_ENTER("write_keys_varlen"); + + qsort2((byte*) sort_keys,count,sizeof(byte*),(qsort2_cmp) info->key_cmp, + info); + if (!my_b_inited(tempfile) && + open_cached_file(tempfile, my_tmpdir(info->tmpdir), "ST", + DISK_BUFFER_SIZE, info->sort_info->param->myf_rw)) + DBUG_RETURN(1); /* purecov: inspected */ + + buffpek->file_pos=my_b_tell(tempfile); + buffpek->count=count; + for (end=sort_keys+count ; sort_keys != end ; sort_keys++) + { + if (err=my_var_write(info,tempfile,*sort_keys)) + DBUG_RETURN(err); + } + DBUG_RETURN(0); +} /* write_keys_varlen */ + static int NEAR_F write_key(MI_SORT_PARAM *info, uchar *key, IO_CACHE *tempfile) @@ -683,6 +764,59 @@ static uint NEAR_F read_to_buffer(IO_CACHE *fromfile, BUFFPEK *buffpek, return (count*sort_length); } /* read_to_buffer */ +static uint NEAR_F read_to_buffer_varlen(IO_CACHE *fromfile, BUFFPEK *buffpek, + uint sort_length) +{ + register uint count; + uint16 length_of_key = 0; + uint idx; + uchar *buffp; + + if ((count=(uint) min((ha_rows) buffpek->max_keys,buffpek->count))) + { + buffp = buffpek->base; + + for (idx=1;idx<=count;idx++) + { + if (my_pread(fromfile->file,(byte*)&length_of_key,sizeof(length_of_key), + buffpek->file_pos,MYF_RW)) + return((uint) -1); + buffpek->file_pos+=sizeof(length_of_key); + if (my_pread(fromfile->file,(byte*) buffp,length_of_key, + buffpek->file_pos,MYF_RW)) + return((uint) -1); + buffpek->file_pos+=length_of_key; + buffp = buffp + sort_length; + } + buffpek->key=buffpek->base; + buffpek->count-= count; + buffpek->mem_count= count; + } + return (count*sort_length); +} /* read_to_buffer_varlen */ + + +static int NEAR_F write_merge_key_varlen(MI_SORT_PARAM *info, IO_CACHE *to_file,char* key, + uint sort_length, uint count) +{ + uint idx; + + char *bufs = key; + for (idx=1;idx<=count;idx++) + { + int err; + if (err = my_var_write(info,to_file,bufs)) + return(err); + bufs=bufs+sort_length; + } + return(0); +} + +static int NEAR_F write_merge_key(MI_SORT_PARAM *info, IO_CACHE *to_file,char* key, + uint sort_length, uint count) +{ + return(my_b_write(to_file,(byte*) key,(uint) sort_length*count)); +} /* Merge buffers to one buffer @@ -722,8 +856,7 @@ merge_buffers(MI_SORT_PARAM *info, uint keys, IO_CACHE *from_file, count+= buffpek->count; buffpek->base= strpos; buffpek->max_keys=maxcount; - strpos+= (uint) (error=(int) read_to_buffer(from_file,buffpek, - sort_length)); + strpos+= (uint) (error=(int) info->read_to_buffer(from_file,buffpek,sort_length)); if (error == -1) goto err; /* purecov: inspected */ queue_insert(&queue,(char*) buffpek); @@ -740,7 +873,7 @@ merge_buffers(MI_SORT_PARAM *info, uint keys, IO_CACHE *from_file, buffpek=(BUFFPEK*) queue_top(&queue); if (to_file) { - if (my_b_write(to_file,(byte*) buffpek->key,(uint) sort_length)) + if (info->write_key(info,to_file,(byte*) buffpek->key,(uint) sort_length,1)) { error=1; goto err; /* purecov: inspected */ } @@ -755,7 +888,7 @@ merge_buffers(MI_SORT_PARAM *info, uint keys, IO_CACHE *from_file, buffpek->key+=sort_length; if (! --buffpek->mem_count) { - if (!(error=(int) read_to_buffer(from_file,buffpek,sort_length))) + if (!(error=(int) info->read_to_buffer(from_file,buffpek,sort_length))) { uchar *base=buffpek->base; uint max_keys=buffpek->max_keys; @@ -795,8 +928,8 @@ merge_buffers(MI_SORT_PARAM *info, uint keys, IO_CACHE *from_file, { if (to_file) { - if (my_b_write(to_file,(byte*) buffpek->key, - (sort_length*buffpek->mem_count))) + if (info->write_key(info,to_file,(byte*) buffpek->key, + sort_length,buffpek->mem_count)) { error=1; goto err; /* purecov: inspected */ } @@ -816,7 +949,7 @@ merge_buffers(MI_SORT_PARAM *info, uint keys, IO_CACHE *from_file, } } } - while ((error=(int) read_to_buffer(from_file,buffpek,sort_length)) != -1 && + while ((error=(int) info->read_to_buffer(from_file,buffpek,sort_length)) != -1 && error != 0); lastbuff->count=count;