From 405fb9a5d4f972e6b5ed4e582bfd8b94a8511c75 Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 13 Nov 2002 03:53:45 +0500 Subject: [PATCH 1/3] add functions for store keys to temporary file with variable length myisam/sort.c: add function for store keys to temporary files with variable length BitKeeper/etc/logging_ok: Logging to logging@openlogging.org accepted --- BitKeeper/etc/logging_ok | 1 + myisam/sort.c | 207 +++++++++++++++++++++++++++++++++------ 2 files changed, 179 insertions(+), 29 deletions(-) diff --git a/BitKeeper/etc/logging_ok b/BitKeeper/etc/logging_ok index aec1275d516..8134a08ea5e 100644 --- a/BitKeeper/etc/logging_ok +++ b/BitKeeper/etc/logging_ok @@ -90,6 +90,7 @@ venu@myvenu.com venu@work.mysql.com vva@genie.(none) walrus@mysql.com +wax@mysql.com worm@altair.is.lan zak@balfor.local zak@linux.local diff --git a/myisam/sort.c b/myisam/sort.c index 0e69e41ed5c..acf375a2bca 100644 --- a/myisam/sort.c +++ b/myisam/sort.c @@ -47,6 +47,18 @@ typedef struct st_buffpek { ulong max_keys; /* Max keys in buffert */ } BUFFPEK; + +/* + Pointers of functions for store and read keys from temp file +*/ +typedef struct st_key_store { + NEAR int (*write_keys)(struct st_mi_sort_param *, register uchar **, + uint , struct st_buffpek *, IO_CACHE *); + NEAR uint (*read_to_buffer)(IO_CACHE *,struct st_buffpek *, uint); + NEAR int (*write_key)(struct st_mi_sort_param *, IO_CACHE *,char *, + uint, uint); +} KEY_STORE; + extern void print_error _VARARGS((const char *fmt,...)); /* Functions defined in this file */ @@ -55,8 +67,9 @@ static ha_rows NEAR_F find_all_keys(MI_SORT_PARAM *info,uint keys, uchar **sort_keys, DYNAMIC_ARRAY *buffpek,int *maxbuffer, IO_CACHE *tempfile, - IO_CACHE *tempfile_for_exceptions); -static int NEAR_F write_keys(MI_SORT_PARAM *info,uchar * *sort_keys, + IO_CACHE *tempfile_for_exceptions, + KEY_STORE *key_store); +static int NEAR_F write_keys(MI_SORT_PARAM *info,uchar **sort_keys, uint count, BUFFPEK *buffpek,IO_CACHE *tempfile); static int NEAR_F write_key(MI_SORT_PARAM *info, uchar *key, IO_CACHE *tempfile); @@ -65,16 +78,25 @@ static int NEAR_F write_index(MI_SORT_PARAM *info,uchar * *sort_keys, static int NEAR_F merge_many_buff(MI_SORT_PARAM *info,uint keys, uchar * *sort_keys, BUFFPEK *buffpek,int *maxbuffer, - IO_CACHE *t_file); + IO_CACHE *t_file,KEY_STORE *key_store); static uint NEAR_F read_to_buffer(IO_CACHE *fromfile,BUFFPEK *buffpek, uint sort_length); static int NEAR_F merge_buffers(MI_SORT_PARAM *info,uint keys, IO_CACHE *from_file, IO_CACHE *to_file, uchar * *sort_keys, BUFFPEK *lastbuff, - BUFFPEK *Fb, BUFFPEK *Tb); + BUFFPEK *Fb, BUFFPEK *Tb, + KEY_STORE *key_store); static int NEAR_F merge_index(MI_SORT_PARAM *,uint,uchar **,BUFFPEK *, int, - IO_CACHE *); - + IO_CACHE *,KEY_STORE *); + +static int NEAR_F write_keys_varlen(MI_SORT_PARAM *info,uchar **sort_keys, + uint count, BUFFPEK *buffpek,IO_CACHE *tempfile); +static uint NEAR_F read_to_buffer_varlen(IO_CACHE *fromfile,BUFFPEK *buffpek, + uint sort_length); +static int NEAR_F write_merge_key(MI_SORT_PARAM *info, IO_CACHE *to_file,char* key, + uint sort_length, uint count) ; +static int NEAR_F write_merge_key_varlen(MI_SORT_PARAM *info, IO_CACHE *to_file,char* key, + uint sort_length, uint count); /* Creates a index of sorted keys @@ -99,9 +121,24 @@ int _create_index_by_sort(MI_SORT_PARAM *info,my_bool no_messages, ha_rows records; uchar **sort_keys; IO_CACHE tempfile, tempfile_for_exceptions; + KEY_STORE key_store; DBUG_ENTER("_create_index_by_sort"); DBUG_PRINT("enter",("sort_length: %d", info->key_length)); + if ((info->keyinfo->seg->type == HA_KEYTYPE_TEXT)|| + (info->keyinfo->seg->type == HA_KEYTYPE_VARTEXT)) + { + key_store.write_keys=write_keys_varlen; + key_store.read_to_buffer=read_to_buffer_varlen; + key_store.write_key=write_merge_key_varlen; + } + else + { + key_store.write_keys=write_keys; + key_store.read_to_buffer=read_to_buffer; + key_store.write_key=write_merge_key; + } + my_b_clear(&tempfile); my_b_clear(&tempfile_for_exceptions); bzero((char*) &buffpek,sizeof(buffpek)); @@ -157,7 +194,7 @@ int _create_index_by_sort(MI_SORT_PARAM *info,my_bool no_messages, printf(" - Searching for keys, allocating buffer for %d keys\n",keys); if ((records=find_all_keys(info,keys,sort_keys,&buffpek,&maxbuffer, - &tempfile,&tempfile_for_exceptions)) + &tempfile,&tempfile_for_exceptions,&key_store)) == HA_POS_ERROR) goto err; /* purecov: tested */ if (maxbuffer == 0) @@ -175,7 +212,7 @@ int _create_index_by_sort(MI_SORT_PARAM *info,my_bool no_messages, if (!no_messages) printf(" - Merging %lu keys\n",records); /* purecov: tested */ if (merge_many_buff(info,keys,sort_keys, - dynamic_element(&buffpek,0,BUFFPEK *),&maxbuffer,&tempfile)) + dynamic_element(&buffpek,0,BUFFPEK *),&maxbuffer,&tempfile,&key_store)) goto err; /* purecov: inspected */ } if (flush_io_cache(&tempfile) || @@ -184,7 +221,7 @@ int _create_index_by_sort(MI_SORT_PARAM *info,my_bool no_messages, if (!no_messages) puts(" - Last merge and dumping keys\n"); /* purecov: tested */ if (merge_index(info,keys,sort_keys,dynamic_element(&buffpek,0,BUFFPEK *), - maxbuffer,&tempfile)) + maxbuffer,&tempfile,&key_store)) goto err; /* purecov: inspected */ } @@ -229,7 +266,8 @@ err: static ha_rows NEAR_F find_all_keys(MI_SORT_PARAM *info, uint keys, uchar **sort_keys, DYNAMIC_ARRAY *buffpek, int *maxbuffer, IO_CACHE *tempfile, - IO_CACHE *tempfile_for_exceptions) + IO_CACHE *tempfile_for_exceptions, + KEY_STORE *key_store) { int error; uint idx; @@ -249,7 +287,7 @@ static ha_rows NEAR_F find_all_keys(MI_SORT_PARAM *info, uint keys, if (++idx == keys) { - if (write_keys(info,sort_keys,idx-1,(BUFFPEK *)alloc_dynamic(buffpek), + if (key_store->write_keys(info,sort_keys,idx-1,(BUFFPEK *)alloc_dynamic(buffpek), tempfile)) DBUG_RETURN(HA_POS_ERROR); /* purecov: inspected */ @@ -263,7 +301,7 @@ static ha_rows NEAR_F find_all_keys(MI_SORT_PARAM *info, uint keys, DBUG_RETURN(HA_POS_ERROR); /* Aborted by get_key */ /* purecov: inspected */ if (buffpek->elements) { - if (write_keys(info,sort_keys,idx,(BUFFPEK *)alloc_dynamic(buffpek), + if (key_store->write_keys(info,sort_keys,idx,(BUFFPEK *)alloc_dynamic(buffpek), tempfile)) DBUG_RETURN(HA_POS_ERROR); /* purecov: inspected */ *maxbuffer=buffpek->elements-1; @@ -284,6 +322,7 @@ pthread_handler_decl(thr_find_all_keys,arg) uint memavl,old_memavl,keys,sort_length; uint idx, maxbuffer; uchar **sort_keys=0; + KEY_STORE key_store; error=1; @@ -292,6 +331,20 @@ pthread_handler_decl(thr_find_all_keys,arg) if (info->sort_info->got_error) goto err; + if ((info->keyinfo->seg->type == HA_KEYTYPE_TEXT)|| + (info->keyinfo->seg->type == HA_KEYTYPE_VARTEXT)) + { + key_store.write_keys=write_keys_varlen; + key_store.read_to_buffer=read_to_buffer_varlen; + key_store.write_key=write_merge_key_varlen; + } + else + { + key_store.write_keys=write_keys; + key_store.read_to_buffer=read_to_buffer; + key_store.write_key=write_merge_key; + } + my_b_clear(&info->tempfile); my_b_clear(&info->tempfile_for_exceptions); bzero((char*) &info->buffpek,sizeof(info->buffpek)); @@ -365,7 +418,7 @@ pthread_handler_decl(thr_find_all_keys,arg) if (++idx == keys) { - if (write_keys(info,sort_keys,idx-1, + if (key_store.write_keys(info,sort_keys,idx-1, (BUFFPEK *)alloc_dynamic(&info->buffpek), &info->tempfile)) goto err; @@ -379,7 +432,7 @@ pthread_handler_decl(thr_find_all_keys,arg) goto err; if (info->buffpek.elements) { - if (write_keys(info,sort_keys, idx, + if (key_store.write_keys(info,sort_keys, idx, (BUFFPEK *) alloc_dynamic(&info->buffpek), &info->tempfile)) goto err; info->keys=(info->buffpek.elements-1)*(keys-1)+idx; @@ -421,6 +474,7 @@ int thr_write_keys(MI_SORT_PARAM *sort_param) MI_INFO *info=sort_info->info; MYISAM_SHARE *share=info->s; MI_SORT_PARAM *sinfo; + KEY_STORE key_store; byte *mergebuf=0; LINT_INIT(length); @@ -464,6 +518,19 @@ int thr_write_keys(MI_SORT_PARAM *sort_param) { if (got_error) continue; + if ((sinfo->keyinfo->seg->type == HA_KEYTYPE_TEXT)|| + (sinfo->keyinfo->seg->type == HA_KEYTYPE_VARTEXT)) + { + key_store.write_keys=write_keys_varlen; + key_store.read_to_buffer=read_to_buffer_varlen; + key_store.write_key=write_merge_key_varlen; + } + else + { + key_store.write_keys=write_keys; + key_store.read_to_buffer=read_to_buffer; + key_store.write_key=write_merge_key; + } if (sinfo->buffpek.elements) { uint maxbuffer=sinfo->buffpek.elements-1; @@ -488,7 +555,7 @@ int thr_write_keys(MI_SORT_PARAM *sort_param) printf("Key %d - Merging %u keys\n",sinfo->key+1, sinfo->keys); if (merge_many_buff(sinfo, keys, (uchar **)mergebuf, dynamic_element(&sinfo->buffpek, 0, BUFFPEK *), - &maxbuffer, &sinfo->tempfile)) + &maxbuffer, &sinfo->tempfile,&key_store)) { got_error=1; continue; @@ -504,7 +571,7 @@ int thr_write_keys(MI_SORT_PARAM *sort_param) printf("Key %d - Last merge and dumping keys\n", sinfo->key+1); if (merge_index(sinfo, keys, (uchar **)mergebuf, dynamic_element(&sinfo->buffpek,0,BUFFPEK *), - maxbuffer,&sinfo->tempfile) || + maxbuffer,&sinfo->tempfile,&key_store) || flush_pending_blocks(sinfo)) { got_error=1; @@ -568,6 +635,32 @@ static int NEAR_F write_keys(MI_SORT_PARAM *info, register uchar **sort_keys, DBUG_RETURN(0); } /* write_keys */ +static int NEAR_F write_keys_varlen(MI_SORT_PARAM *info, register uchar **sort_keys, + uint count, BUFFPEK *buffpek, IO_CACHE *tempfile) +{ + uchar **end; + DBUG_ENTER("write_keys_varlen"); + + qsort2((byte*) sort_keys,count,sizeof(byte*),(qsort2_cmp) info->key_cmp, + info); + if (!my_b_inited(tempfile) && + open_cached_file(tempfile, my_tmpdir(info->tmpdir), "ST", + DISK_BUFFER_SIZE, info->sort_info->param->myf_rw)) + DBUG_RETURN(1); /* purecov: inspected */ + + buffpek->file_pos=my_b_tell(tempfile); + buffpek->count=count; + for (end=sort_keys+count ; sort_keys != end ; sort_keys++) + { + uint16 len = _mi_keylength(info->keyinfo,*sort_keys); + if (my_b_write(tempfile,(byte*)&len,sizeof(len))) + DBUG_RETURN(1); + if (my_b_write(tempfile,(byte*) *sort_keys,(uint) len)) + DBUG_RETURN(1); + } + DBUG_RETURN(0); +} /* write_keys_varlen */ + static int NEAR_F write_key(MI_SORT_PARAM *info, uchar *key, IO_CACHE *tempfile) @@ -609,7 +702,8 @@ static int NEAR_F write_index(MI_SORT_PARAM *info, register uchar **sort_keys, static int NEAR_F merge_many_buff(MI_SORT_PARAM *info, uint keys, uchar **sort_keys, BUFFPEK *buffpek, - int *maxbuffer, IO_CACHE *t_file) + int *maxbuffer, IO_CACHE *t_file, + KEY_STORE *key_store) { register int i; IO_CACHE t_file2, *from_file, *to_file, *temp; @@ -632,11 +726,11 @@ static int NEAR_F merge_many_buff(MI_SORT_PARAM *info, uint keys, for (i=0 ; i <= *maxbuffer-MERGEBUFF*3/2 ; i+=MERGEBUFF) { if (merge_buffers(info,keys,from_file,to_file,sort_keys,lastbuff++, - buffpek+i,buffpek+i+MERGEBUFF-1)) + buffpek+i,buffpek+i+MERGEBUFF-1,key_store)) break; /* purecov: inspected */ } if (merge_buffers(info,keys,from_file,to_file,sort_keys,lastbuff++, - buffpek+i,buffpek+ *maxbuffer)) + buffpek+i,buffpek+ *maxbuffer,key_store)) break; /* purecov: inspected */ if (flush_io_cache(to_file)) break; /* purecov: inspected */ @@ -683,6 +777,62 @@ static uint NEAR_F read_to_buffer(IO_CACHE *fromfile, BUFFPEK *buffpek, return (count*sort_length); } /* read_to_buffer */ +static uint NEAR_F read_to_buffer_varlen(IO_CACHE *fromfile, BUFFPEK *buffpek, + uint sort_length) +{ + register uint count; + uint16 length_of_key = 0; + uint idx; + uchar *buffp; + + if ((count=(uint) min((ha_rows) buffpek->max_keys,buffpek->count))) + { + buffp = buffpek->base; + + for (idx=1;idx<=count;idx++) + { + if (my_pread(fromfile->file,(byte*)&length_of_key,sizeof(length_of_key), + buffpek->file_pos,MYF_RW)) + return((uint) -1); + buffpek->file_pos+=sizeof(length_of_key); + if (my_pread(fromfile->file,(byte*) buffp,length_of_key, + buffpek->file_pos,MYF_RW)) + return((uint) -1); + buffpek->file_pos+=length_of_key; + buffp = buffp + sort_length; + } + buffpek->key=buffpek->base; + buffpek->count-= count; + buffpek->mem_count= count; + } + return (count*sort_length); +} /* read_to_buffer_varlen */ + + +static int NEAR_F write_merge_key_varlen(MI_SORT_PARAM *info, IO_CACHE *to_file,char* key, + uint sort_length, uint count) +{ + uint idx; + int err; + + uchar *bufs = key; + for (idx=1;idx<=count;idx++) + { + uint16 len = _mi_keylength(info->keyinfo,bufs); + if (err=my_b_write(to_file,(byte*)&len,sizeof(len))) + return(err); + if (err=my_b_write(to_file,(byte*)bufs,(uint) len)) + return(err); + bufs=bufs+sort_length; + } + return(0); +} + +static int NEAR_F write_merge_key(MI_SORT_PARAM *info, IO_CACHE *to_file,char* key, + uint sort_length, uint count) +{ + return(my_b_write(to_file,(byte*) key,(uint) sort_length*count)); +} /* Merge buffers to one buffer @@ -692,7 +842,7 @@ static uint NEAR_F read_to_buffer(IO_CACHE *fromfile, BUFFPEK *buffpek, static int NEAR_F merge_buffers(MI_SORT_PARAM *info, uint keys, IO_CACHE *from_file, IO_CACHE *to_file, uchar **sort_keys, BUFFPEK *lastbuff, - BUFFPEK *Fb, BUFFPEK *Tb) + BUFFPEK *Fb, BUFFPEK *Tb, KEY_STORE *key_store) { int error; uint sort_length,maxcount; @@ -722,8 +872,7 @@ merge_buffers(MI_SORT_PARAM *info, uint keys, IO_CACHE *from_file, count+= buffpek->count; buffpek->base= strpos; buffpek->max_keys=maxcount; - strpos+= (uint) (error=(int) read_to_buffer(from_file,buffpek, - sort_length)); + strpos+= (uint) (error=(int) key_store->read_to_buffer(from_file,buffpek,sort_length)); if (error == -1) goto err; /* purecov: inspected */ queue_insert(&queue,(char*) buffpek); @@ -740,7 +889,7 @@ merge_buffers(MI_SORT_PARAM *info, uint keys, IO_CACHE *from_file, buffpek=(BUFFPEK*) queue_top(&queue); if (to_file) { - if (my_b_write(to_file,(byte*) buffpek->key,(uint) sort_length)) + if (key_store->write_key(info,to_file,(byte*) buffpek->key,(uint) sort_length,1)) { error=1; goto err; /* purecov: inspected */ } @@ -755,7 +904,7 @@ merge_buffers(MI_SORT_PARAM *info, uint keys, IO_CACHE *from_file, buffpek->key+=sort_length; if (! --buffpek->mem_count) { - if (!(error=(int) read_to_buffer(from_file,buffpek,sort_length))) + if (!(error=(int) key_store->read_to_buffer(from_file,buffpek,sort_length))) { uchar *base=buffpek->base; uint max_keys=buffpek->max_keys; @@ -795,8 +944,8 @@ merge_buffers(MI_SORT_PARAM *info, uint keys, IO_CACHE *from_file, { if (to_file) { - if (my_b_write(to_file,(byte*) buffpek->key, - (sort_length*buffpek->mem_count))) + if (key_store->write_key(info,to_file,(byte*) buffpek->key, + sort_length,buffpek->mem_count)) { error=1; goto err; /* purecov: inspected */ } @@ -816,7 +965,7 @@ merge_buffers(MI_SORT_PARAM *info, uint keys, IO_CACHE *from_file, } } } - while ((error=(int) read_to_buffer(from_file,buffpek,sort_length)) != -1 && + while ((error=(int) key_store->read_to_buffer(from_file,buffpek,sort_length)) != -1 && error != 0); lastbuff->count=count; @@ -832,11 +981,11 @@ err: static int NEAR_F merge_index(MI_SORT_PARAM *info, uint keys, uchar **sort_keys, - BUFFPEK *buffpek, int maxbuffer, IO_CACHE *tempfile) + BUFFPEK *buffpek, int maxbuffer, IO_CACHE *tempfile,KEY_STORE *key_store) { DBUG_ENTER("merge_index"); if (merge_buffers(info,keys,tempfile,(IO_CACHE*) 0,sort_keys,buffpek,buffpek, - buffpek+maxbuffer)) + buffpek+maxbuffer,key_store)) DBUG_RETURN(1); /* purecov: inspected */ DBUG_RETURN(0); } /* merge_index */ From b9a431c41ca843805ac630d0cf521d2c19083e01 Mon Sep 17 00:00:00 2001 From: unknown Date: Sun, 17 Nov 2002 05:41:42 +0500 Subject: [PATCH 2/3] Move MI_SORT_PARAM and BUFFPEK to myisamdef.h, delete KEY_STORE, change functions of store keys to temp. file include/myisam.h: move MI_SORT_PARAM to myisamdef.h myisam/myisamdef.h: Move MI_SORT_PARAM and BUFFPEK. Change MI_SORT_PARAM myisam/sort.c: Delete KEY_STORE, change functions of store keys to temp. file --- include/myisam.h | 27 -------- myisam/myisamdef.h | 37 +++++++++++ myisam/sort.c | 151 ++++++++++++++++++++------------------------- 3 files changed, 104 insertions(+), 111 deletions(-) diff --git a/include/myisam.h b/include/myisam.h index acaf8bb7618..74ba0501f9f 100644 --- a/include/myisam.h +++ b/include/myisam.h @@ -360,31 +360,6 @@ typedef struct st_sort_info pthread_cond_t cond; } SORT_INFO; - -typedef struct st_mi_sort_param -{ - pthread_t thr; - IO_CACHE read_cache, tempfile, tempfile_for_exceptions; - DYNAMIC_ARRAY buffpek; - ulonglong unique[MI_MAX_KEY_SEG+1]; - my_off_t pos,max_pos,filepos,start_recpos; - uint key, key_length,real_key_length,sortbuff_size; - uint maxbuffers, keys, find_length, sort_keys_length; - my_bool fix_datafile, master; - MI_KEYDEF *keyinfo; - SORT_INFO *sort_info; - uchar **sort_keys; - byte *rec_buff; - void *wordlist, *wordptr; - char *record; - MY_TMPDIR *tmpdir; - int (*key_cmp)(struct st_mi_sort_param *, const void *, const void *); - int (*key_read)(struct st_mi_sort_param *,void *); - int (*key_write)(struct st_mi_sort_param *, const void *); - void (*lock_in_memory)(MI_CHECK *); -} MI_SORT_PARAM; - - /* functions in mi_check */ void myisamchk_init(MI_CHECK *param); int chk_status(MI_CHECK *param, MI_INFO *info); @@ -415,9 +390,7 @@ int filecopy(MI_CHECK *param, File to,File from,my_off_t start, my_off_t length, const char *type); int movepoint(MI_INFO *info,byte *record,my_off_t oldpos, my_off_t newpos, uint prot_key); -int sort_write_record(MI_SORT_PARAM *sort_param); int write_data_suffix(SORT_INFO *sort_info, my_bool fix_datafile); -int _create_index_by_sort(MI_SORT_PARAM *info,my_bool no_messages, ulong); int test_if_almost_full(MI_INFO *info); int recreate_table(MI_CHECK *param, MI_INFO **org_info, char *filename); void mi_disable_non_unique_index(MI_INFO *info, ha_rows rows); diff --git a/myisam/myisamdef.h b/myisam/myisamdef.h index 6499021861e..5bd63fd8cf6 100644 --- a/myisam/myisamdef.h +++ b/myisam/myisamdef.h @@ -275,7 +275,41 @@ struct st_myisam_info { int rtree_recursion_depth; }; +typedef struct st_buffpek { + my_off_t file_pos; /* Where we are in the sort file */ + uchar *base,*key; /* Key pointers */ + ha_rows count; /* Number of rows in table */ + ulong mem_count; /* numbers of keys in memory */ + ulong max_keys; /* Max keys in buffert */ +} BUFFPEK; +typedef struct st_mi_sort_param +{ + pthread_t thr; + IO_CACHE read_cache, tempfile, tempfile_for_exceptions; + DYNAMIC_ARRAY buffpek; + ulonglong unique[MI_MAX_KEY_SEG+1]; + my_off_t pos,max_pos,filepos,start_recpos; + uint key, key_length,real_key_length,sortbuff_size; + uint maxbuffers, keys, find_length, sort_keys_length; + my_bool fix_datafile, master; + MI_KEYDEF *keyinfo; + SORT_INFO *sort_info; + uchar **sort_keys; + byte *rec_buff; + void *wordlist, *wordptr; + char *record; + MY_TMPDIR *tmpdir; + int (*key_cmp)(struct st_mi_sort_param *, const void *, const void *); + int (*key_read)(struct st_mi_sort_param *,void *); + int (*key_write)(struct st_mi_sort_param *, const void *); + void (*lock_in_memory)(MI_CHECK *); + NEAR int (*write_keys)(struct st_mi_sort_param *, register uchar **, + uint , struct st_buffpek *, IO_CACHE *); + NEAR uint (*read_to_buffer)(IO_CACHE *,struct st_buffpek *, uint); + NEAR int (*write_key)(struct st_mi_sort_param *, IO_CACHE *,char *, + uint, uint); +} MI_SORT_PARAM; /* Some defines used by isam-funktions */ #define USE_WHOLE_KEY MI_MAX_KEY_BUFF*2 /* Use whole key in _mi_search() */ @@ -658,6 +692,9 @@ int thr_write_keys(MI_SORT_PARAM *sort_param); pthread_handler_decl(thr_find_all_keys,arg); #endif +int sort_write_record(MI_SORT_PARAM *sort_param); +int _create_index_by_sort(MI_SORT_PARAM *info,my_bool no_messages, ulong); + #ifdef __cplusplus } #endif diff --git a/myisam/sort.c b/myisam/sort.c index acf375a2bca..7152855a54b 100644 --- a/myisam/sort.c +++ b/myisam/sort.c @@ -39,25 +39,10 @@ #define MYF_RW MYF(MY_NABP | MY_WME | MY_WAIT_IF_FULL) #define DISK_BUFFER_SIZE (IO_SIZE*16) -typedef struct st_buffpek { - my_off_t file_pos; /* Where we are in the sort file */ - uchar *base,*key; /* Key pointers */ - ha_rows count; /* Number of rows in table */ - ulong mem_count; /* numbers of keys in memory */ - ulong max_keys; /* Max keys in buffert */ -} BUFFPEK; - /* Pointers of functions for store and read keys from temp file */ -typedef struct st_key_store { - NEAR int (*write_keys)(struct st_mi_sort_param *, register uchar **, - uint , struct st_buffpek *, IO_CACHE *); - NEAR uint (*read_to_buffer)(IO_CACHE *,struct st_buffpek *, uint); - NEAR int (*write_key)(struct st_mi_sort_param *, IO_CACHE *,char *, - uint, uint); -} KEY_STORE; extern void print_error _VARARGS((const char *fmt,...)); @@ -67,8 +52,7 @@ static ha_rows NEAR_F find_all_keys(MI_SORT_PARAM *info,uint keys, uchar **sort_keys, DYNAMIC_ARRAY *buffpek,int *maxbuffer, IO_CACHE *tempfile, - IO_CACHE *tempfile_for_exceptions, - KEY_STORE *key_store); + IO_CACHE *tempfile_for_exceptions); static int NEAR_F write_keys(MI_SORT_PARAM *info,uchar **sort_keys, uint count, BUFFPEK *buffpek,IO_CACHE *tempfile); static int NEAR_F write_key(MI_SORT_PARAM *info, uchar *key, @@ -78,16 +62,15 @@ static int NEAR_F write_index(MI_SORT_PARAM *info,uchar * *sort_keys, static int NEAR_F merge_many_buff(MI_SORT_PARAM *info,uint keys, uchar * *sort_keys, BUFFPEK *buffpek,int *maxbuffer, - IO_CACHE *t_file,KEY_STORE *key_store); + IO_CACHE *t_file); static uint NEAR_F read_to_buffer(IO_CACHE *fromfile,BUFFPEK *buffpek, uint sort_length); static int NEAR_F merge_buffers(MI_SORT_PARAM *info,uint keys, IO_CACHE *from_file, IO_CACHE *to_file, uchar * *sort_keys, BUFFPEK *lastbuff, - BUFFPEK *Fb, BUFFPEK *Tb, - KEY_STORE *key_store); + BUFFPEK *Fb, BUFFPEK *Tb); static int NEAR_F merge_index(MI_SORT_PARAM *,uint,uchar **,BUFFPEK *, int, - IO_CACHE *,KEY_STORE *); + IO_CACHE *); static int NEAR_F write_keys_varlen(MI_SORT_PARAM *info,uchar **sort_keys, uint count, BUFFPEK *buffpek,IO_CACHE *tempfile); @@ -97,7 +80,7 @@ static int NEAR_F write_merge_key(MI_SORT_PARAM *info, IO_CACHE *to_file,char* k uint sort_length, uint count) ; static int NEAR_F write_merge_key_varlen(MI_SORT_PARAM *info, IO_CACHE *to_file,char* key, uint sort_length, uint count); - +inline int my_var_write(MI_SORT_PARAM *info,IO_CACHE *to_file,char *bufs); /* Creates a index of sorted keys @@ -121,22 +104,20 @@ int _create_index_by_sort(MI_SORT_PARAM *info,my_bool no_messages, ha_rows records; uchar **sort_keys; IO_CACHE tempfile, tempfile_for_exceptions; - KEY_STORE key_store; DBUG_ENTER("_create_index_by_sort"); DBUG_PRINT("enter",("sort_length: %d", info->key_length)); - if ((info->keyinfo->seg->type == HA_KEYTYPE_TEXT)|| - (info->keyinfo->seg->type == HA_KEYTYPE_VARTEXT)) + if (info->keyinfo->flag && HA_VAR_LENGTH_KEY) { - key_store.write_keys=write_keys_varlen; - key_store.read_to_buffer=read_to_buffer_varlen; - key_store.write_key=write_merge_key_varlen; + info->write_keys=write_keys_varlen; + info->read_to_buffer=read_to_buffer_varlen; + info->write_key=write_merge_key_varlen; } else { - key_store.write_keys=write_keys; - key_store.read_to_buffer=read_to_buffer; - key_store.write_key=write_merge_key; + info->write_keys=write_keys; + info->read_to_buffer=read_to_buffer; + info->write_key=write_merge_key; } my_b_clear(&tempfile); @@ -194,7 +175,7 @@ int _create_index_by_sort(MI_SORT_PARAM *info,my_bool no_messages, printf(" - Searching for keys, allocating buffer for %d keys\n",keys); if ((records=find_all_keys(info,keys,sort_keys,&buffpek,&maxbuffer, - &tempfile,&tempfile_for_exceptions,&key_store)) + &tempfile,&tempfile_for_exceptions)) == HA_POS_ERROR) goto err; /* purecov: tested */ if (maxbuffer == 0) @@ -212,7 +193,7 @@ int _create_index_by_sort(MI_SORT_PARAM *info,my_bool no_messages, if (!no_messages) printf(" - Merging %lu keys\n",records); /* purecov: tested */ if (merge_many_buff(info,keys,sort_keys, - dynamic_element(&buffpek,0,BUFFPEK *),&maxbuffer,&tempfile,&key_store)) + dynamic_element(&buffpek,0,BUFFPEK *),&maxbuffer,&tempfile)) goto err; /* purecov: inspected */ } if (flush_io_cache(&tempfile) || @@ -221,7 +202,7 @@ int _create_index_by_sort(MI_SORT_PARAM *info,my_bool no_messages, if (!no_messages) puts(" - Last merge and dumping keys\n"); /* purecov: tested */ if (merge_index(info,keys,sort_keys,dynamic_element(&buffpek,0,BUFFPEK *), - maxbuffer,&tempfile,&key_store)) + maxbuffer,&tempfile)) goto err; /* purecov: inspected */ } @@ -266,8 +247,7 @@ err: static ha_rows NEAR_F find_all_keys(MI_SORT_PARAM *info, uint keys, uchar **sort_keys, DYNAMIC_ARRAY *buffpek, int *maxbuffer, IO_CACHE *tempfile, - IO_CACHE *tempfile_for_exceptions, - KEY_STORE *key_store) + IO_CACHE *tempfile_for_exceptions) { int error; uint idx; @@ -287,7 +267,7 @@ static ha_rows NEAR_F find_all_keys(MI_SORT_PARAM *info, uint keys, if (++idx == keys) { - if (key_store->write_keys(info,sort_keys,idx-1,(BUFFPEK *)alloc_dynamic(buffpek), + if (info->write_keys(info,sort_keys,idx-1,(BUFFPEK *)alloc_dynamic(buffpek), tempfile)) DBUG_RETURN(HA_POS_ERROR); /* purecov: inspected */ @@ -301,7 +281,7 @@ static ha_rows NEAR_F find_all_keys(MI_SORT_PARAM *info, uint keys, DBUG_RETURN(HA_POS_ERROR); /* Aborted by get_key */ /* purecov: inspected */ if (buffpek->elements) { - if (key_store->write_keys(info,sort_keys,idx,(BUFFPEK *)alloc_dynamic(buffpek), + if (info->write_keys(info,sort_keys,idx,(BUFFPEK *)alloc_dynamic(buffpek), tempfile)) DBUG_RETURN(HA_POS_ERROR); /* purecov: inspected */ *maxbuffer=buffpek->elements-1; @@ -322,7 +302,6 @@ pthread_handler_decl(thr_find_all_keys,arg) uint memavl,old_memavl,keys,sort_length; uint idx, maxbuffer; uchar **sort_keys=0; - KEY_STORE key_store; error=1; @@ -331,18 +310,17 @@ pthread_handler_decl(thr_find_all_keys,arg) if (info->sort_info->got_error) goto err; - if ((info->keyinfo->seg->type == HA_KEYTYPE_TEXT)|| - (info->keyinfo->seg->type == HA_KEYTYPE_VARTEXT)) + if (info->keyinfo->flag && HA_VAR_LENGTH_KEY) { - key_store.write_keys=write_keys_varlen; - key_store.read_to_buffer=read_to_buffer_varlen; - key_store.write_key=write_merge_key_varlen; + info->write_keys=write_keys_varlen; + info->read_to_buffer=read_to_buffer_varlen; + info->write_key=write_merge_key_varlen; } else { - key_store.write_keys=write_keys; - key_store.read_to_buffer=read_to_buffer; - key_store.write_key=write_merge_key; + info->write_keys=write_keys; + info->read_to_buffer=read_to_buffer; + info->write_key=write_merge_key; } my_b_clear(&info->tempfile); @@ -418,7 +396,7 @@ pthread_handler_decl(thr_find_all_keys,arg) if (++idx == keys) { - if (key_store.write_keys(info,sort_keys,idx-1, + if (info->write_keys(info,sort_keys,idx-1, (BUFFPEK *)alloc_dynamic(&info->buffpek), &info->tempfile)) goto err; @@ -432,7 +410,7 @@ pthread_handler_decl(thr_find_all_keys,arg) goto err; if (info->buffpek.elements) { - if (key_store.write_keys(info,sort_keys, idx, + if (info->write_keys(info,sort_keys, idx, (BUFFPEK *) alloc_dynamic(&info->buffpek), &info->tempfile)) goto err; info->keys=(info->buffpek.elements-1)*(keys-1)+idx; @@ -474,7 +452,6 @@ int thr_write_keys(MI_SORT_PARAM *sort_param) MI_INFO *info=sort_info->info; MYISAM_SHARE *share=info->s; MI_SORT_PARAM *sinfo; - KEY_STORE key_store; byte *mergebuf=0; LINT_INIT(length); @@ -518,18 +495,17 @@ int thr_write_keys(MI_SORT_PARAM *sort_param) { if (got_error) continue; - if ((sinfo->keyinfo->seg->type == HA_KEYTYPE_TEXT)|| - (sinfo->keyinfo->seg->type == HA_KEYTYPE_VARTEXT)) + if (sinfo->keyinfo->flag && HA_VAR_LENGTH_KEY) { - key_store.write_keys=write_keys_varlen; - key_store.read_to_buffer=read_to_buffer_varlen; - key_store.write_key=write_merge_key_varlen; + sinfo->write_keys=write_keys_varlen; + sinfo->read_to_buffer=read_to_buffer_varlen; + sinfo->write_key=write_merge_key_varlen; } else { - key_store.write_keys=write_keys; - key_store.read_to_buffer=read_to_buffer; - key_store.write_key=write_merge_key; + sinfo->write_keys=write_keys; + sinfo->read_to_buffer=read_to_buffer; + sinfo->write_key=write_merge_key; } if (sinfo->buffpek.elements) { @@ -555,7 +531,7 @@ int thr_write_keys(MI_SORT_PARAM *sort_param) printf("Key %d - Merging %u keys\n",sinfo->key+1, sinfo->keys); if (merge_many_buff(sinfo, keys, (uchar **)mergebuf, dynamic_element(&sinfo->buffpek, 0, BUFFPEK *), - &maxbuffer, &sinfo->tempfile,&key_store)) + &maxbuffer, &sinfo->tempfile)) { got_error=1; continue; @@ -571,7 +547,7 @@ int thr_write_keys(MI_SORT_PARAM *sort_param) printf("Key %d - Last merge and dumping keys\n", sinfo->key+1); if (merge_index(sinfo, keys, (uchar **)mergebuf, dynamic_element(&sinfo->buffpek,0,BUFFPEK *), - maxbuffer,&sinfo->tempfile,&key_store) || + maxbuffer,&sinfo->tempfile) || flush_pending_blocks(sinfo)) { got_error=1; @@ -635,10 +611,24 @@ static int NEAR_F write_keys(MI_SORT_PARAM *info, register uchar **sort_keys, DBUG_RETURN(0); } /* write_keys */ +inline int my_var_write(MI_SORT_PARAM *info,IO_CACHE *to_file,char *bufs) +{ + int err; + uint16 len = _mi_keylength(info->keyinfo,bufs); + + if (err=my_b_write(to_file,(byte*)&len,sizeof(len))) + return(err); + if (err=my_b_write(to_file,(byte*)bufs,(uint) len)) + return(err); + return(0); +} + + static int NEAR_F write_keys_varlen(MI_SORT_PARAM *info, register uchar **sort_keys, uint count, BUFFPEK *buffpek, IO_CACHE *tempfile) { uchar **end; + int err; DBUG_ENTER("write_keys_varlen"); qsort2((byte*) sort_keys,count,sizeof(byte*),(qsort2_cmp) info->key_cmp, @@ -652,11 +642,8 @@ static int NEAR_F write_keys_varlen(MI_SORT_PARAM *info, register uchar **sort_k buffpek->count=count; for (end=sort_keys+count ; sort_keys != end ; sort_keys++) { - uint16 len = _mi_keylength(info->keyinfo,*sort_keys); - if (my_b_write(tempfile,(byte*)&len,sizeof(len))) - DBUG_RETURN(1); - if (my_b_write(tempfile,(byte*) *sort_keys,(uint) len)) - DBUG_RETURN(1); + if (err=my_var_write(info,tempfile,*sort_keys)) + DBUG_RETURN(err); } DBUG_RETURN(0); } /* write_keys_varlen */ @@ -702,8 +689,7 @@ static int NEAR_F write_index(MI_SORT_PARAM *info, register uchar **sort_keys, static int NEAR_F merge_many_buff(MI_SORT_PARAM *info, uint keys, uchar **sort_keys, BUFFPEK *buffpek, - int *maxbuffer, IO_CACHE *t_file, - KEY_STORE *key_store) + int *maxbuffer, IO_CACHE *t_file) { register int i; IO_CACHE t_file2, *from_file, *to_file, *temp; @@ -726,11 +712,11 @@ static int NEAR_F merge_many_buff(MI_SORT_PARAM *info, uint keys, for (i=0 ; i <= *maxbuffer-MERGEBUFF*3/2 ; i+=MERGEBUFF) { if (merge_buffers(info,keys,from_file,to_file,sort_keys,lastbuff++, - buffpek+i,buffpek+i+MERGEBUFF-1,key_store)) + buffpek+i,buffpek+i+MERGEBUFF-1)) break; /* purecov: inspected */ } if (merge_buffers(info,keys,from_file,to_file,sort_keys,lastbuff++, - buffpek+i,buffpek+ *maxbuffer,key_store)) + buffpek+i,buffpek+ *maxbuffer)) break; /* purecov: inspected */ if (flush_io_cache(to_file)) break; /* purecov: inspected */ @@ -813,16 +799,13 @@ static int NEAR_F write_merge_key_varlen(MI_SORT_PARAM *info, IO_CACHE *to_file, uint sort_length, uint count) { uint idx; - int err; - uchar *bufs = key; + char *bufs = key; for (idx=1;idx<=count;idx++) { - uint16 len = _mi_keylength(info->keyinfo,bufs); - if (err=my_b_write(to_file,(byte*)&len,sizeof(len))) - return(err); - if (err=my_b_write(to_file,(byte*)bufs,(uint) len)) - return(err); + int err; + if (err = my_var_write(info,to_file,bufs)) + return(err); bufs=bufs+sort_length; } return(0); @@ -842,7 +825,7 @@ static int NEAR_F write_merge_key(MI_SORT_PARAM *info, IO_CACHE *to_file,char* k static int NEAR_F merge_buffers(MI_SORT_PARAM *info, uint keys, IO_CACHE *from_file, IO_CACHE *to_file, uchar **sort_keys, BUFFPEK *lastbuff, - BUFFPEK *Fb, BUFFPEK *Tb, KEY_STORE *key_store) + BUFFPEK *Fb, BUFFPEK *Tb) { int error; uint sort_length,maxcount; @@ -872,7 +855,7 @@ merge_buffers(MI_SORT_PARAM *info, uint keys, IO_CACHE *from_file, count+= buffpek->count; buffpek->base= strpos; buffpek->max_keys=maxcount; - strpos+= (uint) (error=(int) key_store->read_to_buffer(from_file,buffpek,sort_length)); + strpos+= (uint) (error=(int) info->read_to_buffer(from_file,buffpek,sort_length)); if (error == -1) goto err; /* purecov: inspected */ queue_insert(&queue,(char*) buffpek); @@ -889,7 +872,7 @@ merge_buffers(MI_SORT_PARAM *info, uint keys, IO_CACHE *from_file, buffpek=(BUFFPEK*) queue_top(&queue); if (to_file) { - if (key_store->write_key(info,to_file,(byte*) buffpek->key,(uint) sort_length,1)) + if (info->write_key(info,to_file,(byte*) buffpek->key,(uint) sort_length,1)) { error=1; goto err; /* purecov: inspected */ } @@ -904,7 +887,7 @@ merge_buffers(MI_SORT_PARAM *info, uint keys, IO_CACHE *from_file, buffpek->key+=sort_length; if (! --buffpek->mem_count) { - if (!(error=(int) key_store->read_to_buffer(from_file,buffpek,sort_length))) + if (!(error=(int) info->read_to_buffer(from_file,buffpek,sort_length))) { uchar *base=buffpek->base; uint max_keys=buffpek->max_keys; @@ -944,7 +927,7 @@ merge_buffers(MI_SORT_PARAM *info, uint keys, IO_CACHE *from_file, { if (to_file) { - if (key_store->write_key(info,to_file,(byte*) buffpek->key, + if (info->write_key(info,to_file,(byte*) buffpek->key, sort_length,buffpek->mem_count)) { error=1; goto err; /* purecov: inspected */ @@ -965,7 +948,7 @@ merge_buffers(MI_SORT_PARAM *info, uint keys, IO_CACHE *from_file, } } } - while ((error=(int) key_store->read_to_buffer(from_file,buffpek,sort_length)) != -1 && + while ((error=(int) info->read_to_buffer(from_file,buffpek,sort_length)) != -1 && error != 0); lastbuff->count=count; @@ -981,11 +964,11 @@ err: static int NEAR_F merge_index(MI_SORT_PARAM *info, uint keys, uchar **sort_keys, - BUFFPEK *buffpek, int maxbuffer, IO_CACHE *tempfile,KEY_STORE *key_store) + BUFFPEK *buffpek, int maxbuffer, IO_CACHE *tempfile) { DBUG_ENTER("merge_index"); if (merge_buffers(info,keys,tempfile,(IO_CACHE*) 0,sort_keys,buffpek,buffpek, - buffpek+maxbuffer,key_store)) + buffpek+maxbuffer)) DBUG_RETURN(1); /* purecov: inspected */ DBUG_RETURN(0); } /* merge_index */ From 0c349665959e300485b7f64119cb20bdce5cd189 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 28 Nov 2002 15:16:56 +0500 Subject: [PATCH 3/3] Correct view of sort.c and mi_too_big_key_for_sort() myisam/mi_check.c: Correct mi_too_big_key_for_sort() myisam/sort.c: Correct view, according rules --- myisam/mi_check.c | 5 +---- myisam/sort.c | 13 +++++++------ 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/myisam/mi_check.c b/myisam/mi_check.c index ba90b2c5bcf..d3222a770a8 100644 --- a/myisam/mi_check.c +++ b/myisam/mi_check.c @@ -3682,10 +3682,7 @@ static my_bool mi_too_big_key_for_sort(MI_KEYDEF *key, ha_rows rows) key_maxlength+=ft_max_word_len_for_sort-HA_FT_MAXLEN; return (key->flag & (HA_BINARY_PACK_KEY | HA_VAR_LENGTH_KEY | HA_FULLTEXT) && ((ulonglong) rows * key_maxlength > - (ulonglong) myisam_max_temp_length || - (ulonglong) rows * (key_maxlength - key->minlength) / 2 > - myisam_max_extra_temp_length || - (rows == 0 && (key_maxlength / key->minlength) > 2))); + (ulonglong) myisam_max_temp_length)); } diff --git a/myisam/sort.c b/myisam/sort.c index 7152855a54b..24a76f11823 100644 --- a/myisam/sort.c +++ b/myisam/sort.c @@ -73,13 +73,14 @@ static int NEAR_F merge_index(MI_SORT_PARAM *,uint,uchar **,BUFFPEK *, int, IO_CACHE *); static int NEAR_F write_keys_varlen(MI_SORT_PARAM *info,uchar **sort_keys, - uint count, BUFFPEK *buffpek,IO_CACHE *tempfile); + uint count, BUFFPEK *buffpek, + IO_CACHE *tempfile); static uint NEAR_F read_to_buffer_varlen(IO_CACHE *fromfile,BUFFPEK *buffpek, uint sort_length); -static int NEAR_F write_merge_key(MI_SORT_PARAM *info, IO_CACHE *to_file,char* key, - uint sort_length, uint count) ; -static int NEAR_F write_merge_key_varlen(MI_SORT_PARAM *info, IO_CACHE *to_file,char* key, - uint sort_length, uint count); +static int NEAR_F write_merge_key(MI_SORT_PARAM *info, IO_CACHE *to_file, + char* key, uint sort_length, uint count); +static int NEAR_F write_merge_key_varlen(MI_SORT_PARAM *info, IO_CACHE *to_file, + char* key, uint sort_length, uint count); inline int my_var_write(MI_SORT_PARAM *info,IO_CACHE *to_file,char *bufs); /* Creates a index of sorted keys @@ -786,7 +787,7 @@ static uint NEAR_F read_to_buffer_varlen(IO_CACHE *fromfile, BUFFPEK *buffpek, return((uint) -1); buffpek->file_pos+=length_of_key; buffp = buffp + sort_length; - } + } buffpek->key=buffpek->base; buffpek->count-= count; buffpek->mem_count= count;