From 4bd32cae4c3bcea00ec849ae09f163f13ef57dca Mon Sep 17 00:00:00 2001 From: "lenz@mysql.com" <> Date: Tue, 27 May 2003 18:47:50 +0200 Subject: [PATCH 01/40] - removed internals.texi (has been moved to mysqldoc BK tree) --- Docs/Makefile.am | 47 - Docs/internals.texi | 5707 ------------------------------------------- 2 files changed, 5754 deletions(-) delete mode 100644 Docs/internals.texi diff --git a/Docs/Makefile.am b/Docs/Makefile.am index 00eb936c408..f3df055a7dd 100644 --- a/Docs/Makefile.am +++ b/Docs/Makefile.am @@ -157,53 +157,6 @@ manual_letter.de.ps: manual.de.texi include.texi touch $@ -# -# Internals Manual -# - -# GNU Info -internals.info: internals.texi include.texi - cd $(srcdir) && $(MAKEINFO) --no-split -I $(srcdir) $< - -# Plain Text -internals.txt: internals.texi include.texi - cd $(srcdir) && \ - $(MAKEINFO) -I $(srcdir) --no-headers --no-split --output $@ $< - -# HTML, all in one file -internals.html: internals.texi include.texi $(srcdir)/Support/texi2html - cd $(srcdir) && @PERL@ $(srcdir)/Support/texi2html $(TEXI2HTML_FLAGS) $< -internals_toc.html: internals.html - -# PDF, Portable Document Format -internals.pdf: internals.texi - sed -e 's|@image{[^}]*} *||g' <$< >internals-tmp.texi - pdftex --interaction=nonstopmode internals-tmp.texi - texindex internals-tmp.?? - pdftex --interaction=nonstopmode internals-tmp.texi - texindex internals-tmp.?? - pdftex --interaction=nonstopmode internals-tmp.texi - mv internals-tmp.pdf $@ - rm -f internals-tmp.* - touch $@ - -# Postscript, A4 Paper -internals_a4.ps: internals.texi include.texi - TEXINPUTS=$(srcdir):$$TEXINPUTS \ - MAKEINFO='$(MAKEINFO) -I $(srcdir)' \ - $(TEXI2DVI) --batch --texinfo --quiet '@afourpaper' $< - $(DVIPS) -t a4 internals.dvi -o $@ - touch $@ - -# Postscript, US Letter Paper -internals_letter.ps: internals.texi include.texi - TEXINPUTS=$(srcdir):$$TEXINPUTS \ - MAKEINFO='$(MAKEINFO) -I $(srcdir)' \ - $(TEXI2DVI) --batch $< - $(DVIPS) -t letter internals.dvi -o $@ - touch $@ - - # # Miscellaneous # diff --git a/Docs/internals.texi b/Docs/internals.texi deleted file mode 100644 index a54f5098e5d..00000000000 --- a/Docs/internals.texi +++ /dev/null @@ -1,5707 +0,0 @@ -\input texinfo @c -*-texinfo-*- -@c Copyright 2002 MySQL AB -@c -@c %**start of header -@setfilename internals.info - -@c We want the types in the same index -@synindex cp fn - -@iftex -@afourpaper -@end iftex - -@c Get version and other info -@include include.texi - -@ifclear tex-debug -@c This removes the black squares in the right margin -@finalout -@end ifclear - -@c Set background for HTML -@set _body_tags BGCOLOR=#FFFFFF TEXT=#000000 LINK=#101090 VLINK=#7030B0 -@settitle @strong{MySQL} Internals Manual for version @value{mysql_version}. -@setchapternewpage odd -@paragraphindent 0 - -@c %**end of header - -@ifinfo -@format -START-INFO-DIR-ENTRY -* mysql-internals: (mysql-internals). @strong{MySQL} internals. -END-INFO-DIR-ENTRY -@end format -@end ifinfo - -@titlepage -@sp 10 -@center @titlefont{@strong{MySQL} Internals Manual} -@sp 10 -@center Copyright @copyright{} 1998-2002 MySQL AB -@page -@end titlepage - -@node Top, coding guidelines, (dir), (dir) - -@ifinfo -This is a manual about @strong{MySQL} internals. -@end ifinfo - -@menu -* coding guidelines:: Coding Guidelines -* caching:: How MySQL Handles Caching -* join_buffer_size:: -* flush tables:: How MySQL Handles @code{FLUSH TABLES} -* Algorithms:: -* mysys functions:: Functions In The @code{mysys} Library -* DBUG:: DBUG Tags To Use -* protocol:: MySQL Client/Server Protocol -* Fulltext Search:: Fulltext Search in MySQL -* MyISAM Record Structure:: MyISAM Record Structure -* InnoDB Record Structure:: InnoDB Record Structure -* InnoDB Page Structure:: InnoDB Page Structure -* Files in MySQL Sources:: Annotated List Of Files in the MySQL Source Code Distribution -* Files in InnoDB Sources:: Annotated List Of Files in the InnoDB Source Code Distribution -@end menu - - -@node coding guidelines, caching, Top, Top -@chapter Coding Guidelines - -@itemize @bullet - -@item -We use @uref{http://www.bitkeeper.com/, BitKeeper} for source management. - -@item -You should use the @strong{MySQL} 4.1 source for all developments. - -@item -If you have any questions about the @strong{MySQL} source, you can post these -to @email{internals@@mysql.com} and we will answer them. - -@item -Try to write code in a lot of black boxes that can be reused or use at -least a clean, easy to change interface. - -@item -Reuse code; There is already a lot of algorithms in MySQL for list handling, -queues, dynamic and hashed arrays, sorting, etc. that can be reused. - -@item -Use the @code{my_*} functions like @code{my_read()}/@code{my_write()}/ -@code{my_malloc()} that you can find in the @code{mysys} library instead -of the direct system calls; This will make your code easier to debug and -more portable. - -@item -Try to always write optimized code, so that you don't have to -go back and rewrite it a couple of months later. It's better to -spend 3 times as much time designing and writing an optimal function than -having to do it all over again later on. - -@item -Avoid CPU wasteful code, even where it does not matter, so that -you will not develop sloppy coding habits. - -@item -If you can write it in fewer lines, do it (as long as the code will not -be slower or much harder to read). - -@item -Don't use two commands on the same line. - -@item -Do not check the same pointer for @code{NULL} more than once. - -@item -Use long function and variable names in English. This makes your code -easier to read. - -@item -Use @code{my_var} as opposed to @code{myVar} or @code{MyVar} (@samp{_} -rather than dancing SHIFT to seperate words in identifiers). - -@item -Think assembly - make it easier for the compiler to optimize your code. - -@item -Comment your code when you do something that someone else may think -is not ``trivial''. - -@item -Use @code{libstring} functions (in the @file{strings} directory) -instead of standard @code{libc} string functions whenever possible. - -@item -Avoid using @code{malloc()} (its REAL slow); For memory allocations -that only need to live for the lifetime of one thread, one should use -@code{sql_alloc()} instead. - -@item -Before making big design decisions, please first post a summary of -what you want to do, why you want to do it, and how you plan to do -it. This way we can easily provide you with feedback and also -easily discuss it thoroughly if some other developer thinks there is better -way to do the same thing! - -@item -Class names start with a capital letter. - -@item -Structure types are @code{typedef}'ed to an all-caps identifier. - -@item -Any @code{#define}'s are in all-caps. - -@item -Matching @samp{@{} are in the same column. - -@item -Put the @samp{@{} after a @code{switch} on the same line, as this gives -better overall indentation for the switch statement: - -@example -switch (arg) @{ -@end example - -@item -In all other cases, @samp{@{} and @samp{@}} should be on their own line, except -if there is nothing inside @samp{@{} and @samp{@}}. - -@item -Have a space after @code{if} - -@item -Put a space after @samp{,} for function arguments - -@item -Functions return @samp{0} on success, and non-zero on error, so you can do: - -@example -if(a() || b() || c()) @{ error("something went wrong"); @} -@end example - -@item -Using @code{goto} is okay if not abused. - -@item -Avoid default variable initalizations, use @code{LINT_INIT()} if the -compiler complains after making sure that there is really no way -the variable can be used uninitialized. - -@item -Do not instantiate a class if you do not have to. - -@item -Use pointers rather than array indexing when operating on strings. - -@end itemize - -Suggested mode in emacs: - -@example -(load "cc-mode") -(setq c-mode-common-hook '(lambda () - (turn-on-font-lock) - (setq comment-column 48))) -(setq c-style-alist - (cons - '("MY" - (c-basic-offset . 2) - (c-comment-only-line-offset . 0) - (c-offsets-alist . ((statement-block-intro . +) - (knr-argdecl-intro . 0) - (substatement-open . 0) - (label . -) - (statement-cont . +) - (arglist-intro . c-lineup-arglist-intro-after-paren) - (arglist-close . c-lineup-arglist) - )) - ) - c-style-alist)) -(c-set-style "MY") -(setq c-default-style "MY") -@end example - -@node caching, join_buffer_size, coding guidelines, Top -@chapter How MySQL Handles Caching - -@strong{MySQL} has the following caches: -(Note that the some of the filename have a wrong spelling of cache. :) - -@table @strong - -@item Key Cache -A shared cache for all B-tree index blocks in the different NISAM -files. Uses hashing and reverse linked lists for quick caching of the -last used blocks and quick flushing of changed entries for a specific -table. (@file{mysys/mf_keycash.c}) - -@item Record Cache -This is used for quick scanning of all records in a table. -(@file{mysys/mf_iocash.c} and @file{isam/_cash.c}) - -@item Table Cache -This holds the last used tables. (@file{sql/sql_base.cc}) - -@item Hostname Cache -For quick lookup (with reverse name resolving). Is a must when one has a -slow DNS. -(@file{sql/hostname.cc}) - -@item Privilege Cache -To allow quick change between databases the last used privileges are -cached for each user/database combination. -(@file{sql/sql_acl.cc}) - -@item Heap Table Cache -Many use of @code{GROUP BY} or @code{DISTINCT} caches all found rows in -a @code{HEAP} table. (This is a very quick in-memory table with hash index.) - -@item Join buffer Cache -For every full join in a @code{SELECT} statement (a full join here means -there were no keys that one could use to find the next table in a list), -the found rows are cached in a join cache. One @code{SELECT} query can -use many join caches in the worst case. -@end table - -@node join_buffer_size, flush tables, caching, Top -@chapter How MySQL uses the join_buffer cache - -Basic information about @code{join_buffer_size}: - -@itemize @bullet -@item -It's only used in the case when join type is of type @code{ALL} or -@code{index}; In other words: no possible keys can be used. -@item -A join buffer is never allocated for the first not-const table, -even it it would be of type @code{ALL}/@code{index}. -@item -The buffer is allocated when we need to do a each full join between two -tables and freed after the query is done. -@item -Accepted row combinations of tables before the @code{ALL}/@code{index} -able is stored in the cache and is used to compare against each read -row in the @code{ALL} table. -@item -We only store the used fields in the join_buffer cache, not the -whole rows. -@end itemize - -Assume you have the following join: - -@example -Table name Type -t1 range -t2 ref -t3 @code{ALL} -@end example - -The join is then done as follows: - -@example -- While rows in t1 matching range - - Read through all rows in t2 according to reference key - - Store used fields form t1,t2 in cache - - If cache is full - - Read through all rows in t3 - - Compare t3 row against all t1,t2 combination in cache - - If rows satisfying join condition, send it to client - - Empty cache - -- Read through all rows in t3 - - Compare t3 row against all stored t1,t2 combinations in cache - - If rows satisfying join condition, send it to client -@end example - -The above means that table t3 is scanned - -@example -(size-of-stored-row(t1,t2) * accepted-row-cominations(t1,t2))/ -join_buffer_size+1 -@end example -times. - -Some conclusions: - -@itemize @bullet -@item -The larger the join_buff_size, the fewer scans of t3. -If @code{join_buff_size} is already large enough to hold all previous row -combinations then there is no speed to gain by making it bigger. -@item -If there is several tables of @code{ALL}/@code{index} then the we -allocate one @code{join_buffer_size buffer} for each of them and use the -same algorithm described above to handle it. (In other words, we store -the same row combination several times into different buffers) -@end itemize - -@node flush tables, Algorithms, join_buffer_size, Top -@chapter How MySQL Handles @code{FLUSH TABLES} - -@itemize @bullet - -@item -Flush tables is handled in @file{sql/sql_base.cc::close_cached_tables()}. - -@item -The idea of flush tables is to force all tables to be closed. This -is mainly to ensure that if someone adds a new table outside of -@strong{MySQL} (for example with @code{cp}) all threads will start using -the new table. This will also ensure that all table changes are flushed -to disk (but of course not as optimally as simple calling a sync on -all tables)! - -@item -When one does a @code{FLUSH TABLES}, the variable @code{refresh_version} -will be incremented. Every time a thread releases a table it checks if -the refresh version of the table (updated at open) is the same as -the current @code{refresh_version}. If not it will close it and broadcast -a signal on @code{COND_refresh} (to wait any thread that is waiting for -all instanses of a table to be closed). - -@item -The current @code{refresh_version} is also compared to the open -@code{refresh_version} after a thread gets a lock on a table. If the -refresh version is different the thread will free all locks, reopen the -table and try to get the locks again; This is just to quickly get all -tables to use the newest version. This is handled by -@file{sql/lock.cc::mysql_lock_tables()} and -@file{sql/sql_base.cc::wait_for_tables()}. - -@item -When all tables has been closed @code{FLUSH TABLES} will return an ok -to client. - -@item -If the thread that is doing @code{FLUSH TABLES} has a lock on some tables, -it will first close the locked tables, then wait until all other threads -have also closed them, and then reopen them and get the locks. -After this it will give other threads a chance to open the same tables. - -@end itemize - -@node Algorithms, mysys functions, flush tables, Top -@chapter Different algoritms used in MySQL - -MySQL uses a lot of different algorithms. This chapter tries to describe -some of these: - -@menu -* filesort:: -* bulk-insert:: -@end menu - -@node filesort, bulk-insert, Algorithms, Algorithms -@section How MySQL Does Sorting (@code{filesort}) - -@itemize @bullet - -@item -Read all rows according to key or by table scanning. - -@item -Store the sort-key in a buffer (@code{sort_buffer}). - -@item -When the buffer gets full, run a @code{qsort} on it and store the result -in a temporary file. Save a pointer to the sorted block. - -@item -Repeat the above until all rows have been read. - -@item -Repeat the following until there is less than @code{MERGEBUFF2} (15) -blocks left. - -@item -Do a multi-merge of up to @code{MERGEBUFF} (7) regions to one block in -another temporary file. Repeat until all blocks from the first file -are in the second file. - -@item -On the last multi-merge, only the pointer to the row (last part of -the sort-key) is written to a result file. - -@item -Now the code in @file{sql/records.cc} will be used to read through them -in sorted order by using the row pointers in the result file. -To optimize this, we read in a big block of row pointers, sort these -and then we read the rows in the sorted order into a row buffer -(@code{record_buffer}). - -@end itemize - -@node bulk-insert, , filesort, Algorithms -@section Bulk insert - -Logic behind bulk insert optimisation is simple. - -Instead of writing each key value to b-tree (that is to keycache, but -bulk insert code doesn't know about keycache) keys are stored in -balanced binary (red-black) tree, in memory. When this tree reaches its -memory limit it's writes all keys to disk (to keycache, that is). But -as key stream coming from the binary tree is already sorted inserting -goes much faster, all the necessary pages are already in cache, disk -access is minimized, etc. - -@node mysys functions, DBUG, Algorithms, Top -@chapter Functions In The @code{mysys} Library - -Functions in @code{mysys}: (For flags see @file{my_sys.h}) - -@table @code -@item int my_copy _A((const char *from, const char *to, myf MyFlags)); -Copy file from @code{from} to @code{to}. - -@item int my_delete _A((const char *name, myf MyFlags)); -Delete file @code{name}. - -@item int my_getwd _A((string buf, uint size, myf MyFlags)); -@item int my_setwd _A((const char *dir, myf MyFlags)); -Get and set working directory. - -@item string my_tempnam _A((const char *pfx, myf MyFlags)); -Make a unique temporary file name by using dir and adding something after -@code{pfx} to make name unique. The file name is made by adding a unique -six character string and @code{TMP_EXT} after @code{pfx}. -Returns pointer to @code{malloc()}'ed area for filename. Should be freed by -@code{free()}. - -@item File my_open _A((const char *FileName,int Flags,myf MyFlags)); -@item File my_create _A((const char *FileName, int CreateFlags, int AccsesFlags, myf MyFlags)); -@item int my_close _A((File Filedes, myf MyFlags)); -@item uint my_read _A((File Filedes, byte *Buffer, uint Count, myf MyFlags)); -@item uint my_write _A((File Filedes, const byte *Buffer, uint Count, myf MyFlags)); -@item ulong my_seek _A((File fd,ulong pos,int whence,myf MyFlags)); -@item ulong my_tell _A((File fd,myf MyFlags)); -Use instead of open, open-with-create-flag, close, read, and write -to get automatic error messages (flag @code{MYF_WME}) and only have -to test for != 0 if error (flag @code{MY_NABP}). - -@item int my_rename _A((const char *from, const char *to, myf MyFlags)); -Rename file from @code{from} to @code{to}. - -@item FILE *my_fopen _A((const char *FileName,int Flags,myf MyFlags)); -@item FILE *my_fdopen _A((File Filedes,int Flags,myf MyFlags)); -@item int my_fclose _A((FILE *fd,myf MyFlags)); -@item uint my_fread _A((FILE *stream,byte *Buffer,uint Count,myf MyFlags)); -@item uint my_fwrite _A((FILE *stream,const byte *Buffer,uint Count, myf MyFlags)); -@item ulong my_fseek _A((FILE *stream,ulong pos,int whence,myf MyFlags)); -@item ulong my_ftell _A((FILE *stream,myf MyFlags)); -Same read-interface for streams as for files. - -@item gptr _mymalloc _A((uint uSize,const char *sFile,uint uLine, myf MyFlag)); -@item gptr _myrealloc _A((string pPtr,uint uSize,const char *sFile,uint uLine, myf MyFlag)); -@item void _myfree _A((gptr pPtr,const char *sFile,uint uLine)); -@item int _sanity _A((const char *sFile,unsigned int uLine)); -@item gptr _myget_copy_of_memory _A((const byte *from,uint length,const char *sFile, uint uLine,myf MyFlag)); -@code{malloc(size,myflag)} is mapped to these functions if not compiled -with @code{-DSAFEMALLOC}. - -@item void TERMINATE _A((void)); -Writes @code{malloc()} info on @code{stdout} if compiled with -@code{-DSAFEMALLOC}. - -@item int my_chsize _A((File fd, ulong newlength, myf MyFlags)); -Change size of file @code{fd} to @code{newlength}. - -@item void my_error _D((int nr, myf MyFlags, ...)); -Writes message using error number (see @file{mysys/errors.h}) on @code{stdout}, -or using curses, if @code{MYSYS_PROGRAM_USES_CURSES()} has been called. - -@item void my_message _A((const char *str, myf MyFlags)); -Writes @code{str} on @code{stdout}, or using curses, if -@code{MYSYS_PROGRAM_USES_CURSES()} has been called. - -@item void my_init _A((void )); -Start each program (in @code{main()}) with this. - -@item void my_end _A((int infoflag)); -Gives info about program. -If @code{infoflag & MY_CHECK_ERROR}, prints if some files are left open. -If @code{infoflag & MY_GIVE_INFO}, prints timing info and malloc info -about program. - -@item int my_redel _A((const char *from, const char *to, int MyFlags)); -Delete @code{from} before rename of @code{to} to @code{from}. Copies state -from old file to new file. If @code{MY_COPY_TIME} is set, sets old time. - -@item int my_copystat _A((const char *from, const char *to, int MyFlags)); -Copy state from old file to new file. If @code{MY_COPY_TIME} is set, -sets old time. - -@item string my_filename _A((File fd)); -Returns filename of open file. - -@item int dirname _A((string to, const char *name)); -Copy name of directory from filename. - -@item int test_if_hard_path _A((const char *dir_name)); -Test if @code{dir_name} is a hard path (starts from root). - -@item void convert_dirname _A((string name)); -Convert dirname according to system. -In MSDOS, changes all characters to capitals and changes @samp{/} to @samp{\}. - -@item string fn_ext _A((const char *name)); -Returns pointer to extension in filename. - -@item string fn_format _A((string to,const char *name,const char *dsk,const char *form,int flag)); - format a filename with replace of library and extension and - converts between different systems. - params to and name may be identicall - function dosn't change name if name != to - Flag may be: 1 force replace filnames library with 'dsk' - 2 force replace extension with 'form' */ - 4 force Unpack filename (replace ~ with home) - 8 Pack filename as short as possibly for output to - user. - All open requests should allways use at least: - "open(fn_format(temp_buffe,name,"","",4),...)" to unpack home and - convert filename to system-form. - -@item string fn_same _A((string toname, const char *name, int flag)); -Copys directory and extension from @code{name} to @code{toname} if neaded. -Copying can be forced by same flags used in @code{fn_format()}. - -@item int wild_compare _A((const char *str, const char *wildstr)); -Compare if @code{str} matches @code{wildstr}. @code{wildstr} can contain -@samp{*} and @samp{?} as wildcard characters. -Returns 0 if @code{str} and @code{wildstr} match. - -@item void get_date _A((string to, int timeflag)); -Get current date in a form ready for printing. - -@item void soundex _A((string out_pntr, string in_pntr)) -Makes @code{in_pntr} to a 5 char long string. All words that sound -alike have the same string. - -@item int init_key_cache _A((ulong use_mem, ulong leave_this_much_mem)); -Use caching of keys in MISAM, PISAM, and ISAM. -@code{KEY_CACHE_SIZE} is a good size. -Remember to lock databases for optimal caching. - -@item void end_key_cache _A((void)); -End key caching. -@end table - - - -@node DBUG, protocol, mysys functions, Top -@chapter DBUG Tags To Use - -Here is some of the tags we now use: -(We should probably add a couple of new ones) - -@table @code -@item enter -Arguments to the function. - -@item exit -Results from the function. - -@item info -Something that may be interesting. - -@item warning -When something doesn't go the usual route or may be wrong. - -@item error -When something went wrong. - -@item loop -Write in a loop, that is probably only useful when debugging -the loop. These should normally be deleted when one is -satisfied with the code and it has been in real use for a while. -@end table - -Some specific to mysqld, because we want to watch these carefully: - -@table @code -@item trans -Starting/stopping transactions. - -@item quit -@code{info} when mysqld is preparing to die. - -@item query -Print query. -@end table - - -@node protocol, Fulltext Search, DBUG, Top -@chapter MySQL Client/Server Protocol - -@menu -* raw packet without compression:: -* raw packet with compression:: -* basic packets:: -* communication:: -* fieldtype codes:: -* protocol functions:: -* protocol version 2:: -* 4.1 protocol changes:: -* 4.1 field packet:: -* 4.1 field desc:: -* 4.1 ok packet:: -* 4.1 end packet:: -* 4.1 error packet:: -* 4.1 prep init:: -* 4.1 long data:: -* 4.1 execute:: -* 4.1 binary result:: -@end menu - -@node raw packet without compression, raw packet with compression, protocol, protocol -@section Raw Packet Without Compression - -@example -+-----------------------------------------------+ -| Packet Length | Packet no | Data | -| 3 Bytes | 1 Byte | n Bytes | -+-----------------------------------------------+ -@end example - -@table @asis -@item 3 Byte packet length -The length is calculated with int3store -See include/global.h for details. -The max packetsize can be 16 MB. - -@item 1 Byte packet no -If no compression is used the first 4 bytes of each packet is the header -of the packet. The packet number is incremented for each sent packet. -The first packet starts with 0. -@item n Byte data - -@end table - -The packet length can be recalculated with: - -@example -length = byte1 + (256 * byte2) + (256 * 256 * byte3) -@end example - - -@node raw packet with compression, basic packets, raw packet without compression, protocol -@section Raw Packet With Compression - -@example -+---------------------------------------------------+ -| Packet Length | Packet no | Uncomp. Packet Length | -| 3 Bytes | 1 Byte | 3 Bytes | -+---------------------------------------------------+ -@end example - -@table @asis -@item 3 Byte packet length -The length is calculated with int3store -See include/global.h for details. -The max packetsize can be 16 MB. - -@item 1 Byte packet no -@item 3 Byte uncompressed packet length -@end table - -If compression is used the first 7 bytes of each packet -is the header of the packet. - - -@node basic packets, communication, raw packet with compression, protocol -@section Basic Packets - -@menu -* ok packet:: -* error packet:: -@end menu - - -@node ok packet, error packet, basic packets, basic packets -@subsection OK Packet - -For details, see @file{sql/net_pkg.cc::send_ok()}. - -@example -+-----------------------------------------------+ -| Header | No of Rows | Affected Rows | -| | 1 Byte | 1-8 Byte | -|-----------------------------------------------| -| ID (last_insert_id) | Status | Length | -| 1-8 Byte | 2 Byte | 1-8 Byte | -|-----------------------------------------------| -| Messagetext | -| n Byte | -+-----------------------------------------------+ -@end example - -@table @asis -@item Header -@item 1 byte number of rows ? (always 0 ?) -@item 1-8 bytes affected rows -@item 1-8 byte id (last_insert_id) -@item 2 byte Status (usually 0) -@item If the OK-packege includes a message: -@item 1-8 bytes length of message -@item n bytes messagetext -@end table - - -@node error packet, , ok packet, basic packets -@subsection Error Packet - -@example -+-----------------------------------------------+ -| Header | Status code | Error no | -| | 1 Byte | 2 Byte | -|-----------------------------------------------| -| Messagetext | 0x00 | -| n Byte | 1 Byte | -+-----------------------------------------------+ -@end example - -@table @asis -@item Header -@item 1 byte status code (0xFF = ERROR) -@item 2 byte error number (is only sent to new 3.23 clients. -@item n byte errortext -@item 1 byte 0x00 -@end table - - -@node communication, fieldtype codes, basic packets, protocol -@section Communication - -@example -> Packet from server to client -< Paket from client tor server - - Login - ------ - > 1. packet - Header - 1 byte protocolversion - n byte serverversion - 1 byte 0x00 - 4 byte threadnumber - 8 byte crypt seed - 1 byte 0x00 - 2 byte CLIENT_xxx options (see include/mysql_com.h - that is supported by the server - 1 byte number of current server charset - 2 byte server status variables (SERVER_STATUS_xxx flags) - 13 byte 0x00 (not used yet). - - < 2. packet - Header - 2 byte CLIENT_xxx options - 3 byte max_allowed_packet for the client - n byte username - 1 byte 0x00 - 8 byte crypted password - 1 byte 0x00 - n byte databasename - 1 byte 0x00 - - > 3. packet - OK-packet - - - Command - -------- - < 1. packet - Header - 1 byte command type (e.g.0x03 = query) - n byte query - - Result set (after command) - -------------------------- - > 2. packet - Header - 1-8 byte field_count (packed with net_store_length()) - - If field_count == 0 (command): - 1-8 byte affected rows - 1-8 byte insert id - 2 bytes server_status (SERVER_STATUS_xx) - - If field_count == NULL_LENGTH (251) - LOAD DATA LOCAL INFILE - - If field_count > 0 Result Set: - - > n packets - Header Info - Column description: 5 data object /column - (See code in unpack_fields()) - - Columninfo for each column: - 1 data block table_name - 1 byte length of block - n byte data - 1 data block field_name - 1 byte length of block... - n byte data - 1 data block display length of field - 1 byte length of block - 3 bytes display length of filed - 1 data block type field of type (enum_field_types) - 1 byte length of block - 1 bytexs field of type - 1 data block flags - 1 byte length of block - 2 byte flags for the columns (NOT_NULL_FLAG, ZEROFILL_FLAG....) - 1 byte decimals - - if table definition: - 1 data block default value - - Actual result (one packet per row): - 4 byte header - 1-8 byte length of data - n data -@end example - -@node fieldtype codes, protocol functions, communication, protocol -@section Fieldtype Codes - -@example - display_length |enum_field_type |flags - ---------------------------------------------------- -Blob 03 FF FF 00 |01 FC |03 90 00 00 -Mediumblob 03 FF FF FF |01 FC |03 90 00 00 -Tinyblob 03 FF 00 00 |01 FC |03 90 00 00 -Text 03 FF FF 00 |01 FC |03 10 00 00 -Mediumtext 03 FF FF FF |01 FC |03 10 00 00 -Tinytext 03 FF 00 00 |01 FC |03 10 00 00 -Integer 03 0B 00 00 |01 03 |03 03 42 00 -Mediumint 03 09 00 00 |01 09 |03 00 00 00 -Smallint 03 06 00 00 |01 02 |03 00 00 00 -Tinyint 03 04 00 00 |01 01 |03 00 00 00 -Varchar 03 XX 00 00 |01 FD |03 00 00 00 -Enum 03 05 00 00 |01 FE |03 00 01 00 -Datetime 03 13 00 00 |01 0C |03 00 00 00 -Timestamp 03 0E 00 00 |01 07 |03 61 04 00 -Time 03 08 00 00 |01 0B |03 00 00 00 -Date 03 0A 00 00 |01 0A |03 00 00 00 -@end example - -@node protocol functions, protocol version 2, fieldtype codes, protocol -@section Functions used to implement the protocol - -@c This should be merged with the above one and changed to texi format - -@example - -Raw packets ------------ - -- The my_net_xxxx() functions handles the packaging of a stream of data - into a raw packet that contains a packet number, length and data. - -- This is implemented for the server in sql/net_serv.cc. - The client file, libmysql/net.c, is symlinked to this file - -The important functions are: - -my_net_write() Store a packet (= # number of bytes) to be sent -net_flush() Send the packets stored in the buffer -net_write_command() Send a command (1 byte) + packet to the server. -my_net_read() Read a packet - - -Include files -------------- - -- include/mysql.h is included by all MySQL clients. It includes the - MYSQL and MYSQL_RES structures. -- include/mysql_com.h is include by mysql.h and mysql_priv.h (the - server) and includes a lot of common functions and structures to - handle the client/server protocol. - - -Packets from server to client: ------------------------------ - -sql/net_pkg.cc: - - - Sending of error packets - - Sending of OK packets (= end of data) - - Storing of values in a packet - - -sql/sql_base.cc: - - - Function send_fields() sends the field description to the client. - -sql/sql_show.cc: - - - Sends results for a lot of SHOW commands, including: - SHOW DATABASES [like 'wildcard'] - SHOW TABLES [like 'wildcard'] - - -Packets from client to server: ------------------------------- - -This is done in libmysql/libmysql.c - -The important ones are: - -- mysql_real_connect() Connects to a mysqld server -- mysql_real_query() Sends a query to the server and - reads the ok packet or columns header. -- mysql_store_result() Read a result set from the server to memory -- mysql_use_result() Read a result set row by row from the server. - -- net_safe_read() Read a packet from the server with - error handling. -- net_field_length() Reads the length of a packet string. -- simple_command() Sends a command/query to the server. - - - -Connecting to mysqld (the MySQL server) ---------------------------------------- - -- On the client side: libmysql/libmysql.c::mysql_real_connect(). -- On the server side: sql/sql_parse.cc::check_connections() - -The packets sent during a connection are as follows - -Server: Send greeting package (includes server capabilites, server - version and a random string of bytes to be used to scramble - the password. -Client: Sends package with client capabilites, user name, scrambled - password, database name - -Server: Sends ok package or error package. - -Client: If init command specified, send it t the server and read - ok/error package. - - -Password functions ------------------- - -The passwords are scrambled to a random number and are stored in hex -format on the server. - -The password handling is done in sql/password.c. The important -function is 'scramble()', which takes the a password in clear text -and uses this to 'encrypt' the random string sent by the server -to a new message. - -The encrypted message is sent to the server which uses the stored -random number password to encrypt the random string sent to the -client. If this is equal to the new message the client sends to the -server then the password is accepted. -@end example - -@node protocol version 2, 4.1 protocol changes, protocol functions, protocol -@section Another description of the protocol - -@c This should be merged with the above one and changed to texi format. - -@example -***************************** -* -* PROTOCOL OVERVIEW -* -***************************** - -The MySQL protocol is relatively simple, and is designed for high performance -through minimisation of overhead, and extensibility through versioning and -options flags. It is a request-response protocol, and does not allow -multitasking or multiplexing over a single connection. There are two packet -formats, 'raw' and 'compressed' (which is used when both client and -server support zlib compression, and the client requests that data be -compressed): - -* RAW PACKET, shorter than 16 M * - -+-----------------------------------------------+ -| Packet Length | Packet no | Data | -| 3 Bytes | 1 Byte | n Bytes | -+-----------------------------------------------+ -^ ^ -| 'HEADER' | -+-------------------------------+ - - - * Packet Length: Calculated with int3store. See include/global.h for - details. The basic computation is length = byte1 + - (256 * byte2) + (256 * 256 * byte3). The max packetsize - can be 16 MB. - - * Packet no: The packet number is incremented for each sent packet. - The first packet for each query from the client - starts with 0. - - * Data: Specific to the operation being performed. Most often - used to send string data, such as a SQL query. - -* COMPRESSED PACKET * - -+---------------------------------------------------+-----------------+ -| Packet Length | Packet no | Uncomp. Packet Length | Compressed Data | -| 3 Bytes | 1 Byte | 3 Bytes | n bytes | -+---------------------------------------------------+-----------------+ -^ ^ -| 'HEADER' | -+---------------------------------------------------+ - - * Packet Length: Calculated with int3store. See include/my_global.h for - details. The basic computation is length = byte1 + - (256 * byte2) + (256 * 256 * byte3). The max packetsize - can be 16 MB. - - * Packet no: The packet number is incremented for each sent packet. - The first packet starts with 0. - - * Uncomp. Packet Length: The length of the original, uncompressed packet - If this is zero then the data is not compressed. - - * Compressed Data: The original packet, compressed with zlib compression - - -When using the compressed protocol, the client/server will only compress -send packets where the new packet is smaller than the not compressed one. -In other words, some packets may be compressed while others will not. - -The 'compressed data' is one or more packets in *RAW PACKET* format. - -***************************** -* -* FLOW OF EVENTS -* -***************************** - -To understand how a client communicates with a MySQL server, it is easiest -to start with a high-level flow of events. Each event section will then be -followed by details of the exact contents of each type of packet involved -in the event flow. - -* * -* CONNECTION ESTABLISHMENT * -* * - -Clients connect to the server via a TCP/IP socket (port 3306 by default), a -Unix Domain Socket, or named pipes (on Windows). Once connected, the -following connection establishment sequence is followed: - -+--------+ +--------+ -| Client | | Server | -+--------+ +--------+ - | | - | Handshake initialisation, including MySQL server version, | - | protocol version and options supported, as well as the seed | - | for the password hash | - | | - | <-------------------------------------------------------------- | - | | - | Client options supported, max packet size for client | - | username, password crypted with seed from server, database | - | name. | - | | - | --------------------------------------------------------------> | - | | - | 'OK' packet if authentication succeeds, 'ERROR' packet if | - | authentication fails. | - | | - | <-------------------------------------------------------------- | - | | - - - -* HANDSHAKE INITIALISATION PACKET * - - -+--------------------------------------------------------------------+ -| Header | Prot. Version | Server Version String | 0x00 | -| | 1 Byte | n bytes | 1 byte | -|--------------------------------------------------------------------| -| Thread Number | Crypt Seed | 0x00 | CLIENT_xxx options | -| | | | supported by server | -| 4 Bytes | 8 Bytes | 1 Byte | 2 Bytes | -|--------------------------------------------------------------------| -| Server charset no. | Server status variables | 0x00 padding | -| 1 Byte | 2 Bytes | 13 bytes | -+--------------------------------------------------------------------+ - - * Protocol version (currently '10') - * Server Version String (e.g. '4.0.5-beta-log'). Can be any length as - it's followed by a 0 byte. - * Thread Number - ID of server thread handling this connection - * Crypt seed - seed used to crypt password in auth packet from client - * CLIENT_xxx options - see include/mysql_com.h - * Server charset no. - Index of charset in use by server - * Server status variables - see include/mysql_com.h - * The padding bytes are reserverd for future extensions to the protocol - -* CLIENT AUTH PACKET * - - -+--------------------------------------------------------------------+ -| Header | CLIENT_xxx options supported | max_allowed_packet | -| | by client | for client | -| | 2 Bytes | 3 bytes | -|--------------------------------------------------------------------| -| User Name | 0x00 | Crypted Password | 0x00 | Database Name | -| n Bytes | 1 Byte | 8 Bytes | 1 Byte | n Bytes | -|--------------------------------------------------------------------| -| 0x00 | -| 1 Byte | -+--------------------------------------------------------------------+ - - * CLIENT_xxx options that this client supports: - -#define CLIENT_LONG_PASSWORD 1 /* new more secure passwords */ -#define CLIENT_FOUND_ROWS 2 /* Found instead of affected rows */ -#define CLIENT_LONG_FLAG 4 /* Get all column flags */ -#define CLIENT_CONNECT_WITH_DB 8 /* One can specify db on connect */ -#define CLIENT_NO_SCHEMA 16 /* Don't allow database.table.column */ -#define CLIENT_COMPRESS 32 /* Can use compression protocol */ -#define CLIENT_ODBC 64 /* Odbc client */ -#define CLIENT_LOCAL_FILES 128 /* Can use LOAD DATA LOCAL */ -#define CLIENT_IGNORE_SPACE 256 /* Ignore spaces before '(' */ -#define CLIENT_INTERACTIVE 1024 /* This is an interactive client */ -#define CLIENT_SSL 2048 /* Switch to SSL after handshake */ -#define CLIENT_IGNORE_SIGPIPE 4096 /* IGNORE sigpipes */ -#define CLIENT_TRANSACTIONS 8192 /* Client knows about transactions */ - - * max_allowed_packet for the client (in 'int3store' form) - * User Name - user to authenticate as. Is followed by a null byte. - * Crypted Password - password crypted with seed given in packet from - server, see scramble() in sql/password.c - * Database name (optional) - initial database to use once connected - Is followed by a null byte - -At the end of every client/server exchange there is either an 'OK' packet -or an 'ERROR' packet sent from the server. To determine whether a packet is -an 'OK' packet, or an 'ERROR' packet, check if the first byte (after the -header) is 0xFF. If it has the value of 0xFF, the packet is an 'ERROR' -packet. - - -* OK PACKET * - -For details, see sql/net_pkg.cc::send_ok() - -+-----------------------------------------------+ -| Header | No of Rows | Affected Rows | -| | 1 Byte | 1-9 Byte | -|-----------------------------------------------| -| ID (last_insert_id) | Status | Length | -| 1-9 Byte | 2 Byte | 1-9 Byte | -|-----------------------------------------------| -| Messagetext | -| n Byte | -+-----------------------------------------------+ - - * Number of rows, always 0 - * Affected rows - * ID (last_insert_id) - value for auto_increment column (if any) - * Status (usually 0) - -In general, in the MySQL protocol, fields in a packet that that -represent numeric data, such as lengths, that are labeled as '1-9' -bytes can be decoded by the following logic: - - If the first byte is '251', the - corresponding column value is NULL (only appropriate in - 'ROW DATA' packets). - - If the first byte is '252', the value stored can be read - from the following 2 bytes as a 16-bit integer. - - - If the first byte is '253' the value stored can be read - from the following 4 bytes as a 32-bit long integer - - - If the first byte is '254', the value stored can be read - from the following 8 bytes as a 64-byte long - - Otherwise (values 0-250), the value stored is the value of the - first byte itself. - - -If the OK-packet includes a message: - - * Length of message - * Message Text - - -* ERROR PACKET * - -+-----------------------------------------------+ -| Header | Status code | Error no | -| | 1 Byte | 2 Byte | -|-----------------------------------------------| -| Messagetext | | -| n Byte | | -+-----------------------------------------------+ - - * Status code (0xFF = ERROR) - * Error number (is only sent to 3.23 and newer clients) - * Error message text (ends at end of packet) - -Note that the error message is not null terminated. -The client code can however assume that the packet ends with a null -as my_net_read() will always add an end-null to all read packets to -make things easier for the client. - -Example: - -Packet dump of client connecting to server: - -+------------------------- Protocol Version (10) -| -| +---------------------- Server Version String (0x00 terminated) -| | -| | -0a 34 2e 30 2e 35 2d 62 . 4 . 0 . 5 - b -65 74 61 2d 6c 6f 67 00 e t a - l o g . -15 00 00 00 2b 5a 65 6c . . . . + Z e l - | | - | +------------ First 4 bytes of crypt seed - | - +------------------------ Thread Number - -+------------------------- Last 4 bytes of crypt seed -| -| +-------- CLIENT_XXX Options supported by server -| | -| +-+--+ +--- Server charset index -| | | | -6f 69 41 46 00 2c 28 08 o i A F . , ( . -02 00 00 00 00 00 00 00 . . . . . . . . -| | -| +---------------------- 0x00 padding begins -| -+------------------------- Server status (0x02 = - SERVER_STATUS_AUTOCOMMIT) - -00 00 00 00 00 00 00 00 . . . . . . . . - -* Client Authentication Response (Username 'test', no database - selected) * - - +--------------------- Packet Length (0x13 = 19 bytes) - | - | +--------------- Packet Sequence # - | | - | | +----------- CLIENT_XXX Options supported by client - | | -+---+---+ | +-+-+ -| | | | | -13 00 00 01 03 00 1e 00 . . . . . . . . -00 74 65 73 74 00 48 5e . t e s t . H ^ - | | | - +----+-----+ +------- Scrambled password, 0x00 terminated - | - +----------------- Username, 0x00 terminated - -57 4a 4e 41 4a 4e 00 00 W J N A J N . . -00 . - - ->From this point on, the server waits for 'commands' from the client -which include queries, database shutdown, quit, change user, etc (see -the COM_xxxx values in include/mysql_com.h for the latest -command codes). - -* * -* COMMAND PROCESSING * -* * - -+--------+ +--------+ -| Client | | Server | -+--------+ +--------+ - | | - | A command packet, with a command code, and string data | - | when appropriate (e.g. a query), (see the COM_xxxx values | - | in include/mysql_com.h for the command codes) | - | | - | --------------------------------------------------------------> | - | | - | A 'RESULT' packet if the command completed successfully, | - | an 'ERROR' packet if the command failed. 'RESULT' packets | - | take different forms (see the details following this chart) | - | depending on whether or not the command returns rows. | - | | - | <-------------------------------------------------------------- | - | | - | n 'FIELD PACKET's (if rows are returned) | - | | - | <-------------------------------------------------------------- | - | | - | 'LAST DATA' packet | - | | - | <-------------------------------------------------------------- | - | | - | n 'ROW PACKET's (if rows are returned) | - | | - | <-------------------------------------------------------------- | - | | - | 'LAST DATA' packet | - | | - | <-------------------------------------------------------------- | - | | - - -* Command Packet * - -+------------------------------------------------------+ -| Header | Command type | Query (if applicable) | -| | 1 Byte | n Bytes | -+------------------------------------------------------+ - - * Command type: (e.g.0x03 = query, see the COM_xxxx values in - include/mysql_com.h) - * Query (if applicable) - -Note that my_net_read() null-terminates all packets on the -receiving side of the channel to make it easier for the code -examining the packets. - -The current command codes are: - - 0x00 COM_SLEEP - 0x01 COM_QUIT - 0x02 COM_INIT_DB - 0x03 COM_QUERY - 0x04 COM_FIELD_LIST - 0x05 COM_CREATE_DB - 0x06 COM_DROP_DB - 0x07 COM_REFRESH - 0x08 COM_SHUTDOWN - 0x09 COM_STATISTICS - 0x0a COM_PROCESS_INFO - 0x0b COM_CONNECT - 0x0c COM_PROCESS_KILL - 0x0d COM_DEBUG - 0x0e COM_PING - 0x0f COM_TIME - 0x10 COM_DELAYED_INSERT - 0x11 COM_CHANGE_USER - 0x12 COM_BINLOG_DUMP - 0x13 COM_TABLE_DUMP - 0x14 COM_CONNECT_OUT - 0x15 COM_REGISTER_SLAVE - -* Result Packet * - -Result packet for a command returning _no_ rows: - -+-----------------------------------------------+ -| Header | Field Count | Affected Rows | -| | 1-9 Bytes | 1-9 Bytes | -|-----------------------------------------------| -| ID (last_insert_id) | Server Status | -| 1-9 Bytes | 2 Bytes | -+-----------------------------------------------+ - - * Field Count: Has value of '0' for commands returning _no_ rows - * Affected rows: Count of rows affected by INSERT/UPDATE/DELETE, etc. - * ID: value of auto_increment column in row (if any). 0 if - * Server Status: Usually 0 - -Result packet for a command returning rows: - -+-------------------------------+ -| Header | Field Count | -| | 1-9 Bytes | -+-------------------------------+ - - * Field Count: number of columns/fields in result set, - (packed with net_store_length() in sql/net_pkg.cc) - -This is followed by as many packets as the number of fields ('Field Count') -that contain the metadata for each column/field (see unpack_fields() in -libmysql/libmysql.c): - - -* FIELD PACKET * - -+-----------------------------------------------+ -| Header | Table Name | -| | length-coded-string | -|-----------------------------------------------| -| Field Name | -| length-code-string | -|-----------------------------------------------| -| Display length of field -| length-coded-binary (4 bytes) | -|-----------------------------------------------| -| Field Type (enum_field_types in mysql_com.h) | -| length-coded-binary (2 bytes) | -|-----------------------------------------------| -| Field Flags | Decimal Places| -| length-coded-binary (3 bytes) | 1 Byte | -+--------------+-------------+------------------+ - - * A length coded string is a string where we first have a packet - length (1-9 bytes, packed_with net_store_length()) followed - by a string. - * A length coded binary is a length (1 byte) followed by an integer - value in low-byte-first order. For the moment this type is always - fixed length in this packet. - - * Table Name - the name of the table the column comes from - * Field Name - the name of the column/field - * Display length of field - length of field - * Field Type - Type of field, see enum_field_types in - include/mysql_com.h - - Current field types are: - - 0x00 FIELD_TYPE_DECIMAL - 0x01 FIELD_TYPE_TINY - 0x02 FIELD_TYPE_SHORT - 0x03 FIELD_TYPE_LONG - 0x04 FIELD_TYPE_FLOAT - 0x05 FIELD_TYPE_DOUBLE - 0x06 FIELD_TYPE_NULL - 0x07 FIELD_TYPE_TIMESTAMP - 0x08 FIELD_TYPE_LONGLONG - 0x09 FIELD_TYPE_INT24 - 0x0a FIELD_TYPE_DATE - 0x0b FIELD_TYPE_TIME - 0x0c FIELD_TYPE_DATETIME - 0x0d FIELD_TYPE_YEAR - 0x0e FIELD_TYPE_NEWDATE - 0xf7 FIELD_TYPE_ENUM - 0xf8 FIELD_TYPE_SET - 0xf9 FIELD_TYPE_TINY_BLOB - 0xfa FIELD_TYPE_MEDIUM_BLOB - 0xfb FIELD_TYPE_LONG_BLOB - 0xfc FIELD_TYPE_BLOB - 0xfd FIELD_TYPE_VAR_STRING - 0xfe FIELD_TYPE_STRING - 0xff FIELD_TYPE_GEOMETRY - - * Field Flags - NOT_NULL_FLAG, PRI_KEY_FLAG, xxx_FLAG in - include/mysql_com.h - - -Note that the packet format in 4.1 has slightly changed to allow more values. - - -* ROW PACKET * - -+-----------------------------------------------+ -| Header | Data Length | Column Data | ....for each column -| | 1-9 Bytes | n Bytes | -+-----------------------------------------------+ - - * Data Length: (packed with net_store_length() in sql/net_pkg.cc) - - If 'Data Length' == 0, this is an 'ERROR PACKET'. - - * Column Data: String representation of data. MySQL always sends result set - data as strings. - -* LAST DATA PACKET * - -Packet length is < 9 bytes, and first byte is 0xFE - -+--------+ -| 0xFE | -| 1 Byte | -+--------+ - -Examples: - -*********** -* -* INITDB Command -* -*********** - -A client issuing an 'INITDB' (select the database to use) command, -followed by an 'OK' packet with no rows and no affected rows from -the server: - -* INITDB (select database to use) 'COMMAND' Packet * - - +--------------------- Packet Length (5 bytes) - | - | +--------------- Packet Sequence # - | | - | | +------------ Command # (INITDB = 0x02) - | | -+---+---+ | | +---------- Beginning of query data -| | | | | -05 00 00 00 02 74 65 73 . . . . . t e s -74 t - -* 'OK' Packet with no rows, and no rows affected * - - +--------------------- Packet Length (3 bytes) - | - | +--------------- Packet Sequence # - | | -+---+---+ | -| | | -03 00 00 01 00 00 00 . . . . . . . - - -*********** -* -* SELECT query example -* -*********** - -Client issuing a 'SELECT *' query on the following table: - - CREATE TABLE number_test (minBigInt bigint, - maxBigInt bigint, - testBigInt bigint) - -* 'COMMAND' Packet with QUERY (select ...) * - - +--------------------- Packet Length (26) - | - | +--------------- Packet Sequence # - | | - | | +------------ Command # (QUERY = 0x03) - | | -+---+---+ | | +---------- Beginning of query data -| | | | | -1a 00 00 00 03 53 45 4c . . . . . S E L -45 43 54 20 2a 20 66 72 E C T . * . f r -6f 6d 20 6e 75 6d 62 65 o m . n u m b e -72 5f 74 65 73 74 r _ t e s t - - -and receiving an 'OK' packet with a 'FIELD COUNT' of 3 - - -* 'OK' Packet with 3 fields * - - +--------------------- Packet Length (3 bytes) - | - | +--------------- Packet Sequence # - | | -+---+---+ | -| | | -01 00 00 01 03 . . . . . - -Followed immediately by 3 'FIELD' Packets. Note, the individual packets -are delimitted by =======, so that all fields can be annotated in the first -'FIELD' packet example: - -============================================================= - - +--------------------- Packet Length (0x1f = 31 bytes) - | - | +--------------- Packet Sequence # - | | - | | +------------ Block Length (0x0b = 11 bytes) - | | | -+---+---+ | | +--------- Table Name (11 bytes long) -| | | | | -1f 00 00 02 0b 6e 75 6d . . . . . n u m -62 65 72 5f 74 65 73 74 b e r _ t e s t - - +------------------------ Block Length (9 bytes) - | - | +--------------------- Column Name (9 bytes long) - | | -09 6d 69 6e 42 69 67 49 . m i n B i g I -6e 74 03 14 00 00 01 08 n t . . . . . . - | | | | | - | +---+---+ | +--- Field Type (0x08 = FIELD_TYPE_LONGLONG) - | | | - | | +------ Block Length (1) - | | - | +--------------- Display Length (0x14 = 20 chars) - | - +------------------ Block Length (3) - - +------------------------ Block Length (2) - | - | +-------------------- Field Flags (0 - no flags set) - | | - | +---+ +--------------- Decimal Places (0) - | | | | -02 00 00 00 . . . . - -============================================================= - -'FIELD' packet for the 'number_Test.maxBigInt' column - -1f 00 00 03 0b 6e 75 6d . . . . . n u m -62 65 72 5f 74 65 73 74 b e r _ t e s t -09 6d 61 78 42 69 67 49 . m a x B i g I -6e 74 03 14 00 00 01 08 n t . . . . . . -02 00 00 00 . . . . - -============================================================= - -'FIELD' packet for the 'number_test.testBigInt' column - -20 00 00 04 0b 6e 75 6d . . . . . n u m -62 65 72 5f 74 65 73 74 b e r _ t e s t -0a 74 65 73 74 42 69 67 . t e st B i g -49 6e 74 03 14 00 00 01 I n t . . . . . -08 02 00 00 00 . . . . . -============================================================= - -Followed immediately by one 'LAST DATA' packet: - -fe 00 . . - -Followed immediately by 'n' row packets (in this case, only -one packet is sent from the server, for simplicity's sake): - - - +--------------------- Packet Length (0x52 = 82 bytes) - | - | +--------------- Packet Sequence # - | | - | | +------------ Data Length (0x14 = 20 bytes) - | | | -+---+---+ | | +--------- String Data '-9223372036854775808' -| | | | | (repeat Data Length/Data sequence) - -52 00 00 06 14 2d 39 32 . . . . . - 9 2 -32 33 33 37 32 30 33 36 2 3 3 7 2 0 3 6 -38 35 34 37 37 35 38 30 8 5 4 7 7 5 8 0 -38 13 39 32 32 33 33 37 8 . 9 2 2 3 3 7 -32 30 33 36 38 35 34 37 2 0 3 6 8 5 4 7 -37 35 38 30 37 0a 36 31 7 5 8 0 7 . 6 1 -34 37 34 38 33 36 34 37 4 7 4 8 3 6 4 7 - -Followed immediately by one 'LAST DATA' packet: - -fe 00 . . -@end example - - -@c The Index was empty, and ugly, so I removed it. (jcole, Sep 7, 2000) - -@c @node Index -@c @unnumbered Index - -@c @printindex fn - -@c @node 4.1 protocol,,, -@c @chapter MySQL 4.1 protocol - -@node 4.1 protocol changes, 4.1 field packet, protocol version 2, protocol -@section Changes to 4.0 protocol in 4.1 - -All basic packet handling is identical to 4.0. When communication -with an old 4.0 or 3.x client we will use the old protocol. - -The new things that we support with 4.1 are: - -@itemize @bullet -@item -Warnings -@item -Prepared statements -@item -Binary protocol (will be faster than the current protocol that -converts everything to strings) -@end itemize - - -What has changed in 4.1 are: - -@itemize @bullet -@item -A lot of new field information (database, real table name etc) -@item -The 'ok' packet has more status fields -@item -The 'end' packet (send last for each result set) now contains some -extra information -@item -New protocol for prepared statements. In this case all parameters and -results will sent as binary (low-byte-first). -@end itemize - - -@node 4.1 field packet, 4.1 field desc, 4.1 protocol changes, protocol -@section 4.1 field description packet - -The field description packet is sent as a response to a query that -contains a result set. It can be distinguished from a ok packet by -the fact that the first byte can't be 0 for a field packet. -@xref{4.1 ok packet}. - -The header packet has the following structure: - -@multitable @columnfractions .10 .90 -@item Size @tab Comment -@item 1-9 @tab Number of columns in result set (never 0) -@item 1-9 @tab Extra information sent be some command (SHOW COLUMNS -uses this to send the number of rows in the table) -@end multitable - -This packet is always followed by a field description set. -@xref{4.1 field desc}. - -@node 4.1 field desc, 4.1 ok packet, 4.1 field packet, protocol -@section 4.1 field description result set - -The field description result set contains the meta info for a result set. - -@multitable @columnfractions .20 .80 -@item Type @tab Comment -@item string @tab Database name -@item string @tab Table name alias (or table name if no alias) -@item string @tab Real table name -@item string @tab Alias for column name (or column name if not used) -@item 3 byte int @tab Length of column definition -@item 1 byte int @tab Enum value for field type -@item 3 byte int @tab 2 byte column flags (NOT_NULL_FLAG etc..) + 1 byte number of decimals. -@item string int @tab Default value, only set when using mysql_list_fields(). -@end multitable - - -@node 4.1 ok packet, 4.1 end packet, 4.1 field desc, protocol -@section 4.1 ok packet - -The ok packet is the first that is sent as an response for a query -that didn't return a result set. - -The ok packet has the following structure: - -@multitable @columnfractions .10 .90 -@item Size @tab Comment -@item 1 @tab 0 ; Marker for ok packet -@item 1-9 @tab Affected rows -@item 1-9 @tab Last insert id (0 if one wasn't used) -@item 2 @tab Server status; Can be used by client to check if we are inside an transaction -@item 2 @tab Warning count -@item 1-9 @tab Message length (optional) -@item xxx @tab Message (optional) -@end multitable - -Size 1-9 means that the parameter is packed in to 1-9 bytes depending on -the value. (See function sql/net_pkg.cc::net_store_length). - -The message is optional. For example for multi line INSERT it -contains a string for how many rows was inserted / deleted. - - -@node 4.1 end packet, 4.1 error packet, 4.1 ok packet, protocol -@section 4.1 end packet - -The end packet is sent as the last packet for - -@itemize @bullet -@item -End of field information -@item -End of parameter type information -@item -End of result set -@end itemize - -The end packet has the following structure: - -@multitable @columnfractions .10 .90 -@item Size @tab Comment -@item 1 @tab 254 ; Marker for EOF packet -@item 2 @tab Warning count -@item 2 @tab Status flags (For flags like SERVER_STATUS_MORE_RESULTS) -@end multitable - -Note that a normal packet may start with byte 254, which means -'length stored in 9 bytes'. One can different between these cases -by checking the packet length < 9 bytes (in which case it's and end -packet). - - -@node 4.1 error packet, 4.1 prep init, 4.1 end packet, protocol -@section 4.1 error packet. - -The error packet is sent when something goes wrong. -The error packet has the following structure: - -@multitable @columnfractions .10 .90 -@item Size @tab Comment -@item 1 @tab 255 Error packet marker -@item 2 @tab Error code -@item 1-255 @tab Null terminated error message -@end multitable - -The client/server protocol is designed in such a way that a packet -can only start with 255 if it's an error packet. - - -@node 4.1 prep init, 4.1 long data, 4.1 error packet, protocol -@section 4.1 prepared statement init packet - -This is the return packet when one sends a query with the COM_PREPARE -command. - -@multitable @columnfractions .10 .90 -@item Size @tab Comment -@item 4 @tab Statement handler id -@item 2 @tab Number of columns in result set -@item 2 @tab Number of parameters in query -@end multitable - -After this, there is a packet that contains the following for each -parameter in the query: - -@multitable @columnfractions .10 .90 -@item Size @tab Comment -@item 2 @tab Enum value for field type. (MYSQL_TYPE_UNKNOWN if not known) -@item 2 @tab 2 byte column flags (NOT_NULL_FLAG etc) -@item 1 @tab Number of decimals -@item 4 @tab Max column length. -@end multitable - -Note that the above is not yet in 4.1 but will be added this month. - -As MySQL can have a parameter 'anywhere' it will in many cases not be -able to provide the optimal information for all parameters. - -If number of columns, in the header packet, is not 0 then the -prepared statement will contain a result set. In this case the packet -is followed by a field description result set. @xref{4.1 field desc}. - - -@node 4.1 long data, 4.1 execute, 4.1 prep init, protocol -@section 4.1 long data handling - -This is used by mysql_send_long_data() to set any parameter to a string -value. One can call mysql_send_long_data() multiple times for the -same parameter; The server will concatenate the results to a one big -string. - -The server will not require an end packet for the string. -mysql_send_long_data() is responsible updating a flag that all data -has been sent. (Ie; That the last call to mysql_send_long_data() has -the 'last_data' flag set). - -This packet is sent from client -> server: - -@multitable @columnfractions .10 .90 -@item Size @tab Comment -@item 4 @tab Statement handler -@item 2 @tab Parameter number -@item 2 @tab Type of parameter (not used at this point) -@item # @tab data (Rest of packet) -@end multitable - -The server will NOT send an @code{ok} or @code{error} packet in -responce for this. If there is any errors (like to big string), one -will get the error when calling execute. - -@node 4.1 execute, 4.1 binary result, 4.1 long data, protocol -@section 4.1 execute - -On execute we send all parameters to the server in a COM_EXECUTE -packet. - -The packet contains the following information: - -@multitable @columnfractions .30 .70 -@item Size @tab Comment -@item (param_count+9)/8 @tab Null bit map (2 bits reserved for protocol) -@item 1 @tab new_parameter_bound flag. Is set to 1 for first -execute or if one has rebound the parameters. -@item 2*param_count @tab Type of parameters (only given if new_parameter_bound flag is 1) -@item # @tab Parameter data, repeated for each parameter that are -NOT NULL and not used with mysql_send_long_data(). -@end multitable - -The null-bit-map is for all parameters (including parameters sent with -'mysql_send_long_data). If parameter 0 is NULL, then bit 0 in the -null-bit-map should be 1 (ie: first byte should be 1) - -The parameters are stored the following ways: - -@multitable @columnfractions .20 .10 .70 -@item Type @tab Size @tab Comment -@item tinyint @tab 1 @tab One byte integer -@item short @tab 2 @tab -@item int @tab 4 @tab -@item longlong @tab 8 @tab -@item float @tab 4 @tab -@item double @tab 8 @tab -@item string @tab 1-9 + # @tab Packed string length + string -@end multitable - -The result for this will be either an ok packet or a binary result -set. - -@node 4.1 binary result, , 4.1 execute, protocol -@section 4.1 binary result set - -A binary result are sent the following way. - -For each result row: - -@itemize -@item -null bit map with first two bits set to 01 (bit 0,1 value 1) -@item -parameter data, repeated for each not null result column. -@end itemize - -The idea with the reserving two bits in the null map is that we can -use standard error (first byte 255) and ok packets (first byte 0) -to end a result sets. - -Except that the null-bit-map is shifted two steps, the server is -sending the data to the client the same way that the server is sending -bound parameters to the client. The server is always sending the data -as type given for 'column type' for respective column. It's up to the -client to convert the parameter to the requested type. - -DATETIME, DATE and TIME are sent to the server in a binary format as follows: - -@multitable @columnfractions .20 .10 .70 -@item Type @tab Size @tab Comment -@item date @tab 1 + 0-11 @tab Length + 2 byte year, 1 byte MMDDHHMMSS, 4 byte billionth of a second -@item datetime @tab 1 + 0-11 @tab Length + 2 byte year, 1 byte MMDDHHMMSS, 4 byte billionth of a second -@item time @tab 1 + 0-14 @tab Length + sign (0 = pos, 1= neg), 4 byte days, 1 byte HHMMDD, 4 byte billionth of a second -@end multitable - -The first byte is a length byte and then comes all parameters that are -not 0. (Always counted from the beginning). - -@node Fulltext Search, MyISAM Record Structure, protocol, Top -@chapter Fulltext Search in MySQL - -Hopefully, sometime there will be complete description of -fulltext search algorithms. -Now it's just unsorted notes. - -@menu -* Weighting in boolean mode:: -@end menu - -@node Weighting in boolean mode, , Fulltext Search, Fulltext Search -@section Weighting in boolean mode - -The basic idea is as follows: in expression -@code{A or B or (C and D and E)}, either @code{A} or @code{B} alone -is enough to match the whole expression. While @code{C}, -@code{D}, and @code{E} should @strong{all} match. So it's -reasonable to assign weight 1 to @code{A}, @code{B}, and -@code{(C and D and E)}. And @code{C}, @code{D}, and @code{E} -should get a weight of 1/3. - -Things become more complicated when considering boolean -operators, as used in MySQL FTB. Obvioulsy, @code{+A +B} -should be treated as @code{A and B}, and @code{A B} - -as @code{A or B}. The problem is, that @code{+A B} can @strong{not} -be rewritten in and/or terms (that's the reason why this - extended - -set of operators was chosen). Still, aproximations can be used. -@code{+A B C} can be approximated as @code{A or (A and (B or C))} -or as @code{A or (A and B) or (A and C) or (A and B and C)}. -Applying the above logic (and omitting mathematical -transformations and normalization) one gets that for -@code{+A_1 +A_2 ... +A_N B_1 B_2 ... B_M} the weights -should be: @code{A_i = 1/N}, @code{B_j=1} if @code{N==0}, and, -otherwise, in the first rewritting approach @code{B_j = 1/3}, -and in the second one - @code{B_j = (1+(M-1)*2^M)/(M*(2^(M+1)-1))}. - -The second expression gives somewhat steeper increase in total -weight as number of matched B's increases, because it assigns -higher weights to individual B's. Also the first expression in -much simplier. So it is the first one, that is implemented in MySQL. - - -@node MyISAM Record Structure, InnoDB Record Structure, Fulltext Search, Top -@chapter MyISAM Record Structure - -@section Introduction - -When you say: -@* - -@strong{CREATE TABLE Table1 ...} -@* - -MySQL creates files named Table1.MYD ("MySQL Data"), Table1.MYI -("MySQL Index"), and Table1.frm ("Format"). These files will be in the -directory: @* -/// -@* - -For example, if you use Linux, you might find the files here (assume -your database name is "test"): @* -/usr/local/var/test -@* - -And if you use Windows, you might find the files in this directory: @* -\mysql\data\test\ -@*@* - -Let's look at the .MYD Data file (MyISAM SQL Data file) more closely. -There are three possible formats -- fixed, dynamic, and packed. First, -let's discuss the fixed format. - - -@table @strong -@item Page Size -Unlike most DBMSs, MySQL doesn't store on disk using pages. Therefore -you will not see filler space between rows. (Reminder: This does not -refer to BDB and InnoDB tables, which do use pages). -@* - -@item Record Header -The minimal record header is a set of flags: -@itemize @bullet -@item -"X bit" = 0 if row is deleted, = 1 if row is not deleted -@item -"Null Bits" = 0 if column is not NULL, = 1 if column is NULL -@item -"Filler Bits" = 1 -@end itemize -@end table -@* - -The length of the record header is thus:@* -(1 + number of NULL columns + 7) / 8 bytes@* -After the header, all columns are stored in -the order that they were created, which is the -same order that you would get from SHOW COLUMNS. - -Here's an example. Suppose you say: -@* - -@strong{CREATE TABLE Table1 (column1 CHAR(1), column2 CHAR(1), column3 CHAR(1))} -@* - -@strong{INSERT INTO Table1 VALUES ('a', 'b', 'c')} -@* - -@strong{INSERT INTO Table1 VALUES ('d', NULL, 'e')} -@* - -A CHAR(1) column takes precisely one byte (plus one bit of overhead -that is assigned to every column -- I'll describe the details of -column storage later). So the file Table1.MYD looks like this: -@* - -@strong{Hexadecimal Display of Table1.MYD file}@* -@code{ -F1 61 62 63 00 F5 64 00 66 00 ... .abc..d e. -} -@* - -Here's how to read this hexadecimal-dump display:@* -@itemize @bullet -@item -The hexadecimal numbers @code{F1 61 62 63 00 F5 64 20 66 00} are byte -values and the column on the right is an attempt to show the -same bytes in ASCII. -@item -The @code{F1} byte means that there are no null fields in the first row. -@item -The @code{F5} byte means that the second column of the second row is NULL. -@end itemize - -(It's probably easier to understand the flag setting if you restate -@code{F5} as @code{11110101 binary}, and (a) notice that the third flag bit from the -right is @code{on}, and (b) remember that the first flag bit is the X bit.) -@* - -There are complications -- the record header is more complex if there -are variable-length fields -- but the simple display shown in the -example is exactly what you'd see if you looked at the MySQL Data file -with a debugger or a hexadecimal file dumper. -@* - -So much for the fixed format. Now, let's discuss the dynamic format. -@* - -The dynamic file format is necessary if rows can vary in size. That will -be the case if there are BLOB columns, or "true" VARCHAR columns. (Remember -that MySQL may treat VARCHAR columns as if they're CHAR columns, in which -case the fixed format is used.) A dynamic row has more fields in the header. -The important ones are "the actual length", "the unused length", and "the -overflow pointer". The actual length is the total number of bytes in all the -columns. The unused length is the total number of bytes between one physical -record and the next one. The overflow pointer is the location of the rest of -the record if there are multiple parts. -@* - -For example, here is a dynamic row: -@* -@example -03, 00 start of header -04 actual length -0c unused length -01, fc flags + overflow pointer -**** data in the row -************ unused bytes - <-- next row starts here) -@end example - -In the example, the actual length and the unused length -are short (one byte each) because the table definition -says that the columns are short -- if the columns were -potentially large, then the actual length and the unused -length could be two bytes each, three bytes each, and so -on. In this case, actual length plus unused length is 10 -hexadecimal (sixteen decimal), which is a minimum. - -As for the third format -- packed -- we will only say -briefly that: -@itemize @bullet -@item -Numeric values are stored in a form that depends on the -range (start/end values) for the data type. -@item -All columns are packed using either Huffman or enum coding. -@end itemize - -For details, see the source files /myisam/mi_statrec.c -(for fixed format), /myisam/mi_dynrec.c (for dynamic -format), and /myisam/mi_packrec.c (for packed format). - -Note: Internally, MySQL uses a format much like the fixed format -which it uses for disk storage. The main differences are: -@enumerate -@item -BLOBs have a length and a memory pointer rather than being stored inline. -@item -"True VARCHAR" (a column storage which will be fully implemented in -version 5.0) will have a 16-bit length plus the data. -@item -All integer or floating-point numbers are stored with the low byte first. -Point (3) does not apply for ISAM storage or internals. -@end enumerate -@* - - -@section Physical Attributes of Columns - -Next I'll describe the physical attributes of each column in a row. -The format depends entirely on the data type and the size of the -column, so, for every data type, I'll give a description and an example. -@* - -@table @strong -@item The character data types - -@strong{CHAR} -@itemize @bullet -@item -Storage: fixed-length string with space padding on the right. -@item -Example: a CHAR(5) column containing the value 'A' looks like:@* -@code{hexadecimal 41 20 20 20 20} -- (length = 5, value = @code{'A '}) -@end itemize - -@strong{VARCHAR} -@itemize @bullet -@item -Storage: variable-length string with a preceding length. -@item -Example: a VARCHAR(7) column containing 'A' looks like:@* -@code{hexadecimal 01 41} -- (length = 1, value = @code{'A'}) -@end itemize - -@item The numeric data types - -Important: MySQL almost always stores multi-byte binary numbers with -the low byte first. This is called "little-endian" numeric storage; -it's normal on Intel x86 machines; MySQL uses it even for non-Intel -machines so that databases will be portable. -@* - -@strong{TINYINT} -@itemize @bullet -@item -Storage: fixed-length binary, always one byte. -@item -Example: a TINYINT column containing 65 looks like:@* -@code{hexadecimal 41} -- (length = 1, value = 65) -@end itemize - -@strong{SMALLINT} -@itemize @bullet -@item -Storage: fixed-length binary, always two bytes. -@item -Example: a SMALLINT column containing 65 looks like:@* -@code{hexadecimal 41 00} -- (length = 2, value = 65) -@end itemize - -@strong{MEDIUMINT} -@itemize @bullet -@item -Storage: fixed-length binary, always three bytes. -@item -Example: a MEDIUMINT column containing 65 looks like:@* -@code{hexadecimal 41 00 00} -- (length = 3, value = 65) -@end itemize - -@strong{INT} -@itemize @bullet -@item -Storage: fixed-length binary, always four bytes. -@item -Example: an INT column containing 65 looks like:@* -@code{hexadecimal 41 00 00 00} -- (length = 4, value = 65) -@end itemize - -@strong{BIGINT} -@itemize @bullet -@item -Storage: fixed-length binary, always eight bytes. -@item -Example: a BIGINT column containing 65 looks like:@* -@code{hexadecimal 41 00 00 00 00 00 00 00} -- (length = 8, value = 65) -@end itemize - -@strong{FLOAT} -@itemize @bullet -@item -Storage: fixed-length binary, always four bytes. -@item -Example: a FLOAT column containing approximately 65 looks like:@* -@code{hexadecimal 00 00 82 42} -- (length = 4, value = 65) -@end itemize - -@strong{DOUBLE PRECISION} -@itemize @bullet -@item -Storage: fixed-length binary, always eight bytes. -@item -Example: a DOUBLE PRECISION column containing approximately 65 looks like:@* -@code{hexadecimal 00 00 00 00 00 40 50 40} -- (length = 8, value = 65) -@end itemize - -@strong{REAL} -@itemize @bullet -@item -Storage: same as FLOAT, or same as DOUBLE PRECISION, depending on setting of the --ansi switch. -@end itemize - -@strong{DECIMAL} -@itemize @bullet -@item -Storage: fixed-length string, with a leading byte for the sign, if any. -@item -Example: a DECIMAL(2) column containing 65 looks like:@* -@code{hexadecimal 20 36 35} -- (length = 3, value = @code{' 65'}) -@item -Example: a DECIMAL(2) UNSIGNED column containing 65 looks like:@* -@code{hexadecimal 36 35} -- (length = 2, value = @code{'65'}) -@item -Example: a DECIMAL(4,2) UNSIGNED column containing 65 looks like:@* -@code{hexadecimal 36 35 2E 30 30} -- (length = 5, value = @code{'65.00'}) -@end itemize - -@strong{NUMERIC} -@itemize @bullet -@item -Storage: same as DECIMAL. -@end itemize - -@strong{BOOL} -@itemize @bullet -@item -Storage: same as TINYINT. -@end itemize - -@item The temporal data types - -@strong{DATE} -@itemize @bullet -@item -Storage: 3 byte integer, low byte first. -Packed as: 'day + month*32 + year*16*32' -@item -Example: a DATE column containing '1962-01-02' looks like:@* -@code{hexadecimal 22 54 0F} -@end itemize - -@strong{DATETIME} -@itemize @bullet -@item -Storage: eight bytes. -@item -Part 1 is a 32-bit integer containing year*10000 + month*100 + day. -@item -Part 2 is a 32-bit integer containing hour*10000 + minute*100 + second. -@item -Example: a DATETIME column for '0001-01-01 01:01:01' looks like:@* -@code{hexadecimal B5 2E 11 5A 02 00 00 00} -@end itemize - -@strong{TIME} -@itemize @bullet -@item -Storage: 3 bytes, low byte first. -This is stored as seconds: days*24*3600+hours*3600+minutes*60+seconds -@item -Example: a TIME column containing '1 02:03:04' (1 day 2 hour 3 minutes and 4 seconds) looks like:@* -@code{hexadecimal 58 6E 01} -@end itemize - -@strong{TIMESTAMP} -@itemize @bullet -@item -Storage: 4 bytes, low byte first. -Stored as unix @code{time()}, which is seconds since the Epoch -(00:00:00 UTC, January 1, 1970). -@item -Example: a TIMESTAMP column containing '2003-01-01 01:01:01' looks like:@* -@code{hexadecimal 4D AE 12 23} -@end itemize - -@strong{YEAR} -@itemize @bullet -@item -Storage: same as unsigned TINYINT with a base value of 0 = 1901. -@end itemize - -@item Others - -@strong{SET} -@itemize @bullet -@item -Storage: one byte for each eight members in the set. -@item -Maximum length: eight bytes (for maximum 64 members). -@item -This is a bit list. The least significant bit corresponds to the -first listed member of the set. -@item -Example: a SET('A','B','C') column containing 'A' looks like:@* -@code{01} -- (length = 1, value = 'A') -@end itemize - -@strong{ENUM} -@itemize @bullet -@item -Storage: one byte if less than 256 alternatives, else two bytes. -@item -This is an index. The value 1 corresponds to the first listed -alternative. (Note: ENUM always reserves 0 for an erroneous value. This -explains why 'A' is 1 instead of 0.) -@item -Example: an ENUM('A','B','C') column containing 'A' looks like:@* -@code{01} -- (length = 1, value = 'A') -@end itemize - -@item The Large-Object data types - -Warning: Because TINYBLOB's preceding length is one byte long (the -size of a TINYINT) and MEDIUMBLOB's preceding length is three bytes -long (the size of a MEDIUMINT), it's easy to think there's some sort -of correspondence between the BLOB and the INT types. There isn't -- a -BLOB's preceding length is not four bytes long (the size of an INT). -@* - -@strong{TINYBLOB} -@itemize @bullet -@item -Storage: variable-length string with a preceding one-byte length. -@item -Example: a TINYBLOB column containing 'A' looks like:@* -@code{hexadecimal 01 41} -- (length = 2, value = 'A') -@end itemize - -@strong{TINYTEXT} -@itemize @bullet -@item -Storage: same as TINYBLOB. -@end itemize - -@strong{BLOB} -@itemize @bullet -@item -Storage: variable-length string with a preceding two-byte length. -@item -Example: a BLOB column containing 'A' looks like:@* -@code{hexadecimal 01 00 41} -- (length = 2, value = 'A') -@end itemize - -@strong{TEXT} -@itemize @bullet -@item -Storage: same as BLOB. -@end itemize - -@strong{MEDIUMBLOB} -@itemize @bullet -@item -Storage: variable-length string with a preceding length. -@item -Example: a MEDIUMBLOB column containing 'A' looks like:@* -@code{hexadecimal 01 00 00 41} -- (length = 4, value = 'A') -@end itemize - -@strong{MEDIUMTEXT} -@itemize @bullet -@item -Storage: same as MEDIUMBLOB. -@end itemize - -@strong{LONGBLOB} -@itemize @bullet -@item -Storage: variable-length string with a preceding four-byte length. -@item -Example: a LONGBLOB column containing 'A' looks like:@* -@code{hexadecimal 01 00 00 00 41} -- (length = 5, value = 'A') -@end itemize - -@strong{LONGTEXT} -@itemize @bullet -@item -Storage: same as LONGBLOB. -@end itemize - -@end table - -@section Where to Look For More Information - -@strong{References:} @* -Most of the formatting work for MyISAM columns is visible -in the program /sql/field.cc in the source code directory. -And in the MyISAM directory, the files that do formatting -work for different record formats are: /myisam/mi_statrec.c, -/myisam/mi_dynrec.c, and /myisam/mi_packrec.c. -@* - -@node InnoDB Record Structure, InnoDB Page Structure, MyISAM Record Structure, Top -@chapter InnoDB Record Structure - -This page contains: -@itemize @bullet -@item -A high-altitude "summary" picture of the parts of a MySQL/InnoDB -record structure. -@item -A description of each part. -@item -An example. -@end itemize - -After reading this page, you will know how MySQL/InnoDB stores a -physical record. -@* - -@section High-Altitude Picture - -The chart below shows the three parts of a physical record. - -@multitable @columnfractions .10 .35 - -@item @strong{Name} @tab @strong{Size} -@item Field Start Offsets -@tab (F*1) or (F*2) bytes -@item Extra Bytes -@tab 6 bytes -@item Field Contents -@tab depends on content - -@end multitable - -Legend: The letter 'F' stands for 'Number Of Fields'. - -The meaning of the parts is as follows: -@itemize @bullet -@item -The FIELD START OFFSETS is a list of numbers containing the -information "where a field starts". -@item -The EXTRA BYTES is a fixed-size header. -@item -The FIELD CONTENTS contains the actual data. -@end itemize - -@strong{An Important Note About The Word "Origin"}@* -The "Origin" or "Zero Point" of a record is the first byte of the -Field Contents -- not the first byte of the Field Start Offsets. If -there is a pointer to a record, that pointer is pointing to the -Origin. Therefore the first two parts of the record are addressed by -subtracting from the pointer, and only the third part is addressed by -adding to the pointer. - -@subsection FIELD START OFFSETS - -The Field Start Offsets is a list in which each entry is the -position, relative to the Origin, of the start of the next field. The -entries are in reverse order, that is, the first field's offset is at -the end of the list. -@* - -An example: suppose there are three columns. The first column's length -is 1, the second column's length is 2, and the third column's length is 4. -In this case, the offset values are, respectively, 1, 3 (1+2), and 7 (1+2+4). -Because values are reversed, a core dump of the Field Start Offsets -would look like this: @code{07,03,01}. -@* - -There are two complications for special cases: -@itemize @bullet -@item -Complication #1: The size of each offset can be either one byte or -two bytes. One-byte offsets are only usable if the total record size -is less than 127. There is a flag in the "Extra Bytes" part which will -tell you whether the size is one byte or two bytes. -@item -Complication #2: The most significant bits of an offset may contain -flag values. The next two paragraphs explain what the contents are. -@end itemize - -@strong{When The Size Of Each Offset Is One Byte} -@itemize @bullet -@item -1 bit = 0 if field is non-NULL, = 1 if field is NULL -@item -7 bits = the actual offset, a number between 0 and 127 -@end itemize - -@strong{When The Size Of Each Offset Is Two Bytes} -@itemize @bullet -@item -1 bit = 0 if field is non-NULL, = 1 if field is NULL -@item -1 bit = 0 if field is on same page as offset, = 1 if field and offset are on different pages -@item -14 bits = the actual offset, a number between 0 and 16383 -@end itemize - -It is unlikely that the "field and offset are on different pages" -unless the record contains a large BLOB. - -@subsection EXTRA BYTES - -The Extra Bytes are a fixed six-byte header. - -@multitable @columnfractions .10 .25 .35 - -@item @strong{Name} @tab @strong{Size} @tab @strong{Description} -@item @strong{info_bits:} -@item () -@tab 1 bit -@tab unused or unknown -@item () -@tab 1 bit -@tab unused or unknown -@item deleted_flag -@tab 1 bit -@tab 1 if record is deleted -@item min_rec_flag -@tab 1 bit -@tab 1 if record is predefined minimum record -@item n_owned -@tab 4 bits -@tab number of records owned by this record -@item heap_no -@tab 13 bits -@tab record's order number in heap of index page -@item n_fields -@tab 10 bits -@tab number of fields in this record, 1 to 1023 -@item 1byte_offs_flag -@tab 1 bit -@tab 1 if each Field Start Offsets is 1 byte long (this item is also called the "short" flag) -@item @strong{next 16 bits} -@tab 16 bits -@tab pointer to next record in page -@item @strong{TOTAL} -@tab 48 bits - -@end multitable - -Total size is 48 bits, which is six bytes. -@* - -If you're just trying to read the record, the key bit in the Extra -Bytes is 1byte_offs_flag -- you need to know if 1byte_offs_flag is 1 -(i.e.: "short 1-byteoffsets") or 0 (i.e.: "2-byte offsets"). -@* - -Given a pointer to the Origin, InnoDB finds the start of the record as follows: -@itemize @bullet -@item -Let X = n_fields (the number of fields is by definition equal to the -number of entries in the Field Start Offsets Table). -@item -If 1byte_offs_flag equals 0, then let X = X * 2 because there are -two bytes for each entry instead of just one. -@item -Let X = X + 6, because the fixed size of Extra Bytes is 6. -@item -The start of the record is at (pointer value minus X). -@end itemize - -@subsection FIELD CONTENTS - -The Field Contents part of the record has all the data. Fields are -stored in the order they were defined in. -@* - -There are no markers between fields, and there is no marker or filler -at the end of a record. -@* - -Here's an example. -@itemize @bullet -@item -I made a table with this definition: -@*@* - -@strong{CREATE TABLE T - (FIELD1 VARCHAR(3), FIELD2 VARCHAR(3), FIELD3 VARCHAR(3)) - Type=InnoDB;} -@*@* - -To understand what follows, you must know that table T has six columns --- not three -- because InnoDB automatically added three "system -columns" at the start for its own housekeeping. It happens that these -system columns are the row ID, the transaction ID, and the rollback -pointer, but their values don't matter now. Regard them as three black -boxes. -@*@* - -@item -I put some rows in the table. My last three INSERTs were: -@*@* - -@strong{INSERT INTO T VALUES ('PP', 'PP', 'PP')} -@*@* - -@strong{INSERT INTO T VALUES ('Q', 'Q', 'Q')} -@*@* - -@strong{INSERT INTO T VALUES ('R', NULL, NULL)} -@*@* - -@item -I ran Borland's TDUMP to get a hexadecimal dump of -the contents of \mysql\data\ibdata1, which (in my case) is the -MySQL/InnoDB data file (on Windows). -@end itemize - -Here is an extract of the dump: - -@multitable @columnfractions .05 .95 - -@item @strong{Address Values In Hexadecimal} @tab @strong{Values In ASCII} -@item @code{0D4280: 00 00 2D 00 84 4F 4F 4F 4F 4F 4F 4F 4F 4F 19 17} -@tab @code{..-..OOOOOOOOO..} -@item @code{0D4290: 15 13 0C 06 00 00 78 0D 02 BF 00 00 00 00 04 21} -@tab @code{......x........!} -@item @code{0D42A0: 00 00 00 00 09 2A 80 00 00 00 2D 00 84 50 50 50} -@tab @code{.....*....-..PPP} -@item @code{0D42B0: 50 50 50 16 15 14 13 0C 06 00 00 80 0D 02 E1 00} -@tab @code{PPP.............} -@item @code{0D42C0: 00 00 00 04 22 00 00 00 00 09 2B 80 00 00 00 2D} -@tab @code{....".....+....-} -@item @code{0D42D0: 00 84 51 51 51 94 94 14 13 0C 06 00 00 88 0D 00} -@tab @code{..QQQ...........} -@item @code{0D42E0: 74 00 00 00 00 04 23 00 00 00 00 09 2C 80 00 00} -@tab @code{t.....#.....,...} -@item @code{0D42F0: 00 2D 00 84 52 00 00 00 00 00 00 00 00 00 00 00} -@tab @code{.-..R...........} - -@end multitable - -A reformatted version of the dump, showing only the relevant bytes, -looks like this (I've put a line break after each field and added labels): - -@strong{Reformatted Hexadecimal Dump}@* -@code{ - 19 17 15 13 0C 06 Field Start Offsets /* First Row */@* - 00 00 78 0D 02 BF Extra Bytes@* - 00 00 00 00 04 21 System Column #1@* - 00 00 00 00 09 2A System Column #2@* - 80 00 00 00 2D 00 84 System Column #3@* - 50 50 Field1 'PP'@* - 50 50 Field2 'PP'@* - 50 50 Field3 'PP'}@* - -@code{ - 16 15 14 13 0C 06 Field Start Offsets /* Second Row */@* - 00 00 80 0D 02 E1 Extra Bytes@* - 00 00 00 00 04 22 System Column #1@* - 00 00 00 00 09 2B 80 System Column #2@* - 00 00 00 2D 00 84 System Column #3@* - 51 Field1 'Q'@* - 51 Field2 'Q'@* - 51 Field3 'Q'}@* - -@code{ - 94 94 14 13 0C 06 Field Start Offsets /* Third Row */@* - 00 00 88 0D 00 74 Extra Bytes@* - 00 00 00 00 04 23 System Column #1@* - 00 00 00 00 09 2C System Column #2@* - 80 00 00 00 2D 00 84 System Column #3@* - 52 Field1 'R'}@* -@* - -You won't need explanation if you followed everything I've said, but -I'll add helpful notes for the three trickiest details. -@itemize @bullet -@item -Helpful Notes About "Field Start Offsets": @* -Notice that the sizes of the record's fields, in forward order, are: -6, 6, 7, 2, 2, 2. Since each offset is for the start of the "next" -field, the hexadecimal offsets are 06, 0c (6+6), 13 (6+6+7), 15 -(6+6+7+2), 17 (6+6+7+2+2), 19 (6+6+7+2+2+2). Reversing the order, the -Field Start Offsets of the first record are: @code{19,17,15,13,0c,06}. -@item -Helpful Notes About "Extra Bytes": @* -Look at the Extra Bytes of the first record: @code{00 00 78 0D 02 BF}. The -fourth byte is @code{0D hexadecimal}, which is @code{1101 binary} ... the 110 is the -last bits of n_fields (@code{110 binary} is 6 which is indeed the number of -fields in the record) and the final 1 bit is 1byte_offs_flag. The -fifth and sixth bytes, which contain @code{02 BF}, constitute the "next" -field. Looking at the original hexadecimal dump, at address -@code{0D42BF} (which is position @code{02BF} within the page), you'll see the beginning bytes of -System Column #1 of the second row. In other words, the "next" field -points to the "Origin" of the following row. -@item -Helpful Notes About NULLs:@* -For the third row, I inserted NULLs in FIELD2 and FIELD3. Therefore in -the Field Start Offsets the top bit is @code{on} for these fields (the -values are @code{94 hexadecimal}, @code{94 hexadecimal}, instead of -@code{14 hexadecimal}, @code{14 hexadecimal}). And the row is -shorter because the NULLs take no space. -@end itemize - -@section Where to Look For More Information - -@strong{References:} @* -The most relevant InnoDB source-code files are rem0rec.c, rem0rec.ic, -and rem0rec.h in the rem ("Record Manager") directory. - -@node InnoDB Page Structure, Files in MySQL Sources, InnoDB Record Structure, Top -@chapter InnoDB Page Structure - -InnoDB stores all records inside a fixed-size unit which is commonly called a -"page" (though InnoDB sometimes calls it a "block" instead). -Currently all pages are the same size, 16KB. -@* - -A page contains records, but it also contains headers and trailers. -I'll start this description with a high-altitude view of a page's parts, -then I'll describe each part of a page. Finally, I'll show an example. This -discussion deals only with the most common format, for the leaf page of a data file. -@* - -@section High-Altitude View - -An InnoDB page has seven parts: -@itemize @bullet -@item -Fil Header -@item -Page Header -@item -Infimum + Supremum Records -@item -User Records -@item -Free Space -@item -Page Directory -@item -Fil Trailer -@end itemize - -As you can see, a page has two header/trailer pairs. The inner pair, "Page Header" and -"Page Directory", are mostly the concern of the \page program group, -while the outer pair, "Fil Header" and "Fil Trailer", are mostly the -concern of the \fil program group. The "Fil" header also goes goes by -the name of "File Page Header". -@* - -Sandwiched between the headers and trailers, are the records and -the free (unused) space. A page always begins with two unchanging -records called the Infimum and the Supremum. Then come the user -records. Between the user records (which grow downwards) and the page -directory (which grows upwards) there is space for new records. -@* - -@subsection Fil Header - -The Fil Header has eight parts, as follows: - -@multitable @columnfractions .10 .30 .35 - -@item @strong{Name} @tab @strong{Size} @tab @strong{Remarks} -@item FIL_PAGE_SPACE -@tab 4 -@tab 4 ID of the space the page is in -@item FIL_PAGE_OFFSET -@tab 4 -@tab ordinal page number from start of space -@item FIL_PAGE_PREV -@tab 4 -@tab offset of previous page in key order -@item FIL_PAGE_NEXT -@tab 4 -@tab offset of next page in key order -@item FIL_PAGE_LSN -@tab 8 -@tab log serial number of page's latest log record -@item FIL_PAGE_TYPE -@tab 2 -@tab current defined types are: FIL_PAGE_INDEX, FIL_PAGE_UNDO_LOG, FIL_PAGE_INODE, FIL_PAGE_IBUF_FREE_LIST -@item FIL_PAGE_FILE_FLUSH_LSN -@tab 8 -@tab "the file has been flushed to disk at least up to this lsn" (log serial number), - valid only on the first page of the file -@item FIL_PAGE_ARCH_LOG_NO -@tab 4 -@tab the latest archived log file number at the time that FIL_PAGE_FILE_FLUSH_LSN was written (in the log) -@end multitable - -@itemize -@item -FIL_PAGE_SPACE is a necessary identifier because different pages might belong to -different (table) spaces within the same file. The word -"space" is generic jargon for either "log" or "tablespace". -@*@* - -@item -FIL_PAGE_PREV and FIL_PAGE_NEXT are the page's "backward" and -"forward" pointers. To show what they're about, I'll draw a two-level -B-tree. -@*@* - -@example - -------- - - root - - -------- - | - ---------------------- - | | - | | - -------- -------- - - leaf - <--> - leaf - - -------- -------- -@end example -@* - -Everyone has seen a B-tree and knows that the entries in the root page -point to the leaf pages. (I indicate those pointers with vertical '|' -bars in the drawing.) But sometimes people miss the detail that leaf -pages can also point to each other (I indicate those pointers with a horizontal -two-way pointer '<-->' in the drawing). This feature allows InnoDB to navigate from -leaf to leaf without having to back up to the root level. This is a -sophistication which you won't find in the classic B-tree, which is -why InnoDB should perhaps be called a B+-tree instead. -@*@* - -@item -The fields FIL_PAGE_FILE_FLUSH_LSN, FIL_PAGE_PREV, and FIL_PAGE_NEXT -all have to do with logs, so I'll refer you to my article "How Logs -Work With MySQL And InnoDB" on devarticles.com. -@*@* - -@item -FIL_PAGE_FILE_FLUSH_LSN and FIL_PAGE_ARCH_LOG_NO are only valid for -the first page of a data file. -@end itemize - -@subsection Page Header - -The Page Header has 14 parts, as follows: -@*@* - -@multitable @columnfractions .10 .20 .30 - -@item @strong{Name} @tab @strong{Size} @tab @strong{Remarks} -@item PAGE_N_DIR_SLOTS -@tab 2 -@tab number of directory slots in the Page Directory part; initial value = 2 -@item PAGE_HEAP_TOP -@tab 2 -@tab record pointer to first record in heap -@item PAGE_N_HEAP -@tab 2 -@tab number of heap records; initial value = 2 -@item PAGE_FREE -@tab 2 -@tab record pointer to first free record -@item PAGE_GARBAGE -@tab 2 -@tab "number of bytes in deleted records" -@item PAGE_LAST_INSERT -@tab 2 -@tab record pointer to the last inserted record -@item PAGE_DIRECTION -@tab 2 -@tab either PAGE_LEFT, PAGE_RIGHT, or PAGE_NO_DIRECTION -@item PAGE_N_DIRECTION -@tab 2 -@tab number of consecutive inserts in the same direction, e.g. "last 5 were all to the left" -@item PAGE_N_RECS -@tab 2 -@tab number of user records -@item PAGE_MAX_TRX_ID -@tab 8 -@tab the highest ID of a transaction which might have changed a record on the page (only set for secondary indexes) -@item PAGE_LEVEL -@tab 2 -@tab level within the index (0 for a leaf page) -@item PAGE_INDEX_ID -@tab 8 -@tab identifier of the index the page belongs to -@item PAGE_BTR_SEG_LEAF -@tab 10 -@tab "file segment header for the leaf pages in a B-tree" (this is irrelevant here) -@item PAGE_BTR_SEG_TOP -@tab 10 -@tab "file segment header for the non-leaf pages in a B-tree" (this is irrelevant here) - -@end multitable -@* - -(Note: I'll clarify what a "heap" is when I discuss the User Records part of the page.) -@*@* - -Some of the Page Header parts require further explanation: -@itemize @bullet -@item -PAGE_FREE: @* -Records which have been freed (due to deletion or migration) are in a -one-way linked list. The PAGE_FREE pointer in the page header points -to the first record in the list. The "next" pointer in the record -header (specifically, in the record's Extra Bytes) points to the next -record in the list. -@item -PAGE_DIRECTION and PAGE_N_DIRECTION: @* -It's useful to know whether inserts are coming in a constantly -ascending sequence. That can affect InnoDB's efficiency. -@item -PAGE_HEAP_TOP and PAGE_FREE and PAGE_LAST_INSERT: @* -Warning: Like all record pointers, these point not to the beginning of the -record but to its Origin (see the earlier discussion of Record -Structure). -@item -PAGE_BTR_SEG_LEAF and PAGE_BTR_SEG_TOP: @* -These variables contain information (space ID, page number, and byte offset) about -index node file segments. InnoDB uses the information for allocating new pages. -There are two different variables because InnoDB allocates separately for leaf -pages and upper-level pages. -@end itemize - -@subsection The Infimum And Supremum Records - -"Infimum" and "supremum" are real English words but they are found -only in arcane mathematical treatises, and in InnoDB comments. To -InnoDB, an infimum is lower than the the lowest possible real value -(negative infinity) and a supremum is greater than the greatest -possible real value (positive infinity). InnoDB sets up an infimum -record and a supremum record automatically at page-create time, and -never deletes them. They make a useful barrier to navigation so that -"get-prev" won't pass the beginning and "get-next" won't pass the end. -Also, the infimum record can be a dummy target for temporary record -locks. -@*@* - -The InnoDB code comments distinguish between "the infimum and supremum -records" and the "user records" (all other kinds). -@*@* - -It's sometimes unclear whether InnoDB considers the infimum and -supremum to be part of the header or not. Their size is fixed and -their position is fixed, so I guess so. - -@subsection User Records - -In the User Records part of a page, you'll find all the records that the user -inserted. -@*@* - -There are two ways to navigate through the user records, depending -whether you want to think of their organization as an unordered or an -ordered list. -@*@* - -An unordered list is often called a "heap". If you make a pile of -stones by saying "whichever one I happen to pick up next will go on -top" -- rather than organizing them according to size and colour -- -then you end up with a heap. Similarly, InnoDB does not want to insert -new rows according to the B-tree's key order (that would involve -expensive shifting of large amounts of data), so it inserts new rows -right after the end of the existing rows (at the -top of the Free Space part) or wherever there's space left by a -deleted row. -@*@* - -But by definition the records of a B-tree must be accessible in order -by key value, so there is a record pointer in each record (the "next" -field in the Extra Bytes) which points to the next record in key -order. In other words, the records are a one-way linked list. So -InnoDB can access rows in key order when searching. - -@subsection Free Space - -I think it's clear what the Free Space part of a page is, from the discussion of -other parts. - -@subsection Page Directory - -The Page Directory part of a page has a variable number of record pointers. -Sometimes the record pointers are called "slots" or "directory slots". -Unlike other DBMSs, InnoDB does not have a slot for every record in -the page. Instead it keeps a sparse directory. In a fullish page, -there will be one slot for every six records. -@*@* - -The slots track the records' logical order (the order by key rather -than the order by placement on the heap). Therefore, if the records -are @code{'A' 'B' 'F' 'D'} the slots will be @code{(pointer to 'A') (pointer to -'B') (pointer to 'D') (pointer to 'F')}. Because the slots are in key -order, and each slot has a fixed size, it's easy to do a binary -search of the records on the page via the slots. -@*@* - -(Since the Page Directory does not have a slot for every record, -binary search can only give a rough position and then InnoDB must -follow the "next" record pointers. InnoDB's "sparse slots" policy also -accounts for the n_owned field in the Extra Bytes part of a record: -n_owned indicates how many more records must be gone through because -they don't have their own slots.) - -@subsection Fil Trailer - -The Fil Trailer has one part, as follows: -@*@* - -@multitable @columnfractions .10 .35 .40 - -@item @strong{Name} @tab @strong{Size} @tab @strong{Remarks} -@item FIL_PAGE_END_LSN -@tab 8 -@tab low 4 bytes = checksum of page, last 4 bytes = same as FIL_PAGE_LSN -@end multitable -@* - -The final part of a page, the fil trailer (or File Page Trailer), -exists because InnoDB's architect worried about integrity. It's -impossible for a page to be only half-written, or corrupted by -crashes, because the log-recovery mechanism restores to a consistent -state. But if something goes really wrong, then it's nice to have a -checksum, and to have a value at the very end of the page which must -be the same as a value at the very beginning of the page. - -@section Example - -For this example, I used Borland's TDUMP again, as I did for the earlier chapter on -Record Format. This is what a page looked like: -@*@* - -@multitable @columnfractions .05 .95 - -@item @strong{Address Values In Hexadecimal} @tab @strong{Values In ASCII} -@item @code{0D4000: 00 00 00 00 00 00 00 35 FF FF FF FF FF FF FF FF} -@tab @code{.......5........} -@item @code{0D4010: 00 00 00 00 00 00 E2 64 45 BF 00 00 00 00 00 00} -@tab @code{.......dE.......} -@item @code{0D4020: 00 00 00 00 00 00 00 05 02 F5 00 12 00 00 00 00} -@tab @code{................} -@item @code{0D4030: 02 E1 00 02 00 0F 00 10 00 00 00 00 00 00 00 00} -@tab @code{................} -@item @code{0D4040: 00 00 00 00 00 00 00 00 00 14 00 00 00 00 00 00} -@tab @code{................} -@item @code{0D4050: 00 02 16 B2 00 00 00 00 00 00 00 02 15 F2 08 01} -@tab @code{................} -@item @code{0D4060: 00 00 03 00 89 69 6E 66 69 6D 75 6D 00 09 05 00} -@tab @code{.....infimum....} -@item @code{0D4070: 08 03 00 00 73 75 70 72 65 6D 75 6D 00 22 1D 18} -@tab @code{....supremum."..} -@item @code{0D4080: 13 0C 06 00 00 10 0D 00 B7 00 00 00 00 04 14 00} -@tab @code{................} -@item @code{0D4090: 00 00 00 09 1D 80 00 00 00 2D 00 84 41 41 41 41} -@tab @code{.........-..AAAA} -@item @code{0D40A0: 41 41 41 41 41 41 41 41 41 41 41 1F 1B 17 13 0C} -@tab @code{AAAAAAAAAAA.....} -@item @code{ ... } -@item @code{ ... } -@item @code{0D7FE0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 74} -@tab @code{...............t} -@item @code{0D7FF0: 02 47 01 AA 01 0A 00 65 3A E0 AA 71 00 00 E2 64} -@tab @code{.G.....e:..q...d} -@end multitable -@*@* - -Let's skip past the first 38 bytes, which are Fil Header. The bytes -of the Page Header start at location @code{0d4026 hexadecimal}: -@*@* - -@multitable @columnfractions .10 .45 .60 - -@item @strong{Location} @tab @strong{Name} @tab @strong{Description} -@item @code{00 05} -@tab PAGE_N_DIR_SLOTS -@tab There are 5 directory slots. -@item @code{02 F5} -@tab PAGE_HEAP_TOP -@tab At location @code{0402F5}, not shown, is the beginning of free space. -Maybe a better name would have been PAGE_HEAP_END -@item @code{00 12} -@tab PAGE_N_HEAP -@tab There are 18 (hexadecimal 12) records in the page. -@item @code{00 00} -@tab PAGE_FREE -@tab There are zero free (deleted) records. -@item @code{00 00} -@tab PAGE_GARBAGE -@tab There are zero bytes in deleted records. -@item @code{02 E1} -@tab PAGE_LAST_INSERT -@tab The last record was inserted at location @code{02E1}, not shown, within the page. -@item @code{00 02} -@tab PAGE_DIRECTION -@tab A glance at page0page.h will tell you that 2 is the #defined value for PAGE_RIGHT. -@item @code{00 0F} -@tab PAGE_N_DIRECTION -@tab The last 15 (hexadecimal 0F) inserts were all done "to the right" -because I was inserting in ascending order. -@item @code{00 10} -@tab PAGE_N_RECS -@tab There are 16 (hexadecimal 10) user records. Notice that PAGE_N_RECS is -smaller than the earlier field, PAGE_N_HEAP. -@item @code{00 00 00 00 00 00 00} -@tab PAGE_MAX_TRX_ID -@item @code{00 00} -@tab PAGE_LEVEL -@tab Zero because this is a leaf page. -@item @code{00 00 00 00 00 00 00 14} -@tab PAGE_INDEX_ID -@tab This is index number 20. -@item @code{00 00 00 00 00 00 00 02 16 B2} -@tab PAGE_BTR_SEG_LEAF -@item @code{00 00 00 00 00 00 00 02 15 F2} -@tab PAGE_BTR_SEG_TOP -@end multitable -@* - -Immediately after the page header are the infimum and supremum -records. Looking at the "Values In ASCII" column in the hexadecimal -dump, you will see that the contents are in fact the words "infimum" -and "supremum" respectively. -@*@* - -Skipping past the User Records and the Free Space, many bytes later, -is the end of the 16KB page. The values shown there are the two trailers. -@itemize @bullet -@item -The first trailer (@code{00 74, 02 47, 01 AA, 01 0A, 00 65}) is the page -directory. It has 5 entries, because the header field PAGE_N_DIR_SLOTS -says there are 5. -@item -The next trailer (@code{3A E0 AA 71, 00 00 E2 64}) is the fil trailer. Notice -that the last four bytes, @code{00 00 E2 64}, appeared before in the fil -header. -@end itemize - -@section Where to Look For More Information - -@strong{References:} @* -The most relevant InnoDB source-code files are page0page.c, -page0page.ic, and page0page.h in \page directory. - -@node Files in MySQL Sources, Files in InnoDB Sources, InnoDB Page Structure, Top -@chapter Annotated List Of Files in the MySQL Source Code Distribution - -This is a description of the files that you get when you download the -source code of MySQL. This description begins with a list -of the main directories and a short comment about each one. Then, for -each directory, in alphabetical order, a longer description is -supplied. When a directory contains significant program files, a list of each C -program is given along with an explanation of its intended function. - -@section Directory Listing - -@strong{Directory -- Short Comment} -@itemize @bullet -@item -bdb -- The Berkeley Database table handler -@item -BitKeeper -- BitKeeper administration (not part of the source distribution) -@item -BUILD -- Frequently used build scripts -@item -Build-tools -- Build tools (not part of the source distribution) -@item -client -- Client library -@item -cmd-line-utils -- Command-line utilities (libedit and readline) -@item -dbug -- Fred Fish's dbug library -@item -Docs -- Preliminary documents about internals and new modules; will eventually be moved to the mysqldoc repository -@item -extra -- Some minor standalone utility programs -@item -heap -- The HEAP table handler -@item -include -- Header (*.h) files for most libraries; includes all header files distributed with the MySQL binary distribution -@item -innobase -- The Innobase (InnoDB) table handler -@item -libmysql -- For producing MySQL as a library (e.g. a Windows .DLL) -@item -libmysql_r -- For building a thread-safe libmysql library -@item -libmysqld -- The MySQL Server as an embeddable library -@item -man -- Some user-contributed manual pages -@item -myisam -- The MyISAM table handler -@item -myisammrg -- The MyISAM Merge table handler -@item -mysql-test -- A test suite for mysqld -@item -mysys -- MySQL system library (Low level routines for file access etc.) -@item -netware -- Files related to the Novell NetWare version of MySQL -@item -NEW-RPMS -- Directory to place RPMs while making a distribution -@item -os2 -- Routines for working with the OS/2 operating system -@item -pstack -- Process stack display (not currently used) -@item -regex -- Henry Spencer's Regular Expression library for support of REGEXP function -@item -SCCS -- Source Code Control System (not part of source distribution) -@item -scripts -- SQL batches, e.g. mysqlbug and mysql_install_db -@item -sql -- Programs for handling SQL commands; the "core" of MySQL -@item -sql-bench -- The MySQL benchmarks -@item -SSL -- Secure Sockets Layer; includes an example certification one can use to test an SSL (secure) database connection -@item -strings -- Library for C string routines, e.g. atof, strchr -@item -support-files -- Files used to build MySQL on different systems -@item -tests -- Tests in Perl and in C -@item -tools -- mysqlmanager.c (tool under development, not yet useful) -@item -VC++Files -- Includes this entire directory, repeated for VC++ (Windows) use -@item -vio -- Virtual I/O Library -@item -zlib -- Data compression library, used on Windows -@end itemize - -@subsection bdb - -The Berkeley Database table handler. -@*@* - -The Berkeley Database (BDB) is maintained by Sleepycat Software. -MySQL AB maintains only a few small patches to make BDB work -better with MySQL. -@*@* - -The documentation for BDB is available at -http://www.sleepycat.com/docs/. Since it's reasonably thorough -documentation, a description of the BDB program files is not included -in this document. -@*@* - -@subsection BitKeeper - -BitKeeper administration. -@*@* - -Bitkeeper administration is not part of the source distribution. This -directory may be present if you downloaded the MySQL source using -BitKeeper rather than via the mysql.com site. The files in the -BitKeeper directory are for maintenance purposes only -- they are not -part of the MySQL package. -@*@* - -The MySQL Reference Manual explains how to use Bitkeeper to get the -MySQL source. Please see @url{http://www.mysql.com/doc/en/Installing_source_tree.html} -for more information. -@*@* - -@subsection BUILD - -Frequently used build scripts. -@*@* - -This directory contains the build switches for compilation on various -platforms. There is a subdirectory for each set of options. The main -ones are: -@itemize @bullet -@item -alpha -@item -ia64 -@item -pentium (with and without debug or bdb, etc.) -@item -solaris -@end itemize -@*@* - -@subsection Build-tools - -Build tools. -@*@* - -Build-tools is not part of the source distribution. This directory -contains batch files for extracting, making directories, and making -programs from source files. There are several subdirectories with -different scripts -- for building Linux executables, for compiling, -for performing all build steps, and so on. -@*@* - -@subsection client - -Client library. -@*@* - -The client library includes mysql.cc (the source of the 'mysql' -executable) and other utilities. Most of the utilities are mentioned -in the MySQL Reference Manual. Generally these are standalone C -programs which one runs in "client mode", that is, they call the -server. -@*@* - -The C program files in the directory are: -@itemize @bullet -@item -get_password.c -- ask for a password from the console -@item -mysql.cc -- "The MySQL command tool" -@item -mysqladmin.c -- maintenance of MySQL databases -@item -mysqlcheck.c -- check all databases, check connect, etc. -@item -mysqldump.c -- dump table's contents as SQL statements, suitable to backup a MySQL database -@item -mysqlimport.c -- import text files in different formats into tables -@item -mysqlmanager-pwgen.c -- pwgen stands for "password generation" (not currently maintained) -@item -mysqlmanagerc.c -- entry point for mysql manager (not currently maintained) -@item -mysqlshow.c -- show databases, tables or columns -@item -mysqltest.c -- test program used by the mysql-test suite, mysql-test-run -@item -password.c -- password checking routines (version 4.1 and up) -@end itemize -@*@* - -@subsection cmd-line-utils - -Command-line utilities (libedit and readline). -@*@* - -There are two subdirectories: \readline and \libedit. All the files -here are "non-MySQL" files, in the sense that MySQL AB didn't produce -them, it just uses them. It should be unnecessary to study the -programs in these files unless you are writing or debugging a tty-like -client for MySQL, such as mysql.exe. -@*@* - -The \readline subdirectory contains the files of the GNU Readline -Library, "a library for reading lines of text with interactive input -and history editing". The programs are copyrighted by the Free -Software Foundation. -@*@* - -The \libedit (library of edit functions) subdirectory has files -written by Christos Zoulas. They are distributed and modifed under -the BSD License. These files are for editing the line contents. -@*@* - -These are the program files in the \libedit subdirectory: -@itemize @bullet -@item -chared.c -- character editor -@item -common.c -- common editor functions -@item -el.c -- editline interface functions -@item -emacs.c -- emacs functions -@item -fgetln.c -- get line -@item -hist.c -- history access functions -@item -history.c -- more history access functions -@item -key.c -- procedures for maintaining the extended-key map -@item -map.c -- editor function definitions -@item -parse.c -- parse an editline extended command -@item -prompt.c -- prompt printing functions -@item -read.c -- terminal read functions -@item -readline.c -- read line -@item -refresh.c -- "lower level screen refreshing functions" -@item -search.c -- "history and character search functions" -@item -sig.c -- for signal handling -@item -strlcpy.c -- string copy -@item -term.c -- "editor/termcap-curses interface" -@item -tokenizer.c -- Bourne shell line tokenizer -@item -tty.c -- for a tty interface -@item -vi.c -- commands used when in the vi (editor) mode -@end itemize -@*@* - -@subsection dbug - -Fred Fish's dbug library. -@*@* - -This is not really part of the MySQL package. Rather, it's a set of -public-domain routines which are useful for debugging MySQL programs. -The MySQL Server and all .c and .cc programs support the use of this -package. -@*@* - -How it works: One inserts a function call that begins with DBUG_* in -one of the regular MYSQL programs. For example, in get_password.c, you -will find this line: @* -DBUG_ENTER("get_tty_password"); @* -at the start of a routine, and this line: @* -DBUG_RETURN(my_strdup(to,MYF(MY_FAE))); @* -at the end of the routine. These lines don't affect production code. -Features of the dbug library include extensive reporting and profiling -(the latter has not been used by the MySQL team). -@*@* - -The C programs in this directory are: -@itemize @bullet -@item -dbug.c -- The main module -@item -dbug_analyze.c -- Reads a file produced by trace functions -@item -example1.c -- A tiny example -@item -example2.c -- A tiny example -@item -example3.c -- A tiny example -@item -factorial.c -- A tiny example -@item -main.c -- A tiny example -@item -sanity.c -- Declaration of a variable -@end itemize -@*@* - -@subsection Docs - -Preliminary documents about internals and new modules, which will eventually -be moved to the mysqldoc repository. -@*@* - -This directory doesn't have much at present that's very useful to the -student, but the plan is that some documentation related to the source -files and the internal workings of MySQL, including perhaps some -documentation from developers themselves, will be placed here. Files in -this directory will eventually be moved to the MySQL documentation repository. -@*@* - -These sub-directories are part of this directory: -@itemize @bullet -@item -books -- .gif images and empty .txt files; no real information -@item -flags -- images of flags of countries -@item -images -- flag backgrounds and the MySQL dolphin logo -@item -mysql-logos -- more MySQL-related logos, some of them moving -@item -raw-flags -- more country flags, all .gif files -@item -support -- various files for generating texinfo/docbook documentation -@item -to-be-included... -- contains a MySQL-for-dummies file -@item -translations -- some Portuguese myodbc documentation -@end itemize -@*@* - -In the main directory, you'll find some .txt files related to the -methods that MySQL uses to produce its printed and html documents, odd -bits in various languages, and the single file in the directory which -has any importance -- internals.texi -- The "MySQL Internals" -document. -@*@* - -Despite the name, internals.texi is not yet much of a description of MySQL -internals although work is in progress to make it so. However, there is -some useful description of the functions in the mysys directory (see below), -and of the structure of client/server messages (doubtless very useful for -eople who want to make their own JDBC drivers, or just sniff). -@*@* - -@subsection extra - -Some minor standalone utility programs. -@*@* - -These programs are all standalone utilities, that is, they have -a main() function and their main role is to show information that the -MySQL server needs or produces. Most are unimportant. They are as -follows: -@itemize @bullet -@item -my_print_defaults.c -- print parameters from my.ini files. Can also be used in scripts to enable processing of my.ini files. -@item -mysql_waitpid.c -- wait for a program to terminate. Useful for shell scripts when one needs to wait until a process terminates. -@item -perror.c -- "print error" -- given error number, display message -@item -replace.c -- replace strings in text files or pipe -@item -resolve_stack_dump.c -- show symbolic information from a MySQL stack dump, normally found in the mysql.err file -@item -resolveip.c -- convert an IP address to a hostname, or vice versa -@end itemize -@*@* - -@subsection heap - -The HEAP table handler. -@*@* - -All the MySQL table handlers (i.e. the handlers that MySQL itself -produces) have files with similar names and functions. Thus, this -(heap) directory contains a lot of duplication of the myisam directory -(for the MyISAM table handler). Such duplicates have been marked with -an "*" in the following list. For example, you will find that -\heap\hp_extra.c has a close equivalent in the myisam directory -(\myisam\mi_extra.c) with the same descriptive comment. (Some of the -differences arise because HEAP has different structures. HEAP does not -need to use the sort of B-tree indexing that ISAM and MyISAM use; instead -there is a hash index. Most importantly, HEAP is entirely in memory. -File-I/O routines lose some of their vitality in such a context.) -@*@* - -@itemize -@item -hp_block.c -- Read/write a block (i.e. a page) -@item -hp_clear.c -- Remove all records in the table -@item -hp_close.c -- * close database -@item -hp_create.c -- * create a table -@item -hp_delete.c -- * delete a row -@item -hp_extra.c -- * for setting options and buffer sizes when optimizing -@item -hp_hash.c -- Hash functions used for saving keys -@item -hp_info.c -- * Information about database status -@item -hp_open.c -- * open database -@item -hp_panic.c -- * the hp_panic routine, for shutdowns and flushes -@item -hp_rename.c -- * rename a table -@item -hp_rfirst.c -- * read first row through a specific key (very short) -@item -hp_rkey.c -- * read record using a key -@item -hp_rlast.c -- * read last row with same key as previously-read row -@item -hp_rnext.c -- * read next row with same key as previously-read row -@item -hp_rprev.c -- * read previous row with same key as previously-read row -@item -hp_rrnd.c -- * read a row based on position -@item -hp_rsame.c -- * find current row using positional read or key-based -read -@item -hp_scan.c -- * read all rows sequentially -@item -hp_static.c -- * static variables (very short) -@item -hp_test1.c -- * testing basic functions -@item -hp_test2.c -- * testing database and storing results -@item -hp_update.c -- * update an existing row -@item -hp_write.c -- * insert a new row -@end itemize -@*@* - -There are fewer files in the heap directory than in the myisam -directory, because fewer are necessary. For example, there is no need -for a \myisam\mi_cache.c equivalent (to cache reads) or a -\myisam\log.c equivalent (to log statements). -@*@* - -@subsection include - -Header (*.h) files for most libraries; includes all header files distributed -with the MySQL binary distribution. -@*@* - -These files may be included in C program files. Note that each -individual directory will also have its own *.h files, for including -in its own *.c programs. The *.h files in the include directory are -ones that might be included from more than one place. -@*@* - -For example, the mysys directory contains a C file named rijndael.c, -but does not include rijndael.h. The include directory contains -rijndael.h. Looking further, you'll find that rijndael.h is also -included in other places: by my_aes.c and my_aes.h. -@*@* - -The include directory contains 51 *.h (header) files. -@*@* - -@subsection innobase - -The Innobase (InnoDB) table handler. -@*@* - -A full description of these files can be found elsewhere in this -document. -@*@* - -@subsection libmysql - -The MySQL Library, Part 1. -@*@* - -The files here are for producing MySQL as a library (e.g. a Windows -DLL). The idea is that, instead of producing separate mysql (client) -and mysqld (server) programs, one produces a library. Instead of -sending messages, the client part merely calls the server part. -@*@* - -The libmysql files are split into three directories: libmysql (this -one), libmysql_r (the next one), and libmysqld (the next one after -that). -@*@* - -The "library of mysql" has some client-connection -modules. For example, as described in an earlier -section of this manual, there is a discussion of -libmysql/libmysql.c which sends packets from the -client to the server. Many of the entries in the -libmysql directory (and in the following libmysqld -directory) are 'symlinks' on Linux, that is, they -are in fact pointers to files in other directories. -@*@* - -The program files on this directory are: -@itemize @bullet -@item -conf_to_src.c -- has to do with charsets -@item -dll.c -- initialization of the dll library -@item -errmsg.c -- English error messages, compare \mysys\errors.c -@item -get_password.c -- get password -@item -libmysql.c -- the code that implements the MySQL API, i.e. the functions a client that wants to connect to MySQL will call -@item -manager.c -- initialize/connect/fetch with MySQL manager -@end itemize -@*@* - -@subsection libmysql_r - -The MySQL Library, Part 2. -@*@* - -There is only one file here, used to build a thread-safe libmysql library: -@itemize @bullet -@item -makefile.am -@end itemize -@*@* - -@subsection libmysqld - -The MySQL library, Part 3. -@*@* - -The Embedded MySQL Server Library. The product of libmysqld -is not a client/server affair, but a library. There is a wrapper -to emulate the client calls. The program files on this directory -are: -@itemize @bullet -@item -libmysqld.c -- The called side, compare the mysqld.exe source -@item -lib_vio.c -- Emulate the vio directory's communication buffer -@end itemize -@*@* - -@subsection man - -Some user-contributed manual pages -@*@* - -These are user-contributed "man" (manual) pages in a special markup -format. The format is described in a document with a heading like -"man page for man" or "macros to format man pages" which you can find -in a Linux directory or on the Internet. -@*@* - -@subsection myisam - -The MyISAM table handler. -@*@* - -The C files in this subdirectory come in six main groups: -@itemize @bullet -@item -ft*.c files -- ft stands for "Full Text", code contributed by Sergei Golubchik -@item -mi*.c files -- mi stands for "My Isam", these are the main programs for Myisam -@item -myisam*.c files -- for example, "myisamchk" utility routine functions source -@item -rt*.c files -- rt stands for "rtree", some code was written by Alexander Barkov -@item -sp*.c files -- sp stands for "spatial", some code was written by Ramil Kalimullin -@item -sort.c -- this is a single file that sorts keys for index-create purposes -@end itemize -@*@* - -The "full text" and "rtree" and "spatial" program sets are for special -purposes, so this document focuses only on the mi*.c "myisam" C -programs. They are: -@itemize @bullet -@item -mi_cache.c -- for reading records from a cache -@item -mi_changed.c -- a single routine for setting a "changed" flag (very short) -@item -mi_check.c -- for checking and repairing tables. Used by the myisamchk program and by the MySQL server. -@item -mi_checksum.c -- calculates a checksum for a row -@item -mi_close.c -- close database -@item -mi_create.c -- create a table -@item -mi_dbug.c -- support routines for use with "dbug" (see \dbug description) -@item -mi_delete.c -- delete a row -@item -mi_delete_all.c -- delete all rows -@item -mi_delete_table.c -- delete a table (very short) -@item -mi_dynrec.c -- functions to handle space-packed records and blobs -@item -mi_extra.c -- setting options and buffer sizes when optimizing -@item -mi_info.c -- return useful base information for an open table -@item -mi_key.c -- for handling keys -@item -mi_locking.c -- lock database -@item -mi_log.c -- save commands in a log file which myisamlog program can read. Can be used to exactly replay a set of changes to a table. -@item -mi_open.c -- open database -@item -mi_packrec.c -- read from a data file compresed with myisampack -@item -mi_page.c -- read and write pages containing keys -@item -mi_panic.c -- the mi_panic routine, probably for sudden shutdowns -@item -mi_range.c -- approximate count of how many records lie between two keys -@item -mi_rename.c -- rename a table -@item -mi_rfirst.c -- read first row through a specific key (very short) -@item -mi_rkey.c -- read a record using a key -@item -mi_rlast.c -- read last row with same key as previously-read row -@item -mi_rnext.c -- read next row with same key as previously-read row -@item -mi_rnext_same.c -- same as mi_rnext.c, but abort if the key changes -@item -mi_rprev.c -- read previous row with same key as previously-read row -@item -mi_rrnd.c -- read a row based on position -@item -mi_rsame.c -- find current row using positional read or key-based read -@item -mi_rsamepos.c -- positional read -@item -mi_scan.c -- read all rows sequentially -@item -mi_search.c -- key-handling functions -@item -mi_static.c -- static variables (very short) -@item -mi_statrec.c -- functions to handle fixed-length records -@item -mi_test1.c -- testing basic functions -@item -mi_test2.c -- testing database and storing results -@item -mi_test3.c -- testing locking -@item -mi_unique.c -- functions to check if a row is unique -@item -mi_update.c -- update an existing row -@item -mi_write.c -- insert a new row -@end itemize -@*@* - -@subsection myisammrg - -MyISAM Merge table handler. -@*@* - -As with other table handlers, you'll find that the *.c files in the -myissammrg directory have counterparts in the myisam directory. In -fact, this general description of a myisammrg program is almost always -true: The myisammrg -function checks an argument, the myisammrg function formulates an -expression for passing to a myisam function, the myisammrg calls a -myisam function, the myisammrg function returns. -@*@* - -These are the 21 files in the myisammrg directory, with notes about -the myisam functions or programs they're connected with: -@itemize @bullet -@item -myrg_close.c -- mi_close.c -@item -myrg_create.c -- mi_create.c -@item -myrg_delete.c -- mi_delete.c / delete last-read record -@item -myrg_extra.c -- mi_extra.c / "extra functions we want to do ..." -@item -myrg_info.c -- mi_info.c / display information about a mymerge file -@item -myrg_locking.c -- mi_locking.c / lock databases -@item -myrg_open.c -- mi_open.c / open a MyISAM MERGE table -@item -myrg_panic.c -- mi_panic.c / close in a hurry -@item -myrg_queue.c -- read record based on a key -@item -myrg_range.c -- mi_range.c / find records in a range -@item -myrg_rfirst.c -- mi_rfirst.c / read first record according to -specific key -@item -myrg_rkey.c -- mi_rkey.c / read record based on a key -@item -myrg_rlast.c -- mi_rlast.c / read last row with same key as previous -read -@item -myrg_rnext.c -- mi_rnext.c / read next row with same key as previous -read -@item -myrg_rnext_same.c -- mi_rnext_same.c / read next row with same key -@item -myrg_rprev.c -- mi_rprev.c / read previous row with same key -@item -myrg_rrnd.c -- mi_rrnd.c / read record with random access -@item -myrg_rsame.c -- mi_rsame.c / call mi_rsame function, see -\myisam\mi_rsame.c -@item -myrg_static.c -- mi_static.c / static variable declaration -@item -myrg_update.c -- mi_update.c / call mi_update function, see -\myisam\mi_update.c -@item -myrg_write.c -- mi_write.c / call mi_write function, see -\myisam\mi_write.c -@end itemize -@*@* - -@subsection mysql-test - -A test suite for mysqld. -@*@* - -The directory has a README file which explains how to run the tests, -how to make new tests (in files with the filename extension "*.test"), -and how to report errors. -@*@* - -There are four subdirectories: -@itemize @bullet -@item -\misc -- contains one minor Perl program -@item -\r -- contains *.result, i.e. "what happened" files and -*.required, i.e. "what should happen" file -@item -\std_data -- contains standard data for input to tests -@item -\t -- contains tests -@end itemize -@*@* - -There are 186 *.test files in the \t subdirectory. Primarily these are -SQL scripts which try out a feature, output a result, and compare the -result with what's required. Some samples of what the test files check -are: latin1_de comparisons, date additions, the HAVING clause, outer -joins, openSSL, load data, logging, truncate, and UNION. -@*@* - -There are other tests in these directories: -@itemize @bullet -@item -sql-bench -@item -tests -@end itemize - -@subsection mysys - -MySQL system library. Low level routines for file access and so on. -@*@* - -There are 115 *.c programs in this directory: -@itemize @bullet -@item -array.c -- Dynamic array handling -@item -charset.c -- Using dynamic character sets, set default character set, ... -@item -charset2html.c -- Check what character set a browser is using -@item -checksum.c -- Calculate checksum for a memory block, used for pack_isam -@item -default.c -- Find defaults from *.cnf or *.ini files -@item -errors.c -- English text of global errors -@item -hash.c -- Hash search/compare/free functions "for saving keys" -@item -list.c -- Double-linked lists -@item -make-conf.c -- "Make a charset .conf file out of a ctype-charset.c file" -@item -md5.c -- MD5 ("Message Digest 5") algorithm from RSA Data Security -@item -mf_brkhant.c -- Prevent user from doing a Break during critical execution (not used in MySQL; can be used by standalone MyISAM applications) -@item -mf_cache.c -- "Open a temporary file and cache it with io_cache" -@item -mf_dirname.c -- Parse/convert directory names -@item -mf_fn_ext.c -- Get filename extension -@item -mf_format.c -- Format a filename -@item -mf_getdate.c -- Get date, return in yyyy-mm-dd hh:mm:ss format -@item -mf_iocache.c -- Cached read/write of files in fixed-size units -@item -mf_iocache2.c -- Continuation of mf_iocache.c -@item -mf_keycache.c -- Key block caching for certain file types -@item -mf_loadpath.c -- Return full path name (no ..\ stuff) -@item -mf_pack.c -- Packing/unpacking directory names for create purposes -@item -mf_path.c -- Determine where a program can find its files -@item -mf_qsort.c -- Quicksort -@item -mf_qsort2.c -- Quicksort, part 2 (allows the passing of an extra argument to the sort-compare routine) -@item -mf_radix.c -- Radix sort -@item -mf_same.c -- Determine whether filenames are the same -@item -mf_sort.c -- Sort with choice of Quicksort or Radix sort -@item -mf_soundex.c -- Soundex algorithm derived from EDN Nov. 14, 1985 (pg. 36) -@item -mf_strip.c -- Strip trail spaces from a string -@item -mf_tempdir.c -- Initialize/find/free temporary directory -@item -mf_tempfile.c -- Create a temporary file -@item -mf_unixpath.c -- Convert filename to UNIX-style filename -@item -mf_util.c -- Routines, #ifdef'd, which may be missing on some -machines -@item -mf_wcomp.c -- Comparisons with wildcards -@item -mf_wfile.c -- Finding files with wildcards -@item -mulalloc.c -- Malloc many pointers at the same time -@item -my_aes.c -- AES encryption -@item -my_alarm.c -- Set a variable value when an alarm is received -@item -my_alloc.c -- malloc of results which will be freed simultaneously -@item -my_append.c -- one file to another -@item -my_bit.c -- smallest X where 2^X >= value, maybe useful for -divisions -@item -my_bitmap.c -- Handle uchar arrays as large bitmaps -@item -my_chsize.c -- Truncate file if shorter, else fill with a filler -character -@item -my_clock.c -- Time-of-day ("clock()") function, with OS-dependent -#ifdef's -@item -my_compress.c -- Compress packet (see also description of \zlib -directory) -@item -my_copy.c -- Copy files -@item -my_create.c -- Create file -@item -my_delete.c -- Delete file -@item -my_div.c -- Get file's name -@item -my_dup.c -- Open a duplicated file -@item -my_error.c -- Return formatted error to user -@item -my_fopen.c -- File open -@item -my_fstream.c -- Streaming file read/write -@item -my_getwd.c -- Get working directory -@item -my_gethostbyname.c -- Thread-safe version of standard net -gethostbyname() func -@item -my_getopt.c -- Find out what options are in effect -@item -my_handler.c -- Compare two keys in various possible formats -@item -my_init.c -- Initialize variables and functions in the mysys library -@item -my_lib.c -- Compare/convert directory names and file names -@item -my_lock.c -- Lock part of a file -@item -my_lockmem.c -- "Allocate a block of locked memory" -@item -my_lread.c -- Read a specified number of bytes from a file into -memory -@item -my_lwrite.c -- Write a specified number of bytes from memory into a -file -@item -my_malloc.c -- Malloc (memory allocate) and dup functions -@item -my_messnc.c -- Put out a message on stderr with "no curses" -@item -my_mkdir.c -- Make directory -@item -my_net.c -- Thread-safe version of net inet_ntoa function -@item -my_netware.c -- Functions used only with the Novell Netware version -of MySQL -@item -my_once.c -- Allocation / duplication for "things we don't need to -free" -@item -my_open.c -- Open a file -@item -my_os2cond.c -- OS2-specific: "A simple implementation of posix conditions" -@item -my_os2dirsrch.c -- OS2-specific: Emulate a Win32 directory search -@item -my_os2dlfcn.c -- OS2-specific: Emulate UNIX dynamic loading -@item -my_os2file64.c -- OS2-specific: For File64bit setting -@item -my_os2mutex.c -- OS2-specific: For mutex handling -@item -my_os2thread.c -- OS2-specific: For thread handling -@item -my_os2tls.c -- OS2-specific: For thread-local storage -@item -my_port.c -- OS/machine-dependent porting functions, e.g. AIX-specific my_ulonglong2double() -@item -my_pread.c -- Read a specified number of bytes from a file -@item -my_pthread.c -- A wrapper for thread-handling functions in different OSs -@item -my_quick.c -- Read/write (labelled a "quicker" interface, perhaps -obsolete) -@item -my_read.c -- Read a specified number of bytes from a file, possibly -retry -@item -my_realloc.c -- Reallocate memory allocated with my_alloc.c -(probably) -@item -my_redel.c -- Rename and delete file -@item -my_rename.c -- Rename without delete -@item -my_seek.c -- Seek, i.e. point to a spot within a file -@item -my_semaphore.c -- Semaphore routines, for use on OS that doesn't support them -@item -my_sleep.c -- Wait n microseconds -@item -my_static.c -- Static variables used by the mysys library -@item -my_symlink.c -- Read a symbolic link (symlinks are a UNIX thing, I guess) -@item -my_symlink2.c -- Part 2 of my_symlink.c -@item -my_tempnam.c -- Obsolete temporary-filename routine used by ISAM table handler -@item -my_thr_init.c -- initialize/allocate "all mysys & debug thread variables" -@item -my_wincond.c -- Windows-specific: emulate Posix conditions -@item -my_winsem.c -- Windows-specific: emulate Posix threads -@item -my_winthread.c -- Windows-specific: emulate Posix threads -@item -my_write.c -- Write a specified number of bytes to a file -@item -ptr_cmp.c -- Point to an optimal byte-comparison function -@item -queues.c -- Handle priority queues as in Robert Sedgewick's book -@item -raid2.c -- RAID support (the true implementation is in raid.cc) -@item -rijndael.c -- "Optimised ANSI C code for the Rijndael cipher (now AES") -@item -safemalloc.c -- A version of the standard malloc() with safety checking -@item -sha1.c -- Implementation of Secure Hashing Algorithm 1 -@item -string.c -- Initialize/append/free dynamically-sized strings; see also sql_string.cc in the /sql directory -@item -testhash.c -- Standalone program: test the hash library routines -@item -test_charset.c -- Standalone program: display character set information -@item -test_dir.c -- Standalone program: placeholder for "test all functions" idea -@item -test_fn.c -- Standalone program: apparently tests a function -@item -test_xml.c -- Standalone program: test XML routines -@item -thr_alarm.c -- Thread alarms and signal handling -@item -thr_lock.c -- "Read and write locks for Posix threads" -@item -thr_mutex.c -- A wrapper for mutex functions -@item -thr_rwlock.c -- Synchronizes the readers' thread locks with the writer's lock -@item -tree.c -- Initialize/search/free binary trees -@item -typelib.c -- Find a string in a set of strings; returns the offset to the string found -@end itemize -@*@* - -You can find documentation for the main functions in these files -elsewhere in this document. For example, the main functions in my_getwd.c -are described thus: -@*@* - -@example -"int my_getwd _A((string buf, uint size, myf MyFlags)); @* - int my_setwd _A((const char *dir, myf MyFlags)); @* - Get and set working directory." @* -@end example - -@subsection netware - -Files related to the Novell NetWare version of MySQL. -@*@* - -There are 39 files on this directory. Most have filename extensions of -*.def, *.sql, or *.c. -@*@* - -The twenty-five *.def files are all from Novell Inc. They contain import or -export symbols. (".def" is a common filename extension for -"definition".) -@*@* - -The two *.sql files are short scripts of SQL statements used in -testing. -@*@* - -These are the five *.c files, all from Novell Inc.: -@itemize @bullet -@item -libmysqlmain.c -- Only one function: init_available_charsets() -@item -my_manage.c -- Standalone management utility -@item -mysql_install_db.c -- Compare \scripts\mysql_install_db.sh -@item -mysql_test_run.c -- Short test program -@item -mysqld_safe.c -- Compare \scripts\mysqld_safe.sh -@end itemize - -Perhaps the most important file is: -@itemize @bullet -@item -netware.patch -- NetWare-specific build instructions and switches -(compare \mysql-4.1\ltmain.sh) -@end itemize -@*@* - -For instructions about basic installation, see "Deployment Guide For -NetWare AMP" at: -@url{http://developer.novell.com/ndk/whitepapers/namp.htm} -@* - -@subsection NEW-RPMS - -Directory to place RPMs while making a distribution. -@*@* - -This directory is not part of the Windows distribution. It is -a temporary directory used during RPM builds with Linux distributions. -@*@* - -@subsection os2 - -Routines for working with the OS2 operating system. -@*@* - -The files in this directory are the product of the efforts of three -people from outside MySQL: Yuri Dario, Timo Maier, and John M -Alfredsson. There are no .C program files in this directory. -@*@* - -The contents of \os2 are: -@itemize @bullet -@item -A Readme.Txt file -@item -An \include subdirectory containing .h files which are for OS/2 only -@item -Files used in the build process (configuration, switches, and one -.obj) -@end itemize -@*@* - -The README file refers to MySQL version 3.23, which suggests that -there have been no updates for MySQL 4.0 for this section. -@*@* - -@subsection pstack - -Process stack display (not currently used). -@*@* - -This is a set of publicly-available debugging aids which all do pretty -well the same thing: display the contents of the stack, along with -symbolic information, for a running process. There are versions for -various object file formats (such as ELF and IEEE-695). Most of the -programs are copyrighted by the Free Software Foundation and are -marked as "part of GNU Binutils". -@*@* - -In other words, the pstack files are not really part of the MySQL -library. They are merely useful when you re-program some MYSQL code -and it crashes. -@*@* - -@subsection regex - -Henry Spencer's Regular Expression library for support of REGEXP function. -@*@* - -This is the copyrighted product of Henry Spencer from the University -of Toronto. It's a fairly-well-known implementation of the -requirements of POSIX 1003.2 Section 2.8. The library is bundled with -Apache and is the default implementation for regular-expression -handling in BSD Unix. MySQL's Monty Widenius has made minor changes in -three programs (debug.c, engine.c, regexec.c) but this is not a MySQL -package. MySQL calls it only in order to support two MySQL functions: -REGEXP and RLIKE. -@*@* - -Some of Mr Spencer's documentation for the regex library can be found -in the README and WHATSNEW files. -@*@* - -One MySQL program which uses regex is \cmd-line-utils\libedit\search.c -@*@* - -This program calls the 'regcomp' function, which is the entry point in -\regex\regexp.c. -@*@* - -@subsection SCCS - -Source Code Control System (not part of source distribution). -@*@* - -You will see this directory if and only if you used BitKeeper for -downloading the source. The files here are for BitKeeper -administration and are not of interest to application programmers. -@*@* - -@subsection scripts - -SQL batches, e.g. mysqlbug and mysql_install_db. -@*@* - -The *.sh filename extension stands for "shell script". Linux -programmers use it where Windows programmers would use a *.bat -(batch filename extension). -@*@* - -The *.sh files on this directory are: -@itemize @bullet -@item -fill_help_tables.sh -- Create help-information tables and insert -@item -make_binary_distribution.sh -- Get configure information, make, produce tar -@item -msql2mysql.sh -- Convert (partly) mSQL programs and scripts to MySQL -@item -mysqlbug.sh -- Create a bug report and mail it -@item -mysqld_multi.sh -- Start/stop any number of mysqld instances -@item -mysqld_safe-watch.sh -- Start/restart in safe mode -@item -mysqld_safe.sh -- Start/restart in safe mode -@item -mysqldumpslow.sh -- Parse and summarize the slow query log -@item -mysqlhotcopy.sh -- Hot backup -@item -mysql_config.sh -- Get configuration information that might be needed to compile a client -@item -mysql_convert_table_format.sh -- Conversion, e.g. from ISAM to MyISAM -@item -mysql_explain_log.sh -- Put a log (made with --log) into a MySQL table -@item -mysql_find_rows.sh -- Search for queries containing -@item -mysql_fix_extensions.sh -- Renames some file extensions, not recommended -@item -mysql_fix_privilege_tables.sh -- Fix mysql.user etc. when upgrading. Can be safely run during any upgrade to get the newest -MySQL privilege tables -@item -mysql_install_db.sh -- Create privilege tables and func table -@item -mysql_secure_installation.sh -- Disallow remote root login, eliminate test, etc. -@item -mysql_setpermission.sh -- Aid to add users or databases, sets privileges -@item -mysql_tableinfo.sh -- Puts info re MySQL tables into a MySQL table -@item -mysql_zap.sh -- Kill processes which match pattern -@end itemize -@*@* - -@subsection sql - -Programs for handling SQL commands. The "core" of MySQL. -@*@* - -These are the .c and .cc files in the sql directory: -@itemize @bullet -@item -convert.cc -- convert tables between different character sets -@item -derror.cc -- read language-dependent message file -@item -des_key_file.cc -- load DES keys from plaintext file -@item -field.cc -- "implement classes defined in field.h" (long); defines all storage methods MySQL uses to store field information -into records that are then passed to handlers -@item -field_conv.cc -- functions to copy data between fields -@item -filesort.cc -- sort a result set, using memory or temporary files -@item -frm_crypt.cc -- contains only one short function: get_crypt_for_frm -@item -gen_lex_hash.cc -- Knuth's algorithm from Vol 3 Sorting and Searching, Chapter 6.3; used to search for SQL keywords in a query -@item -gstream.cc -- GTextReadStream, used to read GIS objects -@item -handler.cc -- handler-calling functions -@item -hash_filo.cc -- static-sized hash tables, used to store info like hostname -> ip tables in a FIFO manner -@item -ha_berkeley.cc -- Handler: BDB -@item -ha_heap.cc -- Handler: Heap -@item -ha_innodb.cc -- Handler: InnoDB -@item -ha_isam.cc -- Handler: ISAM -@item -ha_isammrg.cc -- Handler: (ISAM MERGE) -@item -ha_myisam.cc -- Handler: MyISAM -@item -ha_myisammrg.cc -- Handler: (MyISAM MERGE) -@item -hostname.cc -- Given IP, return hostname -@item -init.cc -- Init and dummy functions for interface with unireg -@item -item.cc -- Item functions -@item -item_buff.cc -- Buffers to save and compare item values -@item -item_cmpfunc.cc -- Definition of all compare functions -@item -item_create.cc -- Create an item. Used by lex.h. -@item -item_func.cc -- Numerical functions -@item -item_row.cc -- Row items for comparing rows and for IN on rows -@item -item_sum.cc -- Set functions (SUM, AVG, etc.) -@item -item_strfunc.cc -- String functions -@item -item_subselect.cc -- Item subselect -@item -item_timefunc.cc -- Date/time functions, e.g. week of year -@item -item_uniq.cc -- Empty file, here for compatibility reasons -@item -key.cc -- Functions to create keys from records and compare a key to a key in a record -@item -lock.cc -- Locks -@item -log.cc -- Logs -@item -log_event.cc -- Log event (a binary log consists of a stream of log events) -@item -matherr.c -- Handling overflow, underflow, etc. -@item -mf_iocache.cc -- Caching of (sequential) reads and writes -@item -mini_client.cc -- Client included in server for server-server messaging; used by the replication code -@item -mysqld.cc -- Source of mysqld.exe; includes the main() program that starts mysqld, handling of signals and connections -@item -my_lock.c -- Lock part of a file (like /mysys/my_lock.c, but with timeout handling for threads) -@item -net_serv.cc -- Read/write of packets on a network socket -@item -nt_servc.cc -- Initialize/register/remove an NT service -@item -opt_ft.cc -- Create a FT or QUICK RANGE based on a key (very short) -@item -opt_range.cc -- Range of keys -@item -opt_sum.cc -- Optimize functions in presence of (implied) GROUP BY -@item -password.c -- Password checking -@item -procedure.cc -- Procedure interface, as used in SELECT * FROM Table_name PROCEDURE ANALYSE -@item -protocol.cc -- Low level functions for PACKING data that is sent to client; actual sending done with net_serv.cc -@item -records.cc -- Functions for easy reading of records, possible through a cache -@item -repl_failsafe.cc -- Replication fail-save (not yet implemented) -@item -set_var.cc -- Set and retrieve MySQL user variables -@item -slave.cc -- Procedures for a slave in a master/slave (replication) relation -@item -spatial.cc -- Geometry stuff (lines, points, etc.) -@item -sql_acl.cc -- Functions related to ACL security; checks, stores, retrieves, and deletes MySQL user level privileges -@item -sql_analyse.cc -- Implements the PROCEDURE analyse, which analyses a query result and returns the 'optimal' data type for each result column -@item -sql_base.cc -- Basic functions needed by many modules, like opening and closing tables with table cache management -@item -sql_cache.cc -- SQL query cache, with long comments about how caching works -@item -sql_class.cc -- SQL class; implements the SQL base classes, of which THD (THREAD object) is the most important -@item -sql_crypt.cc -- Encode / decode, very short -@item -sql_db.cc -- Create / drop database -@item -sql_delete.cc -- The DELETE statement -@item -sql_derived.cc -- Derived tables, with long comments -@item -sql_do.cc -- The DO statement -@item -sql_error.cc -- Errors and warnings -@item -sql_handler.cc -- Implements the HANDLER interface, which gives direct access to rows in MyISAM and InnoDB -@item -sql_help.cc -- The HELP statement -@item -sql_insert.cc -- The INSERT statement -@item -sql_lex.cc -- Does lexical analysis of a query; i.e. breaks a query string into pieces and determines the basic type (number, -string, keyword, etc.) of each piece -@item -sql_list.cc -- Only list_node_end_of_list, short (the rest of the list class is implemented in sql_list.h) -@item -sql_load.cc -- The LOAD DATA statement -@item -sql_map.cc -- Memory-mapped files (not yet in use) -@item -sql_manager.cc -- Maintenance tasks, e.g. flushing the buffers periodically; used with BDB table logs -@item -sql_olap.cc -- ROLLUP -@item -sql_parse.cc -- Parse an SQL statement; do initial checks and then jump to the function that should execute the statement -@item -sql_prepare.cc -- Prepare an SQL statement, or use a prepared statement -@item -sql_repl.cc -- Replication -@item -sql_rename.cc -- Rename table -@item -sql_select.cc -- Select and join optimisation -@item -sql_show.cc -- The SHOW statement -@item -sql_string.cc -- String functions: alloc, realloc, copy, convert, etc. -@item -sql_table.cc -- The DROP TABLE and ALTER TABLE statements -@item -sql_test.cc -- Some debugging information -@item -sql_udf.cc -- User-defined functions -@item -sql_union.cc -- The UNION operator -@item -sql_update.cc -- The UPDATE statement -@item -stacktrace.c -- Display stack trace (Linux/Intel only) -@item -table.cc -- Table metadata retrieval; read the table definition from a .frm file and store it in a TABLE object -@item -thr_malloc.cc -- Thread-safe interface to /mysys/my_alloc.c -@item -time.cc -- Date and time functions -@item -udf_example.cc -- Example file of user-defined functions -@item -uniques.cc -- Function to handle quick removal of duplicates -@item -unireg.cc -- Create a unireg form file (.frm) from a FIELD and field-info struct -@end itemize -@*@* - -@subsection sql-bench - -The MySQL Benchmarks. -@*@* - -This directory has the programs and input files which MySQL uses for -its comparisons of MySQL, PostgreSQL, mSQL, Solid, etc. Since MySQL -publishes the comparative results, it's only right that it should make -available all the material necessary to reproduce all the tests. -@*@* - -There are five subdirectories and sub-subdirectories: -@itemize @bullet -@item -\Comments -- Comments about results from tests of Access, Adabas, etc. -@item -\Data\ATIS -- .txt files containing input data for the "ATIS" tests -@item -\Data\Wisconsin -- .txt files containing input data for the "Wisconsin" tests -@item -\Results -- old test results -@item -\Results-win32 -- old test results from Windows 32-bit tests -@end itemize -@*@* - -There are twenty-four *.sh (shell script) files, which involve Perl -programs. -@*@* - -There are three *.bat (batch) files. -@*@* - -There is one README file and one TODO file. -@*@* - -@subsection SSL - -Secure Sockets Layer; includes an example certification one can use -test an SSL (secure) database connection. -@*@* - -This isn't a code directory. It contains a short note from Tonu Samuel -(the NOTES file) and seven *.pem files. PEM stands for "Privacy -Enhanced Mail" and is an Internet standard for adding security to -electronic mail. Finally, there are two short scripts for running -clients and servers over SSL connections. -@*@* - -@subsection strings - -The string library. -@*@* - -Many of the files in this subdirectory are equivalent to well-known -functions that appear in most C string libraries. For those, there is -documentation available in most compiler handbooks. -@*@* - -On the other hand, some of the files are MySQL additions or -improvements. Often the MySQL changes are attempts to optimize the -standard libraries. It doesn't seem that anyone tried to optimize for -recent Pentium class processors, though. -@*@* - -The .C files are: -@itemize @bullet -@item -atof.c -- ascii-to-float, MySQL version -@item -bchange.c -- short replacement routine written by Monty Widenius in -1987 -@item -bcmp.c -- binary compare, rarely used -@item -bcopy-duff.c -- block copy: attempt to copy memory blocks faster -than cmemcpy -@item -bfill.c -- byte fill, to fill a buffer with (length) copies of a -byte -@item -bmove.c -- block move -@item -bmove512.c -- "should be the fastest way to move a multiple of 512 -bytes" -@item -bmove_upp.c -- bmove.c variant, starting with last byte -@item -bzero.c -- something like bfill with an argument of 0 -@item -conf_to_src.c -- reading a configuration file -@item -ctype*.c -- string handling programs for each char type MySQL -handles -@item -do_ctype.c -- display case-conversion and sort-conversion tables -@item -int2str.c -- integer-to-string -@item -is_prefix.c -- checks whether string1 starts with string2 -@item -llstr.c -- convert long long to temporary-buffer string, return -pointer -@item -longlong2str.c -- ditto, but to argument-buffer -@item -memcmp.c -- memory compare -@item -memset.c -- memory set -@item -my_vsnprintf.c -- variant of printf -@item -r_strinstr.c -- see if one string is within another -@item -str2int.c -- convert string to integer -@item -strappend.c -- fill up a string to n characters -@item -strcat.c -- concatenate strings -@item -strcend.c -- point to where a character C occurs within str, or NULL -@item -strchr.c -- point to first place in string where character occurs -@item -strcmp.c -- compare two strings -@item -strcont.c -- point to where any one of a set of characters appears -@item -strend.c -- point to the '\0' byte which terminates str -@item -strfill.c -- fill a string with n copies of a byte -@item -strinstr.c -- find string within string -@item -strlen.c -- return length of string in bytes -@item -strmake.c -- create new string from old string with fixed length, append end \0 if needed -@item -strmov.c -- move source to dest and return pointer to end -@item -strnlen.c -- return min(length of string, n) -@item -strnmov.c -- move source to dest for source size, or for n bytes -@item -strrchr.c -- find a character within string, searching from end -@item -strstr.c -- find an instance of pattern within source -@item -strto.c -- string to long, to long long, to unsigned long, etc. -@item -strtol.c -- string to long -@item -strtoll.c -- string to long long -@item -strtoul.c -- string to unsigned long -@item -strtoull.c -- string to unsigned long long -@item -strxmov.c -- move a series of concatenated source strings to dest -@item -strxnmov.c -- like strxmov.c but with a maximum length n -@item -str_test.c -- test of all the string functions encoded in assembler -@item -udiv.c -- unsigned long divide, for operating systems that don't support these -@item -xml.c -- read and parse XML strings; used to read character definition information stored in /sql/share/charsets -@end itemize -@*@* - -There are also four .ASM files -- macros.asm, ptr_cmp.asm, -strings.asm, and strxmov.asm -- which can replace some of the -C-program functions. But again, they look like optimizations for old -members of the Intel processor family. -@*@* - -@subsection support-files - -Files used to build MySQL on different systems. -@*@* - -The files here are for building ("making") MySQL given a package -manager, compiler, linker, and other build tools. The support files -provide instructions and switches for the build processes. They -include example my.cnf files one can use as a default setup for -MySQL. -@*@* - -@subsection tests - -Tests in Perl and in C. -@*@* - -The files in this directory are test programs that can be used -as a base to write a program to simulate problems in MySQL in various -scenarios: forks, locks, big records, exporting, truncating, and so on. -Some examples are: -@itemize @bullet -@item -connect_test.c -- test that a connect is possible -@item -insert_test.c -- test that an insert is possible -@item -list_test.c -- test that a select is possible -@item -select_test.c -- test that a select is possible -@item -showdb_test.c -- test that a show-databases is possible -@item -ssl_test.c -- test that SSL is possible -@item -thread_test.c -- test that threading is possible -@end itemize -@*@* - -@subsection tools - -Tools -- well, actually, one tool. -@*@* - -The only file is: -@itemize @bullet -@item -mysqlmanager.c -- A "server management daemon" by Sasha Pachev. This -is a tool under development and is not yet useful. Related to fail-safe -replication. -@end itemize -@*@* - -@subsection VC++Files - -Visual C++ Files. -@*@* - -Includes this entire directory, repeated for VC++ (Windows) use. -@*@* - -VC++Files includes a complete environment to compile MySQL with the VC++ -compiler. To use it, just copy the files on this directory; the make_win_src_distribution.sh -script uses these files to create a Windows source installation. -@*@* - -This directory has subdirectories which are copies of the main directories. -For example, there is a subdirectory \VC++Files\heap, which has the Microsoft -developer studio project file to compile \heap with VC++. So for a description -of the files in \VC++Files\heap, see the description of the files in \heap. The -same applies for almost all of VC++Files's subdirectories (bdb, client, -isam, libmysql, etc.). The difference is that the \VC++Files variants -are specifically for compilation with Microsoft Visual C++ in 32-bit -Windows environments. -@*@* - -In addition to the "subdirectories which are duplicates of -directories", VC++Files contains these subdirectories, which are not -duplicates: -@itemize @bullet -@item -comp_err -- (nearly empty) -@item -contrib -- (nearly empty) -@item -InstallShield -- script files -@item -isamchk -- (nearly empty) -@item -libmysqltest -- one small non-MySQL test program: mytest.c -@item -myisamchk -- (nearly empty) -@item -myisamlog -- (nearly empty) -@item -myisammrg -- (nearly empty) -@item -mysqlbinlog -- (nearly empty) -@item -mysqlmanager -- MFC foundation class files created by AppWizard -@item -mysqlserver -- (nearly empty) -@item -mysqlshutdown -- one short program, mysqlshutdown.c -@item -mysqlwatch.c -- Windows service initialization and monitoring -@item -my_print_defaults -- (nearly empty) -@item -pack_isam -- (nearly empty) -@item -perror -- (nearly empty) -@item -prepare -- (nearly empty) -@item -replace -- (nearly empty) -@item -SCCS -- source code control system -@item -test1 -- tests connecting via X threads -@item -thr_insert_test -- (nearly empty) -@item -thr_test -- one short program used to test for memory-allocation bug -@item -winmysqladmin -- the winmysqladmin.exe source -@end itemize -@*@* - -The "nearly empty" subdirectories noted above (e.g. comp_err and isamchk) -are needed because VC++ requires one directory per project (i.e. executable). -We are trying to keep to the MySQL standard source layout and compile only -to different directories. -@*@* - -@subsection vio - -Virtual I/O Library. -@*@* - -The VIO routines are wrappers for the various network I/O calls that -happen with different protocols. The idea is that in the main modules -one won't have to write separate bits of code for each protocol. Thus -vio's purpose is somewhat like the purpose of Microsoft's winsock -library. -@*@* - -The underlying protocols at this moment are: TCP/IP, Named Pipes (for -WindowsNT), Shared Memory, and Secure Sockets (SSL). -@*@* - -The C programs are: -@itemize @bullet -@item -test-ssl.c -- Short standalone test program: SSL -@item -test-sslclient.c -- Short standalone test program: clients -@item -test-sslserver.c -- Short standalone test program: server -@item -vio.c -- Declarations + open/close functions -@item -viosocket.c -- Send/retrieve functions -@item -viossl.c -- SSL variations for the above -@item -viosslfactories.c -- Certification / Verification -@item -viotest.cc -- Short standalone test program: general -@item -viotest-ssl.c -- Short standalone test program: SSL -@item -viotest-sslconnect.cc -- Short standalone test program: SSL connect -@end itemize -@*@* - -The older functions -- raw_net_read, raw_net_write -- are now -obsolete. -@*@* - -@subsection zlib - -Data compression library, used on Windows. -@*@* - -zlib is a data compression library used to support the compressed -protocol and the COMPRESS/UNCOMPRESS functions under Windows. -On Unix, MySQL uses the system libgz.a library for this purpose. -@*@* - -Zlib -- which presumably stands for "Zip Library" -- is not a MySQL -package. It was produced by the GNU Zip (gzip.org) people. Zlib is a -variation of the famous "Lempel-Ziv" method, which is also used by -"Zip". The method for reducing the size of any arbitrary string of -bytes is as follows: -@itemize @bullet -@item -Find a substring which occurs twice in the string. -@item -Replace the second occurrence of the substring with (a) a pointer to -the first occurrence, plus (b) an indication of the length of the -first occurrence. -@end itemize - -There is a full description of the library's functions in the gzip -manual at: @* -@url{http://www.gzip.org/zlib/manual.html} @* -There is therefore no need to list the modules in this document. -@*@* - -The MySQL program \mysys\my_compress.c uses zlib for packet compression. -The client sends messages to the server which are compressed by zlib. -See also: \sql\net_serv.cc. - -@node Files in InnoDB Sources, , Files in MySQL Sources, Top -@chapter Annotated List Of Files in the InnoDB Source Code Distribution - -ERRATUM BY HEIKKI TUURI (START) -@*@* - -Errata about InnoDB row locks:@*@* - -@example - #define LOCK_S 4 /* shared */ - #define LOCK_X 5 /* exclusive */ -... -@strong{/* Waiting lock flag */} - #define LOCK_WAIT 256 -/* this wait bit should be so high that it can be ORed to the lock -mode and type; when this bit is set, it means that the lock has not -yet been granted, it is just waiting for its turn in the wait queue */ -... -@strong{/* Precise modes */} - #define LOCK_ORDINARY 0 -/* this flag denotes an ordinary next-key lock in contrast to LOCK_GAP -or LOCK_REC_NOT_GAP */ - #define LOCK_GAP 512 -/* this gap bit should be so high that it can be ORed to the other -flags; when this bit is set, it means that the lock holds only on the -gap before the record; for instance, an x-lock on the gap does not -give permission to modify the record on which the bit is set; locks of -this type are created when records are removed from the index chain of -records */ - #define LOCK_REC_NOT_GAP 1024 -/* this bit means that the lock is only on the index record and does -NOT block inserts to the gap before the index record; this is used in -the case when we retrieve a record with a unique key, and is also used -in locking plain SELECTs (not part of UPDATE or DELETE) when the user -has set the READ COMMITTED isolation level */ - #define LOCK_INSERT_INTENTION 2048 -/* this bit is set when we place a waiting gap type record lock -request in order to let an insert of an index record to wait until -there are no conflicting locks by other transactions on the gap; note -that this flag remains set when the waiting lock is granted, or if the -lock is inherited to a neighboring record */ -@end example -@* - -ERRATUM BY HEIKKI TUURI (END) -@*@* - -The InnoDB source files are the best place to look for information -about internals of the file structure that MySQLites can optionally -use for transaction support. But when you first look at all the -subdirectories and file names you'll wonder: Where Do I Start? It can -be daunting. -@*@* - -Well, I've been through that phase, so I'll pass on what I had to -learn on the first day that I looked at InnoDB source files. I am very -sure that this will help you grasp, in overview, the organization of -InnoDB modules. I'm also going to add comments about what is going on --- which you should mistrust! These comments are reasonable working -hypotheses; nevertheless, they have not been subjected to expert peer -review. -@*@* - -Here's how I'm going to organize the discussion. I'll take each of the -32 InnoDB subdirectories that come with the MySQL 4.0 source code in -\mysql\innobase (on my Windows directory). The format of each section -will be like this every time: -@*@* - -@strong{\subdirectory-name (LONGER EXPLANATORY NAME)}@* -@multitable @columnfractions .10 .20 .40 .50 -@item @strong{File Name} @tab @strong{What Name Stands For} @tab @strong{Size} @tab @strong{Comment Inside File} -@item file-name -@tab my-own-guess -@tab in-bytes -@tab from-the-file-itself -@end multitable -...@* -My-Comments@* -@* - -For example: @* -@example -" -@strong{\ha (HASHING)} - File Name What Name Stands For Size Comment Inside File - --------- -------------------- ------ ------------------- - ha0ha.c Hashing/Hashing 7,452 Hash table with external chains - - Comments about hashing will be here. -" -@end example -@* - -The "Comment Inside File" column is a direct copy from the first /* -comment */ line inside the file. All other comments are mine. After -I've discussed each directory, I'll finish with some notes about -naming conventions and a short list of URLs that you can use for -further reference. -@*@* - -Now let's begin. -@*@* - -@example - -@strong{\ha (HASHING)} - File Name What Name Stands For Size Comment Inside File - --------- -------------------- ------ ------------------- - ha0ha.c Hashing / Hashing 7,452 Hash table with external chains - -I'll hold my comments until the next section, \hash (HASHING). - -@strong{\hash (HASHING)} - File Name What Name Stands For Size Comment Inside File - --------- -------------------- ------ ------------------- - hash0hash.c Hashing / Hashing 3,257 Simple hash table utility - -The two C programs in the \ha and \hashing directories -- ha0ha.c and -hash0hash.c -- both refer to a "hash table" but hash0hash.c is -specialized, it is mostly about accessing points in the table under -mutex control. - -When a "database" is so small that InnoDB can load it all into memory -at once, it's more efficient to access it via a hash table. After all, -no disk i/o can be saved by using an index lookup, if there's no disk. - -@strong{\os (OPERATING SYSTEM)} - File Name What Name Stands For Size Comment Inside File - --------- -------------------- ------ ------------------- - os0shm.c OS / Shared Memory 3,150 To shared memory primitives - os0file.c OS / File 64,412 To i/o primitives - os0thread.c OS / Thread 6,827 To thread control primitives - os0proc.c OS / Process 3,700 To process control primitives - os0sync.c OS / Synchronization 10,208 To synchronization primitives - -This is a group of utilities that other modules may call whenever they -want to use an operating-system resource. For example, in os0file.c -there is a public InnoDB function named os_file_create_simple(), which -simply calls the Windows-API function CreateFile. Naturally the -contents of this group are somewhat different for other operating systems. - -The "Shared Memory" functions in os0shm.c are only called from the -communications program com0shm.c (see \com COMMUNICATIONS). The i/o -and thread-control primitives are called extensively. The word -"synchronization" in this context refers to the mutex-create and -mutex-wait functionality. - -@strong{\ut (UTILITIES)} - File Name What Name Stands For Size Comment Inside File - --------- -------------------- ------ ------------------- - ut0ut.c Utilities / Utilities 7,041 Various utilities - ut0byte.c Utilities / Debug 1,856 Byte utilities - ut0rnd.c Utilities / Random 1,475 Random numbers and hashing - ut0mem.c Utilities / Memory 5,530 Memory primitives - ut0dbg.c Utilities / Debug 642 Debug utilities - -The two functions in ut0byte.c are just for lower/upper case -conversion and comparison. The single function in ut0rnd.c is for -finding a prime slightly greater than the given argument, which is -useful for hash functions, but unrelated to randomness. The functions -in ut0mem.c are wrappers for "malloc" and "free" calls -- for the -real "memory" module see section \mem (MEMORY). Finally, the -functions in ut0ut.c are a miscellany that didn't fit better elsewhere: -get_high_bytes, clock, time, difftime, get_year_month_day, and "sprintf" -for various diagnostic purposes. - -In short: the \ut group is trivial. - -@strong{\buf (BUFFERING)} - File Name What Name Stands For Size Comment Inside File - --------- -------------------- ------ ------------------- - buf0buf.c Buffering / Buffering 53,246 The database buffer buf_pool - buf0flu.c Buffering / Flush 23,711 ... flush algorithm - buf0lru.c / least-recently-used 20,245 ... replacement algorithm - buf0rea.c Buffering / read 17,399 ... read - -There is a separate file group (\mem MEMORY) which handles memory -requests in general.A "buffer" usually has a more specific -definition, as a memory area which contains copies of pages that -ordinarily are in the main data file. The "buffer pool" is the set -of all buffers (there are lots of them because InnoDB doesn't -depend on the OS's caching to make things faster). - -The pool size is fixed (at the time of this writing) but the rest of -the buffering architecture is sophisticated, involving a host of -control structures. In general: when InnoDB needs to access a new page -it looks first in the buffer pool; InnoDB reads from disk to a new -buffer when the page isn't there; InnoDB chucks old buffers (basing -its decision on a conventional Least-Recently-Used algorithm) when it -has to make space for a new buffer. - -There are routines for checking a page's validity, and for read-ahead. -An example of "read-ahead" use: if a sequential scan is going on, then -a DBMS can read more than one page at a time, which is efficient -because reading 32,768 bytes (two pages) takes less than twice as long -as reading 16,384 bytes (one page). - -@strong{\btr (B-TREE)} - File Name What Name Stands For Size Comment Inside File - --------- -------------------- ------ ------------------- - btr0btr.c B-tree / B-tree 74,255 B-tree - btr0cur.c B-tree / Cursor 94,950 index tree cursor - btr0sea.c B-tree / Search 36,580 index tree adaptive search - btr0pcur.c B-tree / persistent cursor 14,548 index tree persistent cursor - -If you total up the sizes of the C files, you'll see that \btr is the -second-largest file group in InnoDB. This is understandable because -maintaining a B-tree is a relatively complex task. Luckily, there has -been a lot of work done to describe efficient management of B-tree and -B+-tree structures, much of it open-source or public-domain, since -their original invention over thirty years ago. - -InnoDB likes to put everything in B-trees. This is what I'd call a -"distinguishing characteristic" because in all the major DBMSs (like -IBM DB2, Microsoft SQL Server, and Oracle), the main or default or -classic structure is the heap-and-index. In InnoDB the main structure -is just the index. To put it another way: InnoDB keeps the rows in the -leaf node of the index, rather than in a separate file. Compare -Oracle's Index Organized Tables, and Microsoft SQL Server's Clustered -Indexes. - -This, by the way, has some consequences. For example, you may as well -have a primary key since otherwise InnoDB will make one anyway. And -that primary key should be the shortest of the candidate keys, since -InnoDB -will use it as a pointer if there are secondary indexes. - -Most importantly, it means that rows have no fixed address. Therefore -the routines for managing file pages should be good. We'll see about -that when we look at the \row (ROW) program group later. - -@strong{\com (COMMUNCATION)} - File Name What Name Stands For Size Comment Inside File - --------- -------------------- ------ ------------------- - com0com.c Communication 6,913 Communication primitives - com0shm.c Communication / 24,633 ... through shared memory - Shared Memory - -The communication primitives in com0com.c are said to be modelled -after the ones in Microsoft's winsock library (the Windows Sockets -interface). The communication primitives in com0shm.c are at a -slightly lower level, and are called from the routines in com0com.c. - -I was interested in seeing how InnoDB would handle inter-process -communication, since there are many options -- named pipes, TCP/IP, -Windows messaging, and Shared Memory being the main ones that come to -mind. It appears that InnoDB prefers Shared Memory. The main idea is: -there is an area of memory which two different processes (or threads, -of course) can both access. To communicate, a thread gets an -appropriate mutex, puts in a request, and waits for a response. Thread -interaction is also a subject for the os0thread.c program in another -program group, \os (OPERATING SYSTEM). - -@strong{\dyn (DYNAMICALLY ALLOCATED ARRAY)} - File Name What Name Stands For Size Comment Inside File - --------- -------------------- ------ ------------------- - dyn0dyn.c Dynamic / Dynamic 994 dynamically allocated array - -There is a single function in the dyn0dyn.c program, for adding a -block to the dynamically allocated array. InnoDB might use the array -for managing concurrency between threads. - -At the moment, the \dyn program group is trivial. - -@strong{\fil (FILE)} - File Name What Name Stands For Size Comment Inside File - --------- -------------------- ------ ------------------- - fil0fil.c File / File 39,725 The low-level file system - -The reads and writes to the database files happen here, in -co-ordination with the low-level file i/o routines (see os0file.h in -the \os program group). - -Briefly: a table's contents are in pages, which are in files, which -are in tablespaces. Files do not grow; instead one can add new files -to the tablespace. As we saw earlier (discussing the \btr program group) -the pages are nodes of B-trees. Since that's the case, new additions can -happen at various places in the logical file structure, not -necessarily at the end. Reads and writes are asynchronous, and go into -buffers, which are set up by routines in the \buf program group. - -@strong{\fsp (FILE SPACE)} - File Name What Name Stands For Size Comment Inside File - --------- -------------------- ------ ------------------- - fsp0fsp.c File Space Management 100,271 File space management - -I would have thought that the \fil (FILE) and \fsp (FILE SPACE) -MANAGEMENT programs would fit together in the same program group; -however, I guess the InnoDB folk are splitters rather than lumpers. - -It's in fsp0fsp.c that one finds some of the descriptions and comments -of extents, segments, and headers. For example, the "descriptor bitmap -of the pages in the extent" is in here, and you can find as well how -the free-page list is maintained, what's in the bitmaps, and what -various header fields' contents are. - -@strong{\fut (FILE UTILITY)} - File Name What Name Stands For Size Comment Inside File - --------- -------------------- ------ ------------------- - fut0fut.c File Utility / Utility 293 File-based utilities - fut0lst.c File Utility / List 14,129 File-based list utilities - -Mainly these small programs affect only file-based lists, so maybe -saying "File Utility" is too generic. The real work with data files -goes on in the \fsp program group. - -@strong{\log (LOGGING)} - File Name What Name Stands For Size Comment Inside File - --------- -------------------- ------ ------------------- - log0log.c Logging / Logging 77,834 Database log - log0recv.c Logging / Recovery 80,701 Recovery - -I've already written about the \log program group, so here's a link to -my previous article: "How Logs work with MySQL and InnoDB": -@url{http://www.devarticles.com/art/1/181/2} - -@strong{\mem (MEMORY)} - File Name What Name Stands For Size Comment Inside File - --------- -------------------- ------ ------------------- - mem0mem.c Memory / Memory 9,971 The memory management - mem0dbg.c Memory / Debug 21,297 ... the debug code - mem0pool.c Memory / Pool 16,293 ... the lowest level - -There is a long comment at the start of the mem0pool.c program, which -explains what the memory-consumers are, and how InnoDB tries to -satisfy them. The main thing to know is that there are really three -pools: the buffer pool (see the \buf program group), the log pool (see the \log -program group), and the common pool, which is where everything that's -not in the buffer or log pools goes (for example the parsed SQL -statements and the data dictionary cache). - -@strong{\mtr (MINI-TRANSACTION)} - File Name What Name Stands For Size Comment Inside File - --------- -------------------- ------ ------------------- - mtr0mtr.c Mini-transaction / 12,433 Mini-transaction buffer - mtr0log.c Mini-transaction / Log 8,180 ... log routines - -The mini-transaction routines are called from most of the other -program groups. I'd describe this as a low-level utility set. - -@strong{\que (QUERY GRAPH)} - File Name What Name Stands For Size Comment Inside File - --------- -------------------- ------ ------------------- - que0que.c Query Graph / Query 35,964 Query graph - -The program que0que.c ostensibly is about the execution of stored -procedures which contain commit/rollback statements. I took it that -this has little importance for the average MySQL user. - -@strong{\rem (RECORD MANAGER)} - File Name What Name Stands For Size Comment Inside File - --------- -------------------- ------ ------------------- - rem0rec.c Record Manager 14,961 Record Manager - rem0cmp.c Record Manager / 25,263 Comparison services for records - Comparison - -There's an extensive comment near the start of rem0rec.c title -"Physical Record" and it's recommended reading. At some point you'll -ask what are all those bits that surround the data in the rows on a page, -and this is where you'll find the answer. - -@strong{\row (ROW)} - File Name What Name Stands For Size Comment Inside File - --------- -------------------- ------ ------------------- - row0row.c Row / Row 16,764 General row routines - row0uins.c Row / Undo Insert 7,199 Fresh insert undo - row0umod.c Row / Undo Modify 17,147 Undo modify of a row - row0undo.c Row / Undo 10,254 Row undo - row0vers.c Row / Version 12,288 Row versions - row0mysql.c Row / MySQL 63,556 Interface [to MySQL] - row0ins.c Row / Insert 42,829 Insert into a table - row0sel.c Row / Select 85,923 Select - row0upd.c Row / Update 44,456 Update of a row - row0purge.c Row / Purge 14,961 Purge obsolete records - -Rows can be selected, inserted, updated/deleted, or purged (a -maintenance activity). These actions have ancillary actions, for -example after insert there can be an index-update test, but it seems -to me that sometimes the ancillary action has no MySQL equivalent (yet) -and so is inoperative. - -Speaking of MySQL, notice that one of the larger programs in the \row -program group is the "interface between Innobase row operations and -MySQL" (row0mysql.c) -- information interchange happens at this level -because rows in InnoDB and in MySQL are analogous, something which -can't be said for pages and other levels. - -@strong{\srv (Server)} - File Name What Name Stands For Size Comment Inside File - --------- -------------------- ------ ------------------- - srv0srv.c Server / Server 79,058 Server main program - srv0que.c Server / Query 2,361 Server query execution - srv0start.c Server / Start 34,586 Starts the server - -This is where the server reads the initial configuration files, splits -up the threads, and gets going. There is a long comment deep in the -program (you might miss it at first glance) titled "IMPLEMENTATION OF -THE SERVER MAIN PROGRAM" in which you'll find explanations about -thread priority, and about what the responsibiities are for various -thread types. - -InnoDB has many threads, for example "user threads" (which wait for -client requests and reply to them), "parallel communication threads" -(which take part of a user thread's job if a query process can be -split), "utility threads" (background priority), and a "master thread" -(high priority, usually asleep). - -@strong{\thr (Thread Local Storage)} - File Name What Name Stands For Size Comment Inside File - --------- -------------------- ------ ------------------- - thr0loc.c Thread / Local 5,261 The thread local storage - -InnoDB doesn't use the Windows-API thread-local-storage functions, -perhaps because they're not portable enough. - -@strong{\trx (Transaction)} - File Name What Name Stands For Size Comment Inside File - --------- -------------------- ------ ------------------- - trx0trx.c Transaction / 37,447 The transaction - trx0purge.c Transaction / Purge 26,782 ... Purge old versions - trx0rec.c Transaction / Record 36,525 ... Undo log record - trx0sys.c Transaction / System 20,671 ... System - trx0rseg.c / Rollback segment 6,214 ... Rollback segment - trx0undo.c Transaction / Undo 46,595 ... Undo log - -InnoDB's transaction management is supposedly "in the style of Oracle" -and that's close to true but can mislead you. -@itemize -@item -First: InnoDB uses rollback segments like Oracle8i does -- but -Oracle9i uses a different name -@item -Second: InnoDB uses multi-versioning like Oracle does -- but I see -nothing that looks like an Oracle ITL being stored in the InnoDB data -pages. -@item -Third: InnoDB and Oracle both have short (back-to-statement-start) -versioning for the READ COMMITTED isolation level and long -(back-to-transaction-start) versioning for higher levels -- but InnoDB -and Oracle have different "default" isolation levels. -@item -Finally: InnoDB's documentation says it has to lock "the gaps before -index keys" to prevent phantoms -- but any Oracle user will tell you that -phantoms are impossible anyway at the SERIALIZABLE isolation level, so -key-locks are unnecessary. -@end itemize - -The main idea, though, is that InnoDB has multi-versioning. So does -Oracle. This is very different from the way that DB2 and SQL Server do -things. - -@strong{\usr (USER)} - File Name What Name Stands For Size Comment Inside File - --------- -------------------- ------ ------------------- - usr0sess.c User / Session 27,415 Sessions - -One user can have multiple sessions (the session being all the things -that happen betweeen a connect and disconnect). This is where InnoDB -tracks session IDs, and server/client messaging. It's another of those -items which is usually MySQL's job, though. - -@strong{\data (DATA)} - File Name What Name Stands For Size Comment Inside File - --------- -------------------- ------ ------------------- - data0data.c Data / Data 26,002 SQL data field and tuple - data0type.c Data / Type 2,122 Data types - -This is a collection of minor utility routines affecting rows. - -@strong{\dict (DICTIONARY)} - File Name What Name Stands For Size Comment Inside File - --------- -------------------- ------ ------------------- - dict0dict.c Dictionary / Dictionary 84,667 Data dictionary system - dict0boot.c Dictionary / boot 12,134 ... creation and booting - dict0load.c Dictionary / load 26,546 ... load to memory cache - dict0mem.c Dictionary / memory 8,221 ... memory object creation - -The data dictionary (known in some circles as the catalog) has the -metadata information about objects in the database -- column sizes, -table names, and the like. - -@strong{\eval (EVALUATING)} - File Name What Name Stands For Size Comment Inside File - --------- -------------------- ------ ------------------- - eval0eval.c Evaluating/Evaluating 15,682 SQL evaluator - eval0proc.c Evaluating/Procedures 5,000 Executes SQL procedures - -The evaluating step is a late part of the process of interpreting an -SQL statement -- parsing has already occurred during \pars (PARSING). - -The ability to execute SQL stored procedures is an InnoDB feature, but -not a MySQL feature, so the eval0proc.c program is unimportant. - -@strong{\ibuf (INSERT BUFFER)} - File Name What Name Stands For Size Comment Inside File - --------- -------------------- ------ ------------------- - ibuf0ibuf.c Insert Buffer / 69,884 Insert buffer - -The words "Insert Buffer" mean not "buffer used for INSERT" but -"insertion of a buffer into the buffer pool" (see the \buf BUFFER -program group description). The matter is complex due to possibilities -for deadlocks, a problem to which the comments in the ibuf0ibuf.c -program devote considerable attention. - -@strong{\mach (MACHINE FORMAT)} - File Name What Name Stands For Size Comment Inside File - --------- -------------------- ------ ------------------- - mach0data.c Machine/Data 2,319 Utilities for converting - -The mach0data.c program has two small routines for reading compressed -ulints (unsigned long integers). - -@strong{\lock (LOCKING)} - File Name What Name Stands For Size Comment Inside File - --------- -------------------- ------ ------------------- - lock0lock.c Lock / Lock 127,646 The transaction lock system - -If you've used DB2 or SQL Server, you might think that locks have their -own in-memory table, that row locks might need occasional escalation to -table locks, and that there are three lock types: Shared, Update, Exclusive. - -All those things are untrue with InnoDB! Locks are kept in the database -pages. A bunch of row locks can't be rolled together into a single table -lock. And most importantly there's only one lock type. I call this type -"Update" because it has the characteristics of DB2 / SQL Server Update -locks, that is, it blocks other updates but doesn't block reads. -Unfortunately, InnoDB comments refer to them as "x-locks" etc. - -To sum it up: if your background is Oracle you won't find too much -surprising, but if your background is DB2 or SQL Server the locking -concepts and terminology will probably confuse you at first. - -You can find an online article about the differences between -Oracle-style and DB2/SQL-Server-style locks at: -@url{http://dbazine.com/gulutzan6.html} - -@strong{\odbc (ODBC)} - File Name What Name Stands For Size Comment Inside File - --------- -------------------- ------ ------------------- - odbc0odbc.c ODBC / ODBC 16,865 ODBC client library - -The odbc0odbc.c program has a small selection of old ODBC-API -functions: SQLAllocEnv, SQLAllocConnect, SQLAllocStmt, SQLConnect, -SQLError, SQLPrepare, SQLBindParameter, SQLExecute. - -@strong{\page (PAGE)} - File Name What Name Stands For Size Comment Inside File - --------- -------------------- ------ ------------------- - page0page.c Page / Page 44,309 Index page routines - page0cur.c Page / Cursor 30,305 The page cursor - -It's in the page0page.c program that you'll learn as follows: index -pages start with a header, entries in the page are in order, at the -end of the page is a sparse "page directory" (what I would have called -a slot table) which makes binary searches easier. - -Incidentally, the program comments refer to "a page size of 8 kB" -which seems obsolete. In univ.i (a file containing universal -constants) the page size is now #defined as 16KB. - -@strong{\pars (PARSING)} - File Name What Name Stands For Size Comment Inside File - --------- -------------------- ------ ------------------- - pars0pars.c Parsing/Parsing 49,947 SQL parser - pars0grm.c Parsing/Grammar 62,685 A Bison parser - pars0opt.c Parsing/Optimizer 30,809 Simple SQL Optimizer - pars0sym.c Parsing/Symbol Table 5,541 SQL parser symbol table - lexyy.c ?/Lexer 59,948 Lexical scanner - -The job is to input a string containing an SQL statement and output an -in-memory parse tree. The EVALUATING (subdirectory \eval) programs -will use the tree. - -As is common practice, the Bison and Flex tools were used -- pars0grm.c -is what the Bison parser produced from an original file named pars0grm.y -(not supplied), and lexyy.c is what Flex produced. - -Since InnoDB is a DBMS by itself, it's natural to find SQL parsing in -it. But in the MySQL/InnoDB combination, MySQL handles most of the -parsing. These files are unimportant. - -@strong{\read (READ)} - File Name What Name Stands For Size Comment Inside File - --------- -------------------- ------ ------------------- - read0read.c Read / Read 6,244 Cursor read - -The read0read.c program opens a "read view" of a query result, using -some functions in the \trx program group. - -@strong{\sync (SYNCHRONIZATION)} - File Name What Name Stands For Size Comment Inside File - --------- -------------------- ------ ------------------- - sync0sync.c Synchronization / 35,918 Mutex, the basic sync primitive - sync0arr.c ... / array 26,461 Wait array used in primitives - sync0ipm.c ... / interprocess 4,027 for interprocess sync - sync0rw.c ... / read-write 22,220 read-write lock for thread sync - -A mutex (Mutual Exclusion) is an object which only one thread/process -can hold at a time. Any modern operating system API has some functions -for mutexes; however, as the comments in the sync0sync.c code indicate, it -can be faster to write one's own low-level mechanism. In fact the old -assembly-language XCHG trick is in here -- this is the only program -that contains any assembly code. -@end example -@* -@* - -This is the end of the section-by-section account of InnoDB -subdirectories. -@*@* - -@strong{A Note About File Naming} @*@* - -There appears to be a naming convention. The first letters of the file -name are the same as the subdirectory name, then there is a '0' -separator, then there is an individual name. For the main program in a -subdirectory, the individual name may be a repeat of the subdirectory -name. For example, there is a file named ha0ha.c (the first two -letters ha mean "it's in in subdirectory ..\ha", the next letter 0 -means "0 separator", the next two letters mean "this is the main ha -program"). This naming convention is not strict, though: for example -the file lexyy.c is in the \pars subdirectory. -@*@* - -@strong{A Note About Copyrights} @*@* - -Most of the files begin with a copyright notice or a creation date, -for example "Created 10/25/1995 Heikki Tuuri". I don't know a great -deal about the history of InnoDB, but found it interesting that most -creation dates were between 1994 and 1998. -@*@* - -@strong{References} @*@* - -Ryan Bannon, Alvin Chin, Faryaaz Kassam and Andrew Roszko @* -"InnoDB Concrete Architecture" @* -@url{http://www.swen.uwaterloo.ca/~mrbannon/cs798/assignment_02/innodb.pdf} - -A student paper. It's an interesting attempt to figure out InnoDB's -architecture using tools, but I didn't end up using it for the specific -purposes of this article. -@*@* - -Peter Gulutzan @* -"How Logs Work With MySQL And InnoDB" @* -@url{http://www.devarticles.com/art/1/181/2} -@*@* - -Heikki Tuuri @* -"InnoDB Engine in MySQL-Max-3.23.54 / MySQL-4.0.9: The Up-to-Date -Reference Manual of InnoDB" @* -@url{http://www.innodb.com/ibman.html} - -This is the natural starting point for all InnoDB information. Mr -Tuuri also appears frequently on MySQL forums. -@*@* - -@summarycontents -@contents - -@bye From 3b974039f7ba1a35c9767ed0382bd1901f1415fb Mon Sep 17 00:00:00 2001 From: "guilhem@mysql.com" <> Date: Tue, 3 Jun 2003 15:47:29 +0200 Subject: [PATCH 02/40] -- Waiting for Monty's approval before push -- Bug 571: play LOAD DATA INFILE the same way on the slave as it was on the master: if it was with IGNORE, do it with IGNORE, if it was with REPLACE, do it with REPLACE, and (the change) if it was with nothing, do it with nothing (not with IGNORE !!). Bug 573: print a proper error message in case of duplicate entry in LOAD DATA INFILE on the slave, i.e. a message where the keyname and key value appear : 'Duplicate entry '1' for key 1' and not 'Duplicate entry '%-.64s' for key %d' --- mysql-test/r/rpl_loaddata.result | 6 ++++++ mysql-test/t/rpl_loaddata.test | 17 +++++++++++++++ sql/log_event.cc | 37 ++++++++++++++++++++++++++------ 3 files changed, 54 insertions(+), 6 deletions(-) diff --git a/mysql-test/r/rpl_loaddata.result b/mysql-test/r/rpl_loaddata.result index 62071a07d0c..844a9d66cb3 100644 --- a/mysql-test/r/rpl_loaddata.result +++ b/mysql-test/r/rpl_loaddata.result @@ -22,3 +22,9 @@ day id category name drop table t1; drop table t2; drop table t3; +create table t1(a int, b int, unique(b)); +insert into t1 values(1,10); +load data infile '../../std_data/rpl_loaddata.dat' into table t1; +show status like 'slave_running'; +Variable_name Value +Slave_running OFF diff --git a/mysql-test/t/rpl_loaddata.test b/mysql-test/t/rpl_loaddata.test index 1f34aa9d3f9..dc4eadda192 100644 --- a/mysql-test/t/rpl_loaddata.test +++ b/mysql-test/t/rpl_loaddata.test @@ -4,6 +4,9 @@ # # check replication of load data for temporary tables with additional parameters # +# check if duplicate entries trigger an error (they should unless IGNORE or +# REPLACE was used on the master) (bug 571). + source include/master-slave.inc; create table t1(a int not null auto_increment, b int, primary key(a) ); @@ -27,7 +30,21 @@ connection master; drop table t1; drop table t2; drop table t3; +create table t1(a int, b int, unique(b)); save_master_pos; connection slave; sync_with_master; +insert into t1 values(1,10); + +connection master; +load data infile '../../std_data/rpl_loaddata.dat' into table t1; + +save_master_pos; +connection slave; +# don't sync_with_master because the slave SQL thread should be stopped because +# of the error so MASTER_POS_WAIT() will not return; just sleep and hope the +# slave SQL thread will have had time to stop. + +sleep 1; +show status like 'slave_running'; diff --git a/sql/log_event.cc b/sql/log_event.cc index cda2e50c53d..369ef940af2 100644 --- a/sql/log_event.cc +++ b/sql/log_event.cc @@ -1886,9 +1886,27 @@ int Load_log_event::exec_event(NET* net, struct st_relay_log_info* rli, else { char llbuff[22]; - enum enum_duplicates handle_dup = DUP_IGNORE; + enum enum_duplicates handle_dup; if (sql_ex.opt_flags & REPLACE_FLAG) handle_dup= DUP_REPLACE; + else if (sql_ex.opt_flags & IGNORE_FLAG) + handle_dup= DUP_IGNORE; + else + /* + Note that when replication is running fine, if it was DUP_ERROR on the + master then we could choose DUP_IGNORE here, because if DUP_ERROR + suceeded on master, and data is identical on the master and slave, + then there should be no uniqueness errors on slave, so DUP_IGNORE is + the same as DUP_ERROR. But in the unlikely case of uniqueness errors + (because the data on the master and slave happen to be different (user + error or bug), we want LOAD DATA to print an error message on the + slave to discover the problem. + + If reading from net (a 3.23 master), mysql_load() will change this + to DUP_IGNORE. + */ + handle_dup= DUP_ERROR; + sql_exchange ex((char*)fname, sql_ex.opt_flags & DUMPFILE_FLAG); String field_term(sql_ex.field_term,sql_ex.field_term_len); String enclosed(sql_ex.enclosed,sql_ex.enclosed_len); @@ -1949,12 +1967,19 @@ int Load_log_event::exec_event(NET* net, struct st_relay_log_info* rli, close_thread_tables(thd); if (thd->query_error) { - int sql_error= thd->net.last_errno; - if (!sql_error) - sql_error= ER_UNKNOWN_ERROR; - slave_print_error(rli,sql_error, + /* this err/sql_errno code is copy-paste from send_error() */ + const char *err; + int sql_errno; + if ((err=thd->net.last_error)[0]) + sql_errno=thd->net.last_errno; + else + { + sql_errno=ER_UNKNOWN_ERROR; + err=ER(sql_errno); + } + slave_print_error(rli,sql_errno, "Error '%s' running load data infile", - ER_SAFE(sql_error)); + err); free_root(&thd->mem_root,0); return 1; } From 4a80a6c7b9a7f81e239a667145e71e6d4e40212a Mon Sep 17 00:00:00 2001 From: "guilhem@mysql.com" <> Date: Tue, 3 Jun 2003 23:13:06 +0200 Subject: [PATCH 03/40] One-line fix for bug 576 (DBUG_ASSERT failure when using CHANGE MASTER TO RELAY_LOG_POS=4). Plus a changeset which I had committed but forgot to push (and this changeset is lost on another computer, so I recreate it here). This changeset is "user-friendly SHOW BINLOG EVENTS and CHANGE MASTER TO when log positions < 4 are used. --- sql/slave.cc | 2 +- sql/sql_repl.cc | 8 +------- sql/sql_yacc.yy | 14 ++++++++++++++ sql/unireg.h | 8 ++++++-- 4 files changed, 22 insertions(+), 10 deletions(-) diff --git a/sql/slave.cc b/sql/slave.cc index ec1041894bd..c2762dbd6f4 100644 --- a/sql/slave.cc +++ b/sql/slave.cc @@ -269,7 +269,7 @@ int init_relay_log_pos(RELAY_LOG_INFO* rli,const char* log, goto err; rli->cur_log = &rli->cache_buf; } - if (pos > BIN_LOG_HEADER_SIZE) + if (pos >= BIN_LOG_HEADER_SIZE) my_b_seek(rli->cur_log,(off_t)pos); err: diff --git a/sql/sql_repl.cc b/sql/sql_repl.cc index 283dd20a56c..ca993c053a1 100644 --- a/sql/sql_repl.cc +++ b/sql/sql_repl.cc @@ -961,7 +961,7 @@ int show_binlog_events(THD* thd) { LEX_MASTER_INFO *lex_mi = &thd->lex.mi; ha_rows event_count, limit_start, limit_end; - my_off_t pos = lex_mi->pos; + my_off_t pos = max(BIN_LOG_HEADER_SIZE, lex_mi->pos); // user-friendly char search_file_name[FN_REFLEN], *name; const char *log_file_name = lex_mi->log_file_name; pthread_mutex_t *log_lock = mysql_bin_log.get_log_lock(); @@ -989,12 +989,6 @@ int show_binlog_events(THD* thd) if ((file=open_binlog(&log, linfo.log_file_name, &errmsg)) < 0) goto err; - if (pos < 4) - { - errmsg = "Invalid log position"; - goto err; - } - pthread_mutex_lock(log_lock); my_b_seek(&log, pos); diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy index c79750c8014..b0c81d6f6b0 100644 --- a/sql/sql_yacc.yy +++ b/sql/sql_yacc.yy @@ -706,6 +706,18 @@ master_def: MASTER_LOG_POS_SYM EQ ulonglong_num { Lex->mi.pos = $3; + /* + If the user specified a value < BIN_LOG_HEADER_SIZE, adjust it + instead of causing subsequent errors. + We need to do it in this file, because only there we know that + MASTER_LOG_POS has been explicitely specified. On the contrary + in change_master() (sql_repl.cc) we cannot distinguish between 0 + (MASTER_LOG_POS explicitely specified as 0) and 0 (unspecified), + whereas we want to distinguish (specified 0 means "read the binlog + from 0" (4 in fact), unspecified means "don't change the position + (keep the preceding value)"). + */ + Lex->mi.pos = max(BIN_LOG_HEADER_SIZE, Lex->mi.pos); } | MASTER_CONNECT_RETRY_SYM EQ ULONG_NUM @@ -721,6 +733,8 @@ master_def: RELAY_LOG_POS_SYM EQ ULONG_NUM { Lex->mi.relay_log_pos = $3; + /* Adjust if < BIN_LOG_HEADER_SIZE (same comment as Lex->mi.pos) */ + Lex->mi.relay_log_pos = max(BIN_LOG_HEADER_SIZE, Lex->mi.relay_log_pos); }; diff --git a/sql/unireg.h b/sql/unireg.h index f69d67455dd..f2cace51fa7 100644 --- a/sql/unireg.h +++ b/sql/unireg.h @@ -130,9 +130,13 @@ bfill((A)->null_flags,(A)->null_bytes,255);\ */ #define MIN_TURBOBM_PATTERN_LEN 3 -/* Defines for binary logging */ +/* + Defines for binary logging. + Do not decrease the value of BIN_LOG_HEADER_SIZE. + Do not even increase it before checking code. +*/ -#define BIN_LOG_HEADER_SIZE 4 +#define BIN_LOG_HEADER_SIZE 4 /* Include prototypes for unireg */ From 100a66e6cbb6598039fdd0c0b5e9f8ed6e1d025d Mon Sep 17 00:00:00 2001 From: "monty@narttu.mysql.fi" <> Date: Wed, 4 Jun 2003 16:05:27 +0300 Subject: [PATCH 04/40] Added [mysqld-base-version] as a default group for the mysqld server Portability fix for Windows 64 --- include/config-win.h | 8 +++++++- include/my_global.h | 2 +- include/mysql_version.h.in | 1 + innobase/include/univ.i | 4 ++++ sql/mysqld.cc | 2 +- 5 files changed, 14 insertions(+), 3 deletions(-) diff --git a/include/config-win.h b/include/config-win.h index 9931d2c4b95..096c00e4574 100644 --- a/include/config-win.h +++ b/include/config-win.h @@ -130,6 +130,11 @@ typedef uint rf_SetTimer; #define SIZEOF_LONG 4 #define SIZEOF_LONG_LONG 8 #define SIZEOF_OFF_T 8 +#ifdef _WIN64 +#define SIZEOF_CHARP 8 +#else +#define SIZEOF_CHARP 4 +#endif #define HAVE_BROKEN_NETINET_INCLUDES #ifdef __NT__ #define HAVE_NAMED_PIPE /* We can only create pipes on NT */ @@ -196,6 +201,7 @@ inline double ulonglong2double(ulonglong value) /* Optimized store functions for Intel x86 */ +#ifndef _WIN64 #define sint2korr(A) (*((int16 *) (A))) #define sint3korr(A) ((int32) ((((uchar) (A)[2]) & 128) ? \ (((uint32) 255L << 24) | \ @@ -236,7 +242,7 @@ inline double ulonglong2double(ulonglong value) #define float8get(V,M) doubleget((V),(M)) #define float4store(V,M) memcpy((byte*) V,(byte*) (&M),sizeof(float)) #define float8store(V,M) doublestore((V),(M)) - +#endif /* _WIN64 */ #define HAVE_PERROR #define HAVE_VFPRINT diff --git a/include/my_global.h b/include/my_global.h index 90c4801e807..1026e8e3940 100644 --- a/include/my_global.h +++ b/include/my_global.h @@ -848,7 +848,7 @@ typedef char bool; /* Ordinary boolean values 0 1 */ */ /* Optimized store functions for Intel x86 */ -#ifdef __i386__ +#if defined(__i386__) && !defined(_WIN64) #define sint2korr(A) (*((int16 *) (A))) #define sint3korr(A) ((int32) ((((uchar) (A)[2]) & 128) ? \ (((uint32) 255L << 24) | \ diff --git a/include/mysql_version.h.in b/include/mysql_version.h.in index 793bf36e9fe..da184665f6e 100644 --- a/include/mysql_version.h.in +++ b/include/mysql_version.h.in @@ -10,6 +10,7 @@ #else #define PROTOCOL_VERSION @PROTOCOL_VERSION@ #define MYSQL_SERVER_VERSION "@VERSION@" +#define MYSQL_BASE_VERSION "mysqld-@MYSQL_BASE_VERSION@" #ifndef MYSQL_SERVER_SUFFIX #define MYSQL_SERVER_SUFFIX "@MYSQL_SERVER_SUFFIX@" #endif diff --git a/innobase/include/univ.i b/innobase/include/univ.i index e29f3ec92e1..4854e5a7b78 100644 --- a/innobase/include/univ.i +++ b/innobase/include/univ.i @@ -187,7 +187,11 @@ management to ensure correct alignment for doubles etc. */ /* Another basic type we use is unsigned long integer which is intended to be equal to the word size of the machine. */ +#ifdef _WIN64 +typedef unsigned __int64 ulint; +#else typedef unsigned long int ulint; +#endif typedef long int lint; diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 1492d6ddb68..7289d0e72cf 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -1923,7 +1923,7 @@ extern "C" pthread_handler_decl(handle_shutdown,arg) #endif -const char *load_default_groups[]= { "mysqld","server",0 }; +const char *load_default_groups[]= { "mysqld","server",MYSQL_BASE_VERSION,0 }; bool open_log(MYSQL_LOG *log, const char *hostname, const char *opt_name, const char *extension, From 42c80c81f79d5376089a14ede88749a3ae625317 Mon Sep 17 00:00:00 2001 From: "heikki@hundin.mysql.fi" <> Date: Wed, 4 Jun 2003 17:58:41 +0300 Subject: [PATCH 05/40] handler.cc: If the autocommit is on, let handler.cc commit or rollback the whole transaction at an updating SQL statement end. This probably fixes bug number 578. The problem was that when explicit LOCK TABLES is used, then the lock count method in autocommit does not work. --- sql/handler.cc | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/sql/handler.cc b/sql/handler.cc index 45c83355c94..6ee0b1f9c55 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -208,23 +208,45 @@ void ha_close_connection(THD* thd) } /* - This is used to commit or rollback a single statement depending - on the value of error + This is used to commit or rollback a single statement depending on the value + of error. If the autocommit is on, then we will commit or rollback the whole + transaction (= the statement). The autocommit mechanism built into handlers + is based on counting locks, but if the user has used LOCK TABLES then that + mechanism does not know to do the commit. */ int ha_autocommit_or_rollback(THD *thd, int error) { + bool do_autocommit=FALSE; + DBUG_ENTER("ha_autocommit_or_rollback"); #ifdef USING_TRANSACTIONS + + if (!(thd->options & (OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN))) + do_autocommit=TRUE; /* We can commit or rollback the whole transaction */ + if (opt_using_transactions) { if (!error) { - if (ha_commit_stmt(thd)) - error=1; + if (do_autocommit) + { + if (ha_commit(thd)) + error=1; + } + else + { + if (ha_commit_stmt(thd)) + error=1; + } } else - (void) ha_rollback_stmt(thd); + { + if (do_autocommit) + (void) ha_rollback(thd); + else + (void) ha_rollback_stmt(thd); + } thd->variables.tx_isolation=thd->session_tx_isolation; } From 6217b578b9b9a6f65b6891b888be357d3148f4d0 Mon Sep 17 00:00:00 2001 From: "monty@narttu.mysql.fi" <> Date: Wed, 4 Jun 2003 18:22:48 +0300 Subject: [PATCH 06/40] Fixed (not fatal) buffer overflow --- libmysql/libmysql.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libmysql/libmysql.c b/libmysql/libmysql.c index af74182eb22..c008d625900 100644 --- a/libmysql/libmysql.c +++ b/libmysql/libmysql.c @@ -1646,7 +1646,7 @@ mysql_real_connect(MYSQL *mysql,const char *host, const char *user, net->vio = vio_new(sock, VIO_TYPE_SOCKET, TRUE); bzero((char*) &UNIXaddr,sizeof(UNIXaddr)); UNIXaddr.sun_family = AF_UNIX; - strmov(UNIXaddr.sun_path, unix_socket); + strmake(UNIXaddr.sun_path, unix_socket, sizeof(UNIXaddr.sun_path)-1); if (my_connect(sock,(struct sockaddr *) &UNIXaddr, sizeof(UNIXaddr), mysql->options.connect_timeout) <0) { From 7947830b2de3f83f674e8341abcaddcad3b9e500 Mon Sep 17 00:00:00 2001 From: "lenz@mysql.com" <> Date: Wed, 4 Jun 2003 17:31:21 +0200 Subject: [PATCH 07/40] - Updated Default-Stop run levels in the LSB header section to satisfy Red Hat's chkconfig (Bug #272) (The LSB spec is a bit ambigous about what actually needs to be put into this field) --- support-files/mysql.server.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/support-files/mysql.server.sh b/support-files/mysql.server.sh index de01142beac..694e6fa8ebb 100644 --- a/support-files/mysql.server.sh +++ b/support-files/mysql.server.sh @@ -19,7 +19,7 @@ # Required-Start: $local_fs $network $remote_fs # Required-Stop: $local_fs $network $remote_fs # Default-Start: 2 3 4 5 -# Default-Stop: 2 3 4 5 +# Default-Stop: 0 1 6 # Short-Description: start and stop MySQL # Description: MySQL is a very fast and reliable SQL database engine. ### END INIT INFO From 810e3bff14ac879e9c85e573034e50098718218e Mon Sep 17 00:00:00 2001 From: "lenz@mysql.com" <> Date: Wed, 4 Jun 2003 22:31:06 +0200 Subject: [PATCH 08/40] - When compiling the Max package incl. RAID support using gcc, make sure to set CXX=gcc (cannot link the code with g++) - this should help to recompile the RPM on Distributions using gcc 3 - Added a symlink /usr/sbin/rcmysql -> /etc/init.d/mysql --- support-files/mysql.spec.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/support-files/mysql.spec.sh b/support-files/mysql.spec.sh index aab3e298e14..06ba2d63f45 100644 --- a/support-files/mysql.spec.sh +++ b/support-files/mysql.spec.sh @@ -254,6 +254,13 @@ export PATH # Build the 4.0 Max binary (includes BDB and UDFs and therefore # cannot be linked statically against the patched glibc) +# If we want to compile with RAID using gcc 3, we need to use +# gcc instead of g++ to avoid linking problems (RAID code is written in C++) +if gcc -v 2>&1 | grep 'version 3' > /dev/null 2>&1 +then + export CXX="gcc" +fi + BuildMySQL "--enable-shared \ --with-berkeley-db \ --with-innodb \ @@ -318,6 +325,10 @@ install -m644 $MBD/sql/mysqld.sym $RBR/usr/lib/mysql/mysqld.sym install -m644 $MBD/support-files/mysql-log-rotate $RBR/etc/logrotate.d/mysql install -m755 $MBD/support-files/mysql.server $RBR/etc/init.d/mysql +# Create a symlink "rcmysql", pointing to the init.script. SuSE users +# will appreciate that, as all services usually offer this. +ln -s ../../sbin/init.d/mysql $RPM_BUILD_ROOT/usr/sbin/rcmysql + # Create symbolic compatibility link safe_mysqld -> mysqld_safe # (safe_mysqld will be gone in MySQL 4.1) ln -sf ./mysqld_safe $RBR/usr/bin/safe_mysqld @@ -462,6 +473,7 @@ fi %attr(755, root, root) /usr/bin/safe_mysqld %attr(755, root, root) /usr/sbin/mysqld +%attr(755, root, root) /usr/sbin/rcmysql %attr(644, root, root) /usr/lib/mysql/mysqld.sym %attr(644, root, root) /etc/logrotate.d/mysql From cd3b680db00f52d7a502bf62513789f54b2dbcd0 Mon Sep 17 00:00:00 2001 From: "monty@narttu.mysql.fi" <> Date: Thu, 5 Jun 2003 11:55:03 +0300 Subject: [PATCH 09/40] Fixed problem with alarms when reading too big packet --- sql/net_serv.cc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/sql/net_serv.cc b/sql/net_serv.cc index c8c774d365f..0b332f84bac 100644 --- a/sql/net_serv.cc +++ b/sql/net_serv.cc @@ -431,13 +431,13 @@ net_real_write(NET *net,const char *packet,ulong len) big packet */ -static void my_net_skip_rest(NET *net, ulong remain, thr_alarm_t *alarmed) +static void my_net_skip_rest(NET *net, ulong remain, thr_alarm_t *alarmed, + ALARM *alarm_buff) { - ALARM alarm_buff; uint retry_count=0; - if (!thr_alarm_in_use(&alarmed)) + if (!thr_alarm_in_use(alarmed)) { - if (!thr_alarm(alarmed,net->timeout,&alarm_buff) || + if (!thr_alarm(alarmed,net->timeout,alarm_buff) || (!vio_is_blocking(net->vio) && vio_blocking(net->vio,TRUE) < 0)) return; /* Can't setup, abort */ } @@ -606,7 +606,7 @@ my_real_read(NET *net, ulong *complen) { #ifdef MYSQL_SERVER if (i == 1) - my_net_skip_rest(net, len, &alarmed); + my_net_skip_rest(net, len, &alarmed, &alarm_buff); #endif len= packet_error; /* Return error */ goto end; From 772adcbd992108af5d7d74c5448cc16697183e8a Mon Sep 17 00:00:00 2001 From: "monty@narttu.mysql.fi" <> Date: Thu, 5 Jun 2003 12:29:13 +0300 Subject: [PATCH 10/40] Print error if we can't delete an alarm More debugging variables Increment aborted_threads in case of killed or too big packet --- include/thr_alarm.h | 2 +- mysys/thr_alarm.c | 11 +++++------ sql/mysqld.cc | 2 +- sql/net_serv.cc | 7 ++++++- sql/sql_parse.cc | 7 +++++++ 5 files changed, 20 insertions(+), 9 deletions(-) diff --git a/include/thr_alarm.h b/include/thr_alarm.h index 439f046252f..8ff4472f700 100644 --- a/include/thr_alarm.h +++ b/include/thr_alarm.h @@ -100,7 +100,7 @@ typedef struct st_alarm { #define thr_alarm_init(A) (*(A))=0 #define thr_alarm_in_use(A) (*(A)!= 0) void init_thr_alarm(uint max_alarm); -bool thr_alarm(thr_alarm_t *alarmed, uint sec, ALARM *buff); +my_bool thr_alarm(thr_alarm_t *alarmed, uint sec, ALARM *buff); void thr_alarm_kill(pthread_t thread_id); void thr_end_alarm(thr_alarm_t *alarmed); void end_thr_alarm(my_bool free_structures); diff --git a/mysys/thr_alarm.c b/mysys/thr_alarm.c index a2647ec7399..1f9c4c3b068 100644 --- a/mysys/thr_alarm.c +++ b/mysys/thr_alarm.c @@ -127,7 +127,7 @@ void init_thr_alarm(uint max_alarms) Returns 0 if no more alarms are allowed (aborted by process) */ -bool thr_alarm(thr_alarm_t *alrm, uint sec, ALARM *alarm_data) +my_bool thr_alarm(thr_alarm_t *alrm, uint sec, ALARM *alarm_data) { ulong now; sigset_t old_mask; @@ -209,7 +209,7 @@ void thr_end_alarm(thr_alarm_t *alarmed) ALARM *alarm_data; sigset_t old_mask; uint i; - bool found=0; + my_bool found=0; DBUG_ENTER("thr_end_alarm"); pthread_sigmask(SIG_BLOCK,&full_signal_set,&old_mask); @@ -230,10 +230,9 @@ void thr_end_alarm(thr_alarm_t *alarmed) DBUG_ASSERT(!*alarmed || found); if (!found) { -#ifdef MAIN - printf("Warning: Didn't find alarm %lx in queue of %d alarms\n", - (long) *alarmed, alarm_queue.elements); -#endif + if (*alarmed) + fprintf(stderr,"Warning: Didn't find alarm %lx in queue of %d alarms\n", + (long) *alarmed, alarm_queue.elements); DBUG_PRINT("warning",("Didn't find alarm %lx in queue\n", (long) *alarmed)); } diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 7289d0e72cf..8ed183e2f1f 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -380,7 +380,7 @@ uint report_port = MYSQL_PORT; my_bool master_ssl = 0; ulong master_retry_count=0; -ulong bytes_sent = 0L, bytes_received = 0L; +ulong bytes_sent= 0L, bytes_received= 0L, net_big_packet_count= 0L; bool opt_endinfo,using_udf_functions, locked_in_memory; bool opt_using_transactions, using_update_log; diff --git a/sql/net_serv.cc b/sql/net_serv.cc index 3d5055b4f24..a3bb2525f9d 100644 --- a/sql/net_serv.cc +++ b/sql/net_serv.cc @@ -65,11 +65,13 @@ void sql_print_error(const char *format,...); #define USE_QUERY_CACHE extern uint test_flags; extern void query_cache_insert(NET *net, const char *packet, ulong length); -extern ulong bytes_sent, bytes_received; +extern ulong bytes_sent, bytes_received, net_big_packet_count; extern pthread_mutex_t LOCK_bytes_sent , LOCK_bytes_received; #else #undef statistic_add +#undef statistic_increment #define statistic_add(A,B,C) +#define statistic_increment(A,B) #endif #define TEST_BLOCKING 8 @@ -557,6 +559,9 @@ static my_bool my_net_skip_rest(NET *net, uint32 remain, thr_alarm_t *alarmed, DBUG_ENTER("my_net_skip_rest"); DBUG_PRINT("enter",("bytes_to_skip: %u", (uint) remain)); + /* The following is good for debugging */ + statistic_increment(net_big_packet_count,&LOCK_bytes_received); + if (!thr_alarm_in_use(alarmed)) { my_bool old_mode; diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index d9060b4b26e..b06a48f9045 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -729,6 +729,10 @@ pthread_handler_decl(handle_one_connection,arg) send_error(net,net->last_errno,NullS); statistic_increment(aborted_threads,&LOCK_status); } + else if (thd->killed) + { + statistic_increment(aborted_threads,&LOCK_status); + } end_thread: close_connection(net); @@ -905,7 +909,10 @@ bool do_command(THD *thd) vio_description(net->vio))); /* Check if we can continue without closing the connection */ if (net->error != 3) + { + statistic_increment(aborted_threads,&LOCK_status); DBUG_RETURN(TRUE); // We have to close it. + } send_error(net,net->last_errno,NullS); net->error= 0; DBUG_RETURN(FALSE); From 4b55fbe092e7ba5905dc1dfdd497d683c9e7418c Mon Sep 17 00:00:00 2001 From: "jani@ua126d19.elisa.omakaista.fi" <> Date: Thu, 5 Jun 2003 15:06:19 +0300 Subject: [PATCH 11/40] Fixed a bug in concat_ws(), which did not add concat separator in case of an empty string. Bug ID 586. --- mysql-test/r/func_str.result | 2 +- sql/item_strfunc.cc | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/mysql-test/r/func_str.result b/mysql-test/r/func_str.result index a72d32d39f8..1a4cb9217e4 100644 --- a/mysql-test/r/func_str.result +++ b/mysql-test/r/func_str.result @@ -64,7 +64,7 @@ concat_ws(NULL,'a') concat_ws(',',NULL,'') NULL select concat_ws(',','',NULL,'a'); concat_ws(',','',NULL,'a') -a +,a SELECT CONCAT('"',CONCAT_WS('";"',repeat('a',60),repeat('b',60),repeat('c',60),repeat('d',100)), '"'); CONCAT('"',CONCAT_WS('";"',repeat('a',60),repeat('b',60),repeat('c',60),repeat('d',100)), '"') "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb";"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc";"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" diff --git a/sql/item_strfunc.cc b/sql/item_strfunc.cc index ae8bf1dfecb..208be1ecd7f 100644 --- a/sql/item_strfunc.cc +++ b/sql/item_strfunc.cc @@ -495,18 +495,18 @@ String *Item_func_concat_ws::val_str(String *str) str->length(0); // QQ; Should be removed res=str; - // Skip until non-null and non-empty argument is found. + // Skip until non-null argument is found. // If not, return the empty string for (i=0; i < arg_count; i++) - if ((res= args[i]->val_str(str)) && res->length()) + if ((res= args[i]->val_str(str))) break; if (i == arg_count) return &empty_string; for (i++; i < arg_count ; i++) { - if (!(res2= args[i]->val_str(use_as_buff)) || !res2->length()) - continue; // Skip NULL and empty string + if (!(res2= args[i]->val_str(use_as_buff))) + continue; // Skip NULL if (res->length() + sep_str->length() + res2->length() > current_thd->variables.max_allowed_packet) From de0a3d303663be27ff888103af841fc9d9fb3589 Mon Sep 17 00:00:00 2001 From: "monty@narttu.mysql.fi" <> Date: Thu, 5 Jun 2003 15:15:27 +0300 Subject: [PATCH 12/40] Fixed test if thr_alarm() failed --- sql/net_serv.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/net_serv.cc b/sql/net_serv.cc index 0b332f84bac..23a23dbde7b 100644 --- a/sql/net_serv.cc +++ b/sql/net_serv.cc @@ -437,7 +437,7 @@ static void my_net_skip_rest(NET *net, ulong remain, thr_alarm_t *alarmed, uint retry_count=0; if (!thr_alarm_in_use(alarmed)) { - if (!thr_alarm(alarmed,net->timeout,alarm_buff) || + if (thr_alarm(alarmed,net->timeout,alarm_buff) || (!vio_is_blocking(net->vio) && vio_blocking(net->vio,TRUE) < 0)) return; /* Can't setup, abort */ } From c2e5f4841227bb7e4e673ff13132bd5b60f4911c Mon Sep 17 00:00:00 2001 From: "heikki@hundin.mysql.fi" <> Date: Thu, 5 Jun 2003 15:58:23 +0300 Subject: [PATCH 13/40] ha_innodb.cc, handler.cc: Fix the BDB crash in the previous push; to save CPU remove duplicate calls of commit in InnoDB --- sql/ha_innodb.cc | 229 +++++++++++++++++++++++++++++++---------------- sql/handler.cc | 34 ++----- 2 files changed, 158 insertions(+), 105 deletions(-) diff --git a/sql/ha_innodb.cc b/sql/ha_innodb.cc index 50bb4275eaa..9cc86edddf8 100644 --- a/sql/ha_innodb.cc +++ b/sql/ha_innodb.cc @@ -131,9 +131,11 @@ static void innobase_print_error(const char* db_errpfx, char* buffer); /********************************************************************** Releases possible search latch and InnoDB thread FIFO ticket. These should -be released at each SQL statement end. It does no harm to release these -also in the middle of an SQL statement. */ +be released at each SQL statement end, and also when mysqld passes the +control to the client. It does no harm to release these also in the middle +of an SQL statement. */ static +inline void innobase_release_stat_resources( /*============================*/ @@ -896,6 +898,11 @@ innobase_commit_low( /*================*/ trx_t* trx) /* in: transaction handle */ { + if (trx->conc_state == TRX_NOT_STARTED) { + + return; + } + /* TODO: Guilhem should check if master_log_name, pending etc. are right if the master log gets rotated! Possible bug here. Comment by Heikki March 4, 2003. */ @@ -910,11 +917,13 @@ innobase_commit_low( active_mi->rli.event_len + active_mi->rli.pending)); } - trx_commit_for_mysql(trx); + + trx_commit_for_mysql(trx); } /********************************************************************* -Commits a transaction in an InnoDB database. */ +Commits a transaction in an InnoDB database or marks an SQL statement +ended. */ int innobase_commit( @@ -932,29 +941,45 @@ innobase_commit( DBUG_ENTER("innobase_commit"); DBUG_PRINT("trans", ("ending transaction")); + /* The flag thd->transaction.all.innodb_active_trans is set to 1 + in ::external_lock and ::start_stmt, and it is only set to 0 in + a commit or a rollback. If it is 0 we know there cannot be resources + to be freed and we can return immediately. */ + + if (thd->transaction.all.innodb_active_trans == 0) { + + DBUG_RETURN(0); + } + trx = check_trx_exists(thd); - if (trx->auto_inc_lock) { - - /* If we had reserved the auto-inc lock for - some table in this SQL statement, we release it now */ - - srv_conc_enter_innodb(trx); - row_unlock_table_autoinc_for_mysql(trx); - srv_conc_exit_innodb(trx); - } - - if (trx_handle != (void*)&innodb_dummy_stmt_trx_handle) { + if (trx_handle != (void*)&innodb_dummy_stmt_trx_handle + || (!(thd->options & (OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN)))) { + innobase_commit_low(trx); - thd->transaction.all.innodb_active_trans=0; + + thd->transaction.all.innodb_active_trans = 0; + } else { + if (trx->auto_inc_lock) { + /* If we had reserved the auto-inc lock for some + table in this SQL statement we release it now */ + + srv_conc_enter_innodb(trx); + row_unlock_table_autoinc_for_mysql(trx); + srv_conc_exit_innodb(trx); + } + /* Store the current undo_no of the transaction so that we + know where to roll back if we have to roll back the next + SQL statement */ + + trx_mark_sql_stat_end(trx); } - /* Release possible statement level resources */ + /* Release a possible FIFO ticket and search latch */ innobase_release_stat_resources(trx); - trx_mark_sql_stat_end(trx); - /* Tell InnoDB server that there might be work for - utility threads: */ + /* Tell the InnoDB server that there might be work for utility + threads: */ srv_active_wake_master_thread(); @@ -1025,7 +1050,7 @@ innobase_commit_complete( } /********************************************************************* -Rolls back a transaction in an InnoDB database. */ +Rolls back a transaction or the latest SQL statement in an InnoDB database. */ int innobase_rollback( @@ -1066,11 +1091,9 @@ innobase_rollback( srv_conc_exit_innodb(trx); - /* Release possible statement level resources */ + /* Release a possible FIFO ticket and search latch */ innobase_release_stat_resources(trx); - trx_mark_sql_stat_end(trx); - DBUG_RETURN(convert_error_code_to_mysql(error, NULL)); } @@ -2994,6 +3017,8 @@ create_index( KEY* key; KEY_PART_INFO* key_part; ulint ind_type; + ulint col_type; + ulint prefix_len; ulint i; DBUG_ENTER("create_index"); @@ -3021,10 +3046,32 @@ create_index( for (i = 0; i < n_fields; i++) { key_part = key->key_part + i; + if (key_part->length != key_part->field->pack_length()) { + prefix_len = key_part->length; + + col_type = get_innobase_type_from_mysql_type( + key_part->field); + if (col_type == DATA_INT + || col_type == DATA_FLOAT + || col_type == DATA_DOUBLE + || col_type == DATA_DECIMAL) { + fprintf(stderr, +"InnoDB: error: MySQL is trying to create a column prefix index field\n" +"InnoDB: on an inappropriate data type %lu. Table name %s, column name %s.\n", + col_type, table_name, + key_part->field->field_name); + + prefix_len = 0; + } + } else { + prefix_len = 0; + } + /* We assume all fields should be sorted in ascending order, hence the '0': */ dict_mem_index_add_field(index, - (char*) key_part->field->field_name, 0); + (char*) key_part->field->field_name, + 0, prefix_len); } error = row_create_index_for_mysql(index, trx); @@ -3562,8 +3609,7 @@ ha_innobase::records_in_range( /************************************************************************* Gives an UPPER BOUND to the number of rows in a table. This is used in -filesort.cc and its better if the upper bound hold. -*/ +filesort.cc. */ ha_rows ha_innobase::estimate_number_of_rows(void) @@ -3598,11 +3644,11 @@ ha_innobase::estimate_number_of_rows(void) /* Calculate a minimum length for a clustered index record and from that an upper bound for the number of rows. Since we only calculate - new statistics in row0mysql.c when a tablehas grown - by a threshold factor, we must add a safety factor 2 in front - of the formula below. */ + new statistics in row0mysql.c when a table has grown by a threshold + factor, we must add a safety factor 2 in front of the formula below. */ - estimate = 2 * local_data_file_length / dict_index_calc_min_rec_len(index); + estimate = 2 * local_data_file_length / + dict_index_calc_min_rec_len(index); prebuilt->trx->op_info = (char*)""; @@ -3629,27 +3675,36 @@ ha_innobase::scan_time() return((double) (prebuilt->table->stat_clustered_index_size)); } -/* - Calculate the time it takes to read a set of ranges through and index - This enables us to optimise reads for clustered indexes. -*/ +/********************************************************************** +Calculate the time it takes to read a set of ranges through an index +This enables us to optimise reads for clustered indexes. */ -double ha_innobase::read_time(uint index, uint ranges, ha_rows rows) +double +ha_innobase::read_time( +/*===================*/ + /* out: estimated time measured in disk seeks */ + uint index, /* in: key number */ + uint ranges, /* in: how many ranges */ + ha_rows rows) /* in: estimated number of rows in the ranges */ { - ha_rows total_rows; - double time_for_scan; - if (index != table->primary_key) - return handler::read_time(index, ranges, rows); // Not clustered - if (rows <= 2) - return (double) rows; - /* - Assume that the read is proportional to scan time for all rows + one - seek per range. - */ - time_for_scan= scan_time(); - if ((total_rows= estimate_number_of_rows()) < rows) - return time_for_scan; - return (ranges + (double) rows / (double) total_rows * time_for_scan); + ha_rows total_rows; + double time_for_scan; + + if (index != table->primary_key) + return handler::read_time(index, ranges, rows); // Not clustered + + if (rows <= 2) + return (double) rows; + + /* Assume that the read time is proportional to the scan time for all + rows + at most one seek per range. */ + + time_for_scan= scan_time(); + + if ((total_rows= estimate_number_of_rows()) < rows) + return time_for_scan; + + return (ranges + (double) rows / (double) total_rows * time_for_scan); } /************************************************************************* @@ -3992,10 +4047,10 @@ ha_innobase::reset(void) } /********************************************************************** -Inside LOCK TABLES MySQL will not call external_lock() between SQL -statements. It will call this function at the start of each SQL statement. -Note also a spacial case: if a temporary table is created inside LOCK -TABLES, MySQL has not called external_lock() at all on that table. */ +MySQL calls this function at the start of each SQL statement. Inside LOCK +TABLES the ::external_lock method does not work to mark SQL statement +borders. Note also a special case: if a temporary table is created inside +LOCK TABLES, MySQL has not called external_lock() at all on that table. */ int ha_innobase::start_stmt( @@ -4010,8 +4065,14 @@ ha_innobase::start_stmt( trx = prebuilt->trx; + /* Here we release the search latch and the InnoDB thread FIFO ticket + if they were reserved. They should have been released already at the + end of the previous statement, but because inside LOCK TABLES the + lock count method does not work to mark the end of a SELECT statement, + that may not be the case. We MUST release the search latch before an + INSERT, for example. */ + innobase_release_stat_resources(trx); - trx_mark_sql_stat_end(trx); if (trx->isolation_level <= TRX_ISO_READ_COMMITTED && trx->read_view) { @@ -4034,7 +4095,8 @@ ha_innobase::start_stmt( prebuilt->select_lock_type = LOCK_X; } - + + /* Set the MySQL flag to mark that there is an active transaction */ thd->transaction.all.innodb_active_trans = 1; return(0); @@ -4098,17 +4160,20 @@ ha_innobase::external_lock( } if (lock_type != F_UNLCK) { - if (trx->n_mysql_tables_in_use == 0) { - trx_mark_sql_stat_end(trx); - } + /* MySQL is setting a new table lock */ + /* Set the MySQL flag to mark that there is an active + transaction */ thd->transaction.all.innodb_active_trans = 1; + trx->n_mysql_tables_in_use++; prebuilt->mysql_has_locked = TRUE; - trx->isolation_level = innobase_map_isolation_level( + if (trx->n_mysql_tables_in_use == 1) { + trx->isolation_level = innobase_map_isolation_level( (enum_tx_isolation) thd->variables.tx_isolation); + } if (trx->isolation_level == TRX_ISO_SERIALIZABLE && prebuilt->select_lock_type == LOCK_NONE) { @@ -4124,37 +4189,44 @@ ha_innobase::external_lock( trx->mysql_n_tables_locked++; } - } else { - trx->n_mysql_tables_in_use--; - prebuilt->mysql_has_locked = FALSE; - auto_inc_counter_for_this_stat = 0; - if (trx->n_mysql_tables_in_use == 0) { + DBUG_RETURN(error); + } - trx->mysql_n_tables_locked = 0; + /* MySQL is releasing a table lock */ - prebuilt->used_in_HANDLER = FALSE; + trx->n_mysql_tables_in_use--; + prebuilt->mysql_has_locked = FALSE; + auto_inc_counter_for_this_stat = 0; - /* Here we release the search latch and InnoDB - thread FIFO ticket if they were reserved. */ + /* If the MySQL lock count drops to zero we know that the current SQL + statement has ended */ - innobase_release_stat_resources(trx); + if (trx->n_mysql_tables_in_use == 0) { + trx->mysql_n_tables_locked = 0; + prebuilt->used_in_HANDLER = FALSE; + + if (!(thd->options + & (OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN))) { + if (thd->transaction.all.innodb_active_trans != 0) { + innobase_commit(thd, trx); + } + } else { if (trx->isolation_level <= TRX_ISO_READ_COMMITTED && trx->read_view) { - /* At low transaction isolation levels we let + /* At low transaction isolation levels we let each consistent read set its own snapshot */ - read_view_close_for_mysql(trx); + read_view_close_for_mysql(trx); } - - if (!(thd->options - & (OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN))) { - - innobase_commit(thd, trx); - } } + + /* Here we release the search latch and the InnoDB thread FIFO + ticket if they were reserved. */ + + innobase_release_stat_resources(trx); } DBUG_RETURN(error); @@ -4473,4 +4545,3 @@ ha_innobase::get_auto_increment() } #endif /* HAVE_INNOBASE_DB */ - diff --git a/sql/handler.cc b/sql/handler.cc index 6ee0b1f9c55..cae1777e958 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -209,44 +209,26 @@ void ha_close_connection(THD* thd) /* This is used to commit or rollback a single statement depending on the value - of error. If the autocommit is on, then we will commit or rollback the whole - transaction (= the statement). The autocommit mechanism built into handlers - is based on counting locks, but if the user has used LOCK TABLES then that - mechanism does not know to do the commit. + of error. Note that if the autocommit is on, then the following call inside + InnoDB will commit or rollback the whole transaction (= the statement). The + autocommit mechanism built into InnoDB is based on counting locks, but if + the user has used LOCK TABLES then that mechanism does not know to do the + commit. */ int ha_autocommit_or_rollback(THD *thd, int error) { - bool do_autocommit=FALSE; - DBUG_ENTER("ha_autocommit_or_rollback"); #ifdef USING_TRANSACTIONS - - if (!(thd->options & (OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN))) - do_autocommit=TRUE; /* We can commit or rollback the whole transaction */ - if (opt_using_transactions) { if (!error) { - if (do_autocommit) - { - if (ha_commit(thd)) - error=1; - } - else - { - if (ha_commit_stmt(thd)) - error=1; - } + if (ha_commit_stmt(thd)) + error=1; } else - { - if (do_autocommit) - (void) ha_rollback(thd); - else - (void) ha_rollback_stmt(thd); - } + (void) ha_rollback_stmt(thd); thd->variables.tx_isolation=thd->session_tx_isolation; } From e09517e2ac9596d0d011ca87f2d0cb181398da6c Mon Sep 17 00:00:00 2001 From: "heikki@hundin.mysql.fi" <> Date: Thu, 5 Jun 2003 16:06:38 +0300 Subject: [PATCH 14/40] ha_innodb.cc: Revert a change to dict_mem_index_add_field which slipped prematurely into the bk tree --- sql/ha_innodb.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sql/ha_innodb.cc b/sql/ha_innodb.cc index 9cc86edddf8..fd030fff091 100644 --- a/sql/ha_innodb.cc +++ b/sql/ha_innodb.cc @@ -3070,8 +3070,7 @@ create_index( /* We assume all fields should be sorted in ascending order, hence the '0': */ dict_mem_index_add_field(index, - (char*) key_part->field->field_name, - 0, prefix_len); + (char*) key_part->field->field_name, 0); } error = row_create_index_for_mysql(index, trx); From 058d8ed14b22fe3eb63cdc68cd41bffd84561367 Mon Sep 17 00:00:00 2001 From: "jani@ua126d19.elisa.omakaista.fi" <> Date: Thu, 5 Jun 2003 16:56:38 +0300 Subject: [PATCH 15/40] mysqld won't give a warning any more, if --user=user_name is used, if 'user_name' is the current user and it is not root. --- sql/mysqld.cc | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 8ed183e2f1f..315931094c2 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -1013,14 +1013,21 @@ static void set_ports() static void set_user(const char *user) { #if !defined(__WIN__) && !defined(OS2) && !defined(__NETWARE__) - struct passwd *ent; + struct passwd *ent; + uid_t user_id= geteuid(); // don't bother if we aren't superuser - if (geteuid()) + if (user_id) { if (user) - fprintf(stderr, - "Warning: One can only use the --user switch if running as root\n"); + { + /* Don't give a warning, if real user is same as given with --user */ + struct passwd *user_info= getpwnam(user); + + if (!user_info || user_id != user_info->pw_uid) + fprintf(stderr, + "Warning: One can only use the --user switch if running as root\n"); + } return; } else if (!user) From e6cdc816137b679e4c8d7c333eacf48d6afa1622 Mon Sep 17 00:00:00 2001 From: "monty@narttu.mysql.fi" <> Date: Thu, 5 Jun 2003 17:25:09 +0300 Subject: [PATCH 16/40] Added function comment --- mysys/thr_alarm.c | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/mysys/thr_alarm.c b/mysys/thr_alarm.c index 1f9c4c3b068..7a845e3eb00 100644 --- a/mysys/thr_alarm.c +++ b/mysys/thr_alarm.c @@ -122,9 +122,21 @@ void init_thr_alarm(uint max_alarms) /* Request alarm after sec seconds. - A pointer is returned with points to a non-zero int when the alarm has been - given. This can't be called from the alarm-handling thread. - Returns 0 if no more alarms are allowed (aborted by process) + + SYNOPSIS + thr_alarm() + alrm Pointer to alarm detection + alarm_data Structure to store in alarm queue + + NOTES + This function can't be called from the alarm-handling thread. + + RETURN VALUES + 0 ok + 1 If no more alarms are allowed (aborted by process) + + Stores in first argument a pointer to a non-zero int which is set to 0 + when the alarm has been given */ my_bool thr_alarm(thr_alarm_t *alrm, uint sec, ALARM *alarm_data) From 0058593c2d0a116c793968c7c0974f1903a62595 Mon Sep 17 00:00:00 2001 From: "guilhem@mysql.com" <> Date: Thu, 5 Jun 2003 17:02:00 +0200 Subject: [PATCH 17/40] Test for bug 578. And a comment in slave.cc. --- mysql-test/r/lock_tables_lost_commit.result | 8 ++++++++ .../t/lock_tables_lost_commit-master.opt | 1 + mysql-test/t/lock_tables_lost_commit.test | 18 +++++++++++++++++ sql/slave.cc | 20 +++++++++++++++++++ 4 files changed, 47 insertions(+) create mode 100644 mysql-test/r/lock_tables_lost_commit.result create mode 100644 mysql-test/t/lock_tables_lost_commit-master.opt create mode 100644 mysql-test/t/lock_tables_lost_commit.test diff --git a/mysql-test/r/lock_tables_lost_commit.result b/mysql-test/r/lock_tables_lost_commit.result new file mode 100644 index 00000000000..ccf56793f45 --- /dev/null +++ b/mysql-test/r/lock_tables_lost_commit.result @@ -0,0 +1,8 @@ +drop table if exists t1; +create table t1(a int) type=innodb; +lock tables t1 write; +insert into t1 values(10); +select * from t1; +a +10 +drop table t1; diff --git a/mysql-test/t/lock_tables_lost_commit-master.opt b/mysql-test/t/lock_tables_lost_commit-master.opt new file mode 100644 index 00000000000..d357a51cb27 --- /dev/null +++ b/mysql-test/t/lock_tables_lost_commit-master.opt @@ -0,0 +1 @@ +--binlog-ignore-db=test innodb \ No newline at end of file diff --git a/mysql-test/t/lock_tables_lost_commit.test b/mysql-test/t/lock_tables_lost_commit.test new file mode 100644 index 00000000000..a12ee7369cb --- /dev/null +++ b/mysql-test/t/lock_tables_lost_commit.test @@ -0,0 +1,18 @@ +# This is a test for bug 578 + +connect (con1,localhost,root,,); +connect (con2,localhost,root,,); + +connection con1; +drop table if exists t1; +create table t1(a int) type=innodb; +lock tables t1 write; +insert into t1 values(10); +disconnect con1; + +connection con2; +# The bug was that, because of the LOCK TABLES, the handler "forgot" to commit, +# and the other commit when we write to the binlog was not done because of +# binlog-ignore-db +select * from t1; +drop table t1; diff --git a/sql/slave.cc b/sql/slave.cc index c2762dbd6f4..bead4323b0c 100644 --- a/sql/slave.cc +++ b/sql/slave.cc @@ -1186,6 +1186,26 @@ int init_relay_log_info(RELAY_LOG_INFO* rli, const char* info_fname) strmov(strcend(tmp,'.'),"-relay-bin"); opt_relay_logname=my_strdup(tmp,MYF(MY_WME)); } + + /* + The relay log will now be opened, as a SEQ_READ_APPEND IO_CACHE. It is + notable that the last kilobytes of it (8 kB for example) may live in memory, + not on disk (depending on what the thread using it does). While this is + efficient, it has a side-effect one must know: + the size of the relay log on disk (displayed by 'ls -l' on Unix) can be a + few kilobytes less than one would expect by doing SHOW SLAVE STATUS; this + happens when only the IO thread is started (not the SQL thread). The + "missing" kilobytes are in memory, are preserved during 'STOP SLAVE; START + SLAVE IO_THREAD', and are flushed to disk when the slave's mysqld stops. So + this does not cause any bug. Example of how disk size grows by leaps: + + Read_Master_Log_Pos: 7811 -rw-rw---- 1 guilhem qq 4 Jun 5 16:19 gbichot2-relay-bin.002 + ...later... + Read_Master_Log_Pos: 9744 -rw-rw---- 1 guilhem qq 8192 Jun 5 16:27 gbichot2-relay-bin.002 + + See how 4 is less than 7811 and 8192 is less than 9744. + */ + if (open_log(&rli->relay_log, glob_hostname, opt_relay_logname, "-relay-bin", opt_relaylog_index_name, LOG_BIN, 1 /* read_append cache */, From 255a40c2f055be869e59efd5b4beff2d832ffbae Mon Sep 17 00:00:00 2001 From: "heikki@hundin.mysql.fi" <> Date: Fri, 6 Jun 2003 04:18:58 +0300 Subject: [PATCH 18/40] sql_yacc.yy, sql_parse.cc, sql_lex.h, mysqld.cc, lex.h: Add syntax SAVEPOINT id and ROLLBACK TO SAVEPOINT id. This is compatible with DB2 and Oracle but not with SQL Server. Savepoints do not do anything yet, this is just parsing. --- sql/lex.h | 1 + sql/mysqld.cc | 1 + sql/sql_lex.h | 4 +++- sql/sql_parse.cc | 3 +++ sql/sql_yacc.yy | 24 +++++++++++++++++++++--- 5 files changed, 29 insertions(+), 4 deletions(-) diff --git a/sql/lex.h b/sql/lex.h index d9a84dd25b4..3bbe1da185e 100644 --- a/sql/lex.h +++ b/sql/lex.h @@ -309,6 +309,7 @@ static SYMBOL symbols[] = { { "ROLLUP", SYM(ROLLUP_SYM),0,0}, { "ROW", SYM(ROW_SYM),0,0}, { "ROWS", SYM(ROWS_SYM),0,0}, + { "SAVEPOINT", SYM(SAVEPOINT_SYM),0,0}, { "SECOND", SYM(SECOND_SYM),0,0}, { "SELECT", SYM(SELECT_SYM),0,0}, { "SERIALIZABLE", SYM(SERIALIZABLE_SYM),0,0}, diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 315931094c2..7883b9700ad 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -4021,6 +4021,7 @@ struct show_var_st status_vars[]= { {"Com_restore_table", (char*) (com_stat+(uint) SQLCOM_RESTORE_TABLE),SHOW_LONG}, {"Com_revoke", (char*) (com_stat+(uint) SQLCOM_REVOKE),SHOW_LONG}, {"Com_rollback", (char*) (com_stat+(uint) SQLCOM_ROLLBACK),SHOW_LONG}, + {"Com_savepoint", (char*) (com_stat+(uint) SQLCOM_SAVEPOINT),SHOW_LONG}, {"Com_select", (char*) (com_stat+(uint) SQLCOM_SELECT),SHOW_LONG}, {"Com_set_option", (char*) (com_stat+(uint) SQLCOM_SET_OPTION),SHOW_LONG}, {"Com_show_binlog_events", (char*) (com_stat+(uint) SQLCOM_SHOW_BINLOG_EVENTS),SHOW_LONG}, diff --git a/sql/sql_lex.h b/sql/sql_lex.h index a905871e629..7d931399782 100644 --- a/sql/sql_lex.h +++ b/sql/sql_lex.h @@ -54,7 +54,8 @@ enum enum_sql_command { SQLCOM_CREATE_FUNCTION, SQLCOM_DROP_FUNCTION, SQLCOM_REVOKE,SQLCOM_OPTIMIZE, SQLCOM_CHECK, SQLCOM_FLUSH, SQLCOM_KILL, SQLCOM_ANALYZE, - SQLCOM_ROLLBACK, SQLCOM_COMMIT, SQLCOM_SLAVE_START, SQLCOM_SLAVE_STOP, + SQLCOM_ROLLBACK, SQLCOM_COMMIT, SQLCOM_SAVEPOINT, + SQLCOM_SLAVE_START, SQLCOM_SLAVE_STOP, SQLCOM_BEGIN, SQLCOM_LOAD_MASTER_TABLE, SQLCOM_CHANGE_MASTER, SQLCOM_RENAME_TABLE, SQLCOM_BACKUP_TABLE, SQLCOM_RESTORE_TABLE, SQLCOM_RESET, SQLCOM_PURGE, SQLCOM_SHOW_BINLOGS, @@ -154,6 +155,7 @@ typedef struct st_lex SQL_LIST proc_list, auxilliary_table_list; TYPELIB *interval; create_field *last_field; + char* savepoint_name; // Transaction savepoint id Item *default_value; CONVERT *convert_set; CONVERT *thd_convert_set; // Set with SET CHAR SET diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index b06a48f9045..7447ba44e76 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -2533,6 +2533,9 @@ mysql_execute_command(void) res= -1; thd->options&= ~(ulong) (OPTION_BEGIN | OPTION_STATUS_NO_TRANS_UPDATE); break; + case SQLCOM_SAVEPOINT: + send_ok(&thd->net); + break; default: /* Impossible */ send_ok(&thd->net); break; diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy index b0c81d6f6b0..2ef0992cdf7 100644 --- a/sql/sql_yacc.yy +++ b/sql/sql_yacc.yy @@ -135,6 +135,7 @@ bool my_yyoverflow(short **a, YYSTYPE **b,int *yystacksize); %token RESET_SYM %token ROLLBACK_SYM %token ROLLUP_SYM +%token SAVEPOINT_SYM %token SELECT_SYM %token SHOW %token SLAVE @@ -573,7 +574,8 @@ bool my_yyoverflow(short **a, YYSTYPE **b,int *yystacksize); query verb_clause create change select do drop insert replace insert2 insert_values update delete truncate rename show describe load alter optimize flush - reset purge begin commit rollback slave master_def master_defs + reset purge begin commit rollback savepoint + slave master_def master_defs repair restore backup analyze check start field_list field_list_item field_spec kill column_def key_def select_item_list select_item values_list no_braces @@ -649,6 +651,7 @@ verb_clause: | restore | revoke | rollback + | savepoint | select | set | slave @@ -3382,6 +3385,7 @@ keyword: | ROWS_SYM {} | ROW_FORMAT_SYM {} | ROW_SYM {} + | SAVEPOINT_SYM {} | SECOND_SYM {} | SERIALIZABLE_SYM {} | SESSION_SYM {} @@ -3915,8 +3919,22 @@ commit: COMMIT_SYM { Lex->sql_command = SQLCOM_COMMIT;}; rollback: - ROLLBACK_SYM { Lex->sql_command = SQLCOM_ROLLBACK;}; - + ROLLBACK_SYM + { + Lex->sql_command = SQLCOM_ROLLBACK; + Lex->savepoint_name = NULL; + } + | ROLLBACK_SYM TO_SYM SAVEPOINT_SYM ident + { + Lex->sql_command = SQLCOM_ROLLBACK; + Lex->savepoint_name = $4.str; + }; +savepoint: + SAVEPOINT_SYM ident + { + Lex->sql_command = SQLCOM_SAVEPOINT; + Lex->savepoint_name = $2.str; + }; /* ** UNIONS : glue selects together From 60fb005e5e0ed7c15be4132a24f703b321dd41a0 Mon Sep 17 00:00:00 2001 From: "guilhem@mysql.com" <> Date: Fri, 6 Jun 2003 16:41:28 +0200 Subject: [PATCH 19/40] Fix for bug 254 : we now make a distinction between if the master is < 3.23.57, 3.23 && >=57, and 4.x (before the 2 3.23 were one). This is because in 3.23.57 we have a way to distinguish between a Start_log_event written at server startup and one written at FLUSH LOGS, so we have a way to know if the slave must drop old temp tables or not. Change: mi->old_format was bool, now it's enum (to handle 3 cases). However, functions which had 'bool old_format' as an argument have their prototypes unchanged, because the old old_format == 0 now corresponds to the enum value BINLOG_FORMAT_CURRENT which is equal to 0, so boolean tests are left untouched. The only case were we use mi->old_format as an enum instead of casting it implicitly to a bool, is in Start_log_event::exec_event, where we want to distinguish between the 3 possible enum values. --- sql/log_event.cc | 40 ++++++++++++++++++++++++++++------------ sql/slave.cc | 20 ++++++++++++++++++-- sql/slave.h | 11 ++++++++--- 3 files changed, 54 insertions(+), 17 deletions(-) diff --git a/sql/log_event.cc b/sql/log_event.cc index 369ef940af2..ff968babcf0 100644 --- a/sql/log_event.cc +++ b/sql/log_event.cc @@ -2000,11 +2000,8 @@ int Load_log_event::exec_event(NET* net, struct st_relay_log_info* rli, IMPLEMENTATION - To handle the case where the master died without a stop event, - we clean up all temporary tables + locks that we got. - However, we don't clean temporary tables if the master was 3.23 - (this is because a 3.23 master writes a Start_log_event at every - binlog rotation; if we were not careful we would remove temp tables - on the slave when FLUSH LOGS is issued on the master). + we clean up all temporary tables that we got, if we are sure we + can (see below). TODO - Remove all active user locks @@ -2015,18 +2012,37 @@ int Load_log_event::exec_event(NET* net, struct st_relay_log_info* rli, int Start_log_event::exec_event(struct st_relay_log_info* rli) { - if (!rli->mi->old_format) - { + + switch (rli->mi->old_format) { + case BINLOG_FORMAT_CURRENT : /* - If 4.0 master, all temporary tables have been deleted on the master; - if 3.23 master, this is far from sure. + This is 4.x, so a Start_log_event is only at master startup, + so we are sure the master has restarted and cleared his temp tables. */ close_temporary_tables(thd); - /* - If we have old format, load_tmpdir is cleaned up by the I/O thread - */ cleanup_load_tmpdir(); + break; + /* + Now the older formats; in that case load_tmpdir is cleaned up by the I/O + thread. + */ + case BINLOG_FORMAT_323_LESS_57 : + /* + Cannot distinguish a Start_log_event generated at master startup and + one generated by master FLUSH LOGS, so cannot be sure temp tables + have to be dropped. So do nothing. + */ + break; + case BINLOG_FORMAT_323_GEQ_57 : + /* Can distinguish, based on the value of 'created' */ + if (created) /* this was generated at master startup*/ + close_temporary_tables(thd); + break; + default : + /* this case is impossible */ + return 1; } + return Log_event::exec_event(rli); } diff --git a/sql/slave.cc b/sql/slave.cc index bead4323b0c..cc0fa26027f 100644 --- a/sql/slave.cc +++ b/sql/slave.cc @@ -962,13 +962,19 @@ static int check_master_version(MYSQL* mysql, MASTER_INFO* mi) { const char* errmsg= 0; + /* + Note the following switch will bug when we have MySQL branch 30 ;) + */ switch (*mysql->server_version) { case '3': - mi->old_format = 1; + mi->old_format = + (strncmp(mysql->server_version, "3.23.57", 7) < 0) /* < .57 */ ? + BINLOG_FORMAT_323_LESS_57 : + BINLOG_FORMAT_323_GEQ_57 ; break; case '4': case '5': - mi->old_format = 0; + mi->old_format = BINLOG_FORMAT_CURRENT; break; default: errmsg = "Master reported unrecognized MySQL version"; @@ -1204,6 +1210,16 @@ int init_relay_log_info(RELAY_LOG_INFO* rli, const char* info_fname) Read_Master_Log_Pos: 9744 -rw-rw---- 1 guilhem qq 8192 Jun 5 16:27 gbichot2-relay-bin.002 See how 4 is less than 7811 and 8192 is less than 9744. + + WARNING: this is risky because the slave can stay like this for a long time; + then if it has a power failure, master.info says the I/O thread has read + until 9744 while the relay-log contains only until 8192 (the in-memory part + from 8192 to 9744 has been lost), so the SQL slave thread will miss some + events, silently breaking replication. + Ideally we would like to flush master.info only when we know that the relay + log has no in-memory tail. + Note that the above problem may arise only when only the IO thread is + started, which is unlikely. */ if (open_log(&rli->relay_log, glob_hostname, opt_relay_logname, diff --git a/sql/slave.h b/sql/slave.h index 8832302056d..66000f45e69 100644 --- a/sql/slave.h +++ b/sql/slave.h @@ -35,6 +35,11 @@ extern my_bool opt_log_slave_updates; extern ulonglong relay_log_space_limit; struct st_master_info; +enum enum_binlog_formats { + BINLOG_FORMAT_CURRENT=0, /* 0 is important for easy 'if (mi->old_format)' */ + BINLOG_FORMAT_323_LESS_57, + BINLOG_FORMAT_323_GEQ_57 }; + /* TODO: this needs to be redone, but for now it does not matter since we do not have multi-master yet. @@ -266,15 +271,15 @@ typedef struct st_master_info int events_till_abort; #endif bool inited; - bool old_format; /* master binlog is in 3.23 format */ + enum enum_binlog_formats old_format; /* master binlog is in 3.23 format */ volatile bool abort_slave, slave_running; volatile ulong slave_run_id; bool ignore_stop_event; st_master_info() - :fd(-1), io_thd(0), inited(0), old_format(0),abort_slave(0), - slave_running(0), slave_run_id(0) + :fd(-1), io_thd(0), inited(0), old_format(BINLOG_FORMAT_CURRENT), + abort_slave(0),slave_running(0), slave_run_id(0) { host[0] = 0; user[0] = 0; password[0] = 0; bzero(&file, sizeof(file)); From d640ff4a974d5bbb2751c28cfa6db3a53e69bd90 Mon Sep 17 00:00:00 2001 From: "monty@narttu.mysql.fi" <> Date: Tue, 10 Jun 2003 21:42:29 +0300 Subject: [PATCH 20/40] Don't install signal handler for SIGINT by default Added option --gdb Free memory used by replicate_xxx and binglog_xxx options --- mysql-test/mysql-test-run.sh | 6 +++++ mysys/thr_alarm.c | 9 ++++---- sql/mysql_priv.h | 5 +++++ sql/mysqld.cc | 43 +++++++++++++++++++++++++++--------- sql/repl_failsafe.cc | 1 + sql/sql_list.cc | 15 +++++++++++++ 6 files changed, 65 insertions(+), 14 deletions(-) diff --git a/mysql-test/mysql-test-run.sh b/mysql-test/mysql-test-run.sh index ab4a5354dae..3fe9070bbd6 100644 --- a/mysql-test/mysql-test-run.sh +++ b/mysql-test/mysql-test-run.sh @@ -319,11 +319,15 @@ while test $# -gt 0; do $ECHO "Note: you will get more meaningful output on a source distribution compiled with debugging option when running tests with --client-gdb option" fi DO_CLIENT_GDB=1 + EXTRA_MASTER_MYSQLD_OPT="$EXTRA_MASTER_MYSQLD_OPT --gdb" + EXTRA_SLAVE_MYSQLD_OPT="$EXTRA_SLAVE_MYSQLD_OPT --gdb" ;; --manual-gdb ) DO_GDB=1 MANUAL_GDB=1 USE_RUNNING_SERVER="" + EXTRA_MASTER_MYSQLD_OPT="$EXTRA_MASTER_MYSQLD_OPT --gdb" + EXTRA_SLAVE_MYSQLD_OPT="$EXTRA_SLAVE_MYSQLD_OPT --gdb" ;; --ddd ) if [ x$BINARY_DIST = x1 ] ; then @@ -331,6 +335,8 @@ while test $# -gt 0; do fi DO_DDD=1 USE_RUNNING_SERVER="" + EXTRA_MASTER_MYSQLD_OPT="$EXTRA_MASTER_MYSQLD_OPT --gdb" + EXTRA_SLAVE_MYSQLD_OPT="$EXTRA_SLAVE_MYSQLD_OPT --gdb" ;; --valgrind) VALGRIND="valgrind --alignment=8 --leak-check=yes --num-callers=16" diff --git a/mysys/thr_alarm.c b/mysys/thr_alarm.c index 7a845e3eb00..2a16eeec215 100644 --- a/mysys/thr_alarm.c +++ b/mysys/thr_alarm.c @@ -220,8 +220,7 @@ void thr_end_alarm(thr_alarm_t *alarmed) { ALARM *alarm_data; sigset_t old_mask; - uint i; - my_bool found=0; + uint i, found=0; DBUG_ENTER("thr_end_alarm"); pthread_sigmask(SIG_BLOCK,&full_signal_set,&old_mask); @@ -235,11 +234,13 @@ void thr_end_alarm(thr_alarm_t *alarmed) queue_remove(&alarm_queue,i),MYF(0); if (alarm_data->malloced) my_free((gptr) alarm_data,MYF(0)); - found=1; + found++; +#ifndef DBUG_OFF break; +#endif } } - DBUG_ASSERT(!*alarmed || found); + DBUG_ASSERT(!*alarmed || found == 1); if (!found) { if (*alarmed) diff --git a/sql/mysql_priv.h b/sql/mysql_priv.h index 9cf18b53669..702db98748a 100644 --- a/sql/mysql_priv.h +++ b/sql/mysql_priv.h @@ -140,6 +140,7 @@ char* query_table_status(THD *thd,const char *db,const char *table_name); #define TEST_NO_EXTRA 128 #define TEST_CORE_ON_SIGNAL 256 /* Give core if signal */ #define TEST_NO_STACKTRACE 512 +#define TEST_SIGINT 1024 /* Allow sigint on threads */ /* options for select set by the yacc parser (stored in lex->options) */ #define SELECT_DISTINCT 1 @@ -820,6 +821,10 @@ Item *get_system_var(enum_var_type var_type, LEX_STRING name); /* log.cc */ bool flush_error_log(void); +/* sql_list.cc */ +void free_list(I_List *list); +void free_list(I_List *list); + /* Some inline functions for more speed */ inline bool add_item_to_list(Item *item) diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 7883b9700ad..ca98ab96710 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -285,7 +285,7 @@ char log_error_file[FN_REFLEN]; bool opt_log, opt_update_log, opt_bin_log, opt_slow_log; bool opt_error_log= IF_WIN(1,0); bool opt_disable_networking=0, opt_skip_show_db=0; -my_bool opt_enable_named_pipe= 0; +my_bool opt_enable_named_pipe= 0, opt_debugging= 0; my_bool opt_local_infile, opt_external_locking, opt_slave_compressed_protocol; uint delay_key_write_options= (uint) DELAY_KEY_WRITE_ON; @@ -387,7 +387,7 @@ bool opt_using_transactions, using_update_log; bool volatile abort_loop, select_thread_in_use, signal_thread_in_use; bool volatile ready_to_exit, shutdown_in_progress, grant_option; ulong refresh_version=1L,flush_version=1L; /* Increments on each reload */ -ulong query_id=1L,long_query_count,aborted_threads, +ulong query_id=1L,long_query_count,aborted_threads, killed_threads, aborted_connects,delayed_insert_timeout,delayed_insert_limit, delayed_queue_size,delayed_insert_threads,delayed_insert_writes, delayed_rows_in_use,delayed_insert_errors,flush_time, thread_created; @@ -919,6 +919,12 @@ void clean_up(bool print_message) bitmap_free(&temp_pool); free_max_user_conn(); end_slave_list(); + free_list(&replicate_do_db); + free_list(&replicate_ignore_db); + free_list(&binlog_do_db); + free_list(&binlog_ignore_db); + free_list(&replicate_rewrite_db); + #ifdef HAVE_OPENSSL if (ssl_acceptor_fd) my_free((gptr) ssl_acceptor_fd, MYF(MY_ALLOW_ZERO_PTR)); @@ -1273,7 +1279,10 @@ extern "C" sig_handler end_thread_signal(int sig __attribute__((unused))) THD *thd=current_thd; DBUG_ENTER("end_thread_signal"); if (thd && ! thd->bootstrap) + { + statistic_increment(killed_threads, &LOCK_status); end_thread(thd,0); + } DBUG_VOID_RETURN; /* purecov: deadcode */ } @@ -1592,7 +1601,8 @@ static void init_signals(void) struct sigaction sa; DBUG_ENTER("init_signals"); - sigset(THR_KILL_SIGNAL,end_thread_signal); + if (test_flags & TEST_SIGINT) + sigset(THR_KILL_SIGNAL,end_thread_signal); sigset(THR_SERVER_ALARM,print_signal_warning); // Should never be called! if (!(test_flags & TEST_NO_STACKTRACE) || (test_flags & TEST_CORE_ON_SIGNAL)) @@ -1651,7 +1661,8 @@ static void init_signals(void) sigaddset(&set,SIGTSTP); #endif sigaddset(&set,THR_SERVER_ALARM); - sigdelset(&set,THR_KILL_SIGNAL); // May be SIGINT + if (test_flags & TEST_SIGINT) + sigdelset(&set,THR_KILL_SIGNAL); // May be SIGINT sigdelset(&set,THR_CLIENT_ALARM); // For alarms sigprocmask(SIG_SETMASK,&set,NULL); pthread_sigmask(SIG_SETMASK,&set,NULL); @@ -1707,9 +1718,12 @@ extern "C" void *signal_hand(void *arg __attribute__((unused))) */ init_thr_alarm(max_connections+max_insert_delayed_threads+10); #if SIGINT != THR_KILL_SIGNAL - (void) sigemptyset(&set); // Setup up SIGINT for debug - (void) sigaddset(&set,SIGINT); // For debugging - (void) pthread_sigmask(SIG_UNBLOCK,&set,NULL); + if (test_flags & TEST_SIGINT) + { + (void) sigemptyset(&set); // Setup up SIGINT for debug + (void) sigaddset(&set,SIGINT); // For debugging + (void) pthread_sigmask(SIG_UNBLOCK,&set,NULL); + } #endif (void) sigemptyset(&set); // Setup up SIGINT for debug #ifdef USE_ONE_SIGNAL_HAND @@ -2715,6 +2729,7 @@ static void create_new_thread(THD *thd) thread_count--; thd->killed=1; // Safety (void) pthread_mutex_unlock(&LOCK_thread_count); + statistic_increment(aborted_connects,&LOCK_status); net_printf(net,ER_CANT_CREATE_THREAD,error); (void) pthread_mutex_lock(&LOCK_thread_count); close_connection(net,0,0); @@ -3145,7 +3160,7 @@ enum options { OPT_QUERY_CACHE_TYPE, OPT_RECORD_BUFFER, OPT_RECORD_RND_BUFFER, OPT_RELAY_LOG_SPACE_LIMIT, OPT_SLAVE_NET_TIMEOUT, OPT_SLAVE_COMPRESSED_PROTOCOL, OPT_SLOW_LAUNCH_TIME, - OPT_READONLY, + OPT_READONLY, OPT_DEBUGGING, OPT_SORT_BUFFER, OPT_TABLE_CACHE, OPT_THREAD_CONCURRENCY, OPT_THREAD_CACHE_SIZE, OPT_TMP_TABLE_SIZE, OPT_THREAD_STACK, @@ -3277,6 +3292,10 @@ struct my_option my_long_options[] = GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, /* We must always support the next option to make scripts like mysqltest easier to do */ + {"gdb", OPT_DEBUGGING, + "Set up signals usable for debugging", + (gptr*) &opt_debugging, (gptr*) &opt_debugging, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, {"init-rpl-role", OPT_INIT_RPL_ROLE, "Set the replication role", 0, 0, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"innodb_data_file_path", OPT_INNODB_DATA_FILE_PATH, @@ -3469,8 +3488,6 @@ Does nothing yet.", OPT_ARG, 0, 0, 0, 0, 0, 0}, {"port", 'P', "Port number to use for connection.", (gptr*) &mysql_port, (gptr*) &mysql_port, 0, GET_UINT, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, - {"reckless-slave", OPT_RECKLESS_SLAVE, "For debugging", 0, 0, 0, GET_NO_ARG, - NO_ARG, 0, 0, 0, 0, 0, 0}, {"replicate-do-db", OPT_REPLICATE_DO_DB, "Tells the slave thread to restrict replication to the specified database. To specify more than one database, use the directive multiple times, once for each database. Note that this will only work if you do not use cross-database queries such as UPDATE some_db.some_table SET foo='bar' while having selected a different or no database. If you need cross database updates to work, make sure you have 3.23.28 or later, and use replicate-wild-do-table=db_name.%.", 0, 0, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, @@ -4717,6 +4734,12 @@ static void get_options(int argc,char **argv) have_symlink=SHOW_OPTION_DISABLED; } #endif + if (opt_debugging) + { + /* Allow break with SIGINT, no core or stack trace */ + test_flags|= TEST_SIGINT | TEST_NO_STACKTRACE; + test_flags&= ~TEST_CORE_ON_SIGNAL; + } /* Set global MyISAM variables from delay_key_write_options */ fix_delay_key_write((THD*) 0, OPT_GLOBAL); diff --git a/sql/repl_failsafe.cc b/sql/repl_failsafe.cc index 8ed002ca649..1552b3994e9 100644 --- a/sql/repl_failsafe.cc +++ b/sql/repl_failsafe.cc @@ -71,6 +71,7 @@ static int init_failsafe_rpl_thread(THD* thd) if (init_thr_lock() || thd->store_globals()) { close_connection(&thd->net,ER_OUT_OF_RESOURCES); // is this needed? + statistic_increment(aborted_connects,&LOCK_status); end_thread(thd,0); DBUG_RETURN(-1); } diff --git a/sql/sql_list.cc b/sql/sql_list.cc index 1124605ca24..c99cfb8c918 100644 --- a/sql/sql_list.cc +++ b/sql/sql_list.cc @@ -22,3 +22,18 @@ #include "mysql_priv.h" list_node end_of_list; + +void free_list(I_List *list) +{ + i_string_pair *tmp; + while ((tmp= list->get())) + delete tmp; +} + + +void free_list(I_List *list) +{ + i_string *tmp; + while ((tmp= list->get())) + delete tmp; +} From 96c8d91a0cc36d3b66c1c08e82527ebe04d447d2 Mon Sep 17 00:00:00 2001 From: "guilhem@mysql.com" <> Date: Tue, 10 Jun 2003 23:29:49 +0200 Subject: [PATCH 21/40] More error messages. This is intended to help debugging; presently I have a support issue with an unclear message which can have N reasons for appearing. This should help us know at which point it failed, and get the errno when my_open was involved (as the reason for the unclear message is often a permission problem). RESET SLAVE resets last_error and last_errno in SHOW SLAVE STATUS (without this, rpl_loaddata.test, which is expected to generate an error in last_error, influenced rpl_log_pos.test). A small test update. Added STOP SLAVE to mysql-test-run to get rid of several stupid error messages which are printed while the master restarts and the slave attempts/manages to connect to it and sends it nonsense binlog requests. --- mysql-test/mysql-test-run.sh | 16 +++++ mysql-test/r/rpl000018.result | 1 + mysql-test/t/rpl000018.test | 2 + sql/slave.cc | 119 ++++++++++++++++++++++++++-------- sql/sql_repl.cc | 15 ++++- 5 files changed, 125 insertions(+), 28 deletions(-) diff --git a/mysql-test/mysql-test-run.sh b/mysql-test/mysql-test-run.sh index ab4a5354dae..c688380b402 100644 --- a/mysql-test/mysql-test-run.sh +++ b/mysql-test/mysql-test-run.sh @@ -1064,6 +1064,16 @@ stop_slave () fi } +stop_slave_threads () +{ + eval "this_slave_running=\$SLAVE$1_RUNNING" + slave_ident="slave$1" + if [ x$this_slave_running = x1 ] + then + $MYSQLADMIN --no-defaults -uroot --socket=$MYSQL_TMP_DIR/$slave_ident.sock stop-slave > /dev/null 2>&1 + fi +} + stop_master () { if [ x$MASTER_RUNNING = x1 ] @@ -1157,6 +1167,12 @@ run_testcase () return fi + # Stop all slave threads, so that we don't have useless reconnection attempts + # and error messages in case the slave and master servers restart. + stop_slave_threads + stop_slave_threads 1 + stop_slave_threads 2 + if [ -z "$USE_RUNNING_SERVER" ] ; then if [ -f $master_opt_file ] ; diff --git a/mysql-test/r/rpl000018.result b/mysql-test/r/rpl000018.result index ba51406bba0..282c1e492a1 100644 --- a/mysql-test/r/rpl000018.result +++ b/mysql-test/r/rpl000018.result @@ -1,3 +1,4 @@ +reset master; reset slave; slave start; show master logs; diff --git a/mysql-test/t/rpl000018.test b/mysql-test/t/rpl000018.test index 291b482b912..3bd5fd0ef09 100644 --- a/mysql-test/t/rpl000018.test +++ b/mysql-test/t/rpl000018.test @@ -6,6 +6,8 @@ require_manager; connect (master,localhost,root,,test,0,master.sock); connect (slave,localhost,root,,test,0,slave.sock); +connection master; +reset master; server_stop master; server_start master; connection slave; diff --git a/sql/slave.cc b/sql/slave.cc index cc0fa26027f..d2bc5b2f758 100644 --- a/sql/slave.cc +++ b/sql/slave.cc @@ -115,6 +115,8 @@ int init_slave() { DBUG_ENTER("init_slave"); + /* This is called when mysqld starts */ + /* TODO: re-write this to interate through the list of files for multi-master @@ -126,11 +128,16 @@ int init_slave() If master_host is specified, create the master_info file if it doesn't exists. */ - if (!active_mi || - init_master_info(active_mi,master_info_file,relay_log_info_file, + if (!active_mi) + { + sql_print_error("Failed to allocate memory for the master info structure"); + goto err; + } + + if(init_master_info(active_mi,master_info_file,relay_log_info_file, !master_host)) { - sql_print_error("Note: Failed to initialized master info"); + sql_print_error("Failed to initialize the master info structure"); goto err; } @@ -150,7 +157,7 @@ int init_slave() relay_log_info_file, SLAVE_IO | SLAVE_SQL)) { - sql_print_error("Warning: Can't create threads to handle slave"); + sql_print_error("Failed to create slave threads"); goto err; } } @@ -1226,7 +1233,10 @@ int init_relay_log_info(RELAY_LOG_INFO* rli, const char* info_fname) "-relay-bin", opt_relaylog_index_name, LOG_BIN, 1 /* read_append cache */, 1 /* no auto events */)) + { + sql_print_error("Failed in open_log() called from init_relay_log_info()"); DBUG_RETURN(1); + } /* if file does not exist */ if (access(fname,F_OK)) @@ -1237,10 +1247,18 @@ int init_relay_log_info(RELAY_LOG_INFO* rli, const char* info_fname) */ if (info_fd >= 0) my_close(info_fd, MYF(MY_WME)); - if ((info_fd = my_open(fname, O_CREAT|O_RDWR|O_BINARY, MYF(MY_WME))) < 0 || - init_io_cache(&rli->info_file, info_fd, IO_SIZE*2, READ_CACHE, 0L,0, - MYF(MY_WME))) + if ((info_fd = my_open(fname, O_CREAT|O_RDWR|O_BINARY, MYF(MY_WME))) < 0) { + sql_print_error("Failed to create a new relay log info file (\ +file '%s', errno %d)", fname, my_errno); + msg= current_thd->net.last_error; + goto err; + } + if (init_io_cache(&rli->info_file, info_fd, IO_SIZE*2, READ_CACHE, 0L,0, + MYF(MY_WME))) + { + sql_print_error("Failed to create a cache on relay log info file (\ +file '%s')", fname); msg= current_thd->net.last_error; goto err; } @@ -1248,7 +1266,11 @@ int init_relay_log_info(RELAY_LOG_INFO* rli, const char* info_fname) /* Init relay log with first entry in the relay index file */ if (init_relay_log_pos(rli,NullS,BIN_LOG_HEADER_SIZE,0 /* no data lock */, &msg)) + { + sql_print_error("Failed to open the relay log (relay_log_name='FIRST', \ +relay_log_pos=4"); goto err; + } rli->master_log_name[0]= 0; rli->master_log_pos= 0; rli->info_fd= info_fd; @@ -1257,18 +1279,33 @@ int init_relay_log_info(RELAY_LOG_INFO* rli, const char* info_fname) { if (info_fd >= 0) reinit_io_cache(&rli->info_file, READ_CACHE, 0L,0,0); - else if ((info_fd = my_open(fname, O_RDWR|O_BINARY, MYF(MY_WME))) < 0 || - init_io_cache(&rli->info_file, info_fd, - IO_SIZE*2, READ_CACHE, 0L, 0, MYF(MY_WME))) + else { - if (info_fd >= 0) - my_close(info_fd, MYF(0)); - rli->info_fd= -1; - rli->relay_log.close(1); - pthread_mutex_unlock(&rli->data_lock); - DBUG_RETURN(1); + int error=0; + if ((info_fd = my_open(fname, O_RDWR|O_BINARY, MYF(MY_WME))) < 0) + { + sql_print_error("Failed to open the existing relay log info file (\ +file '%s', errno %d)", fname, my_errno); + error= 1; + } + else if (init_io_cache(&rli->info_file, info_fd, + IO_SIZE*2, READ_CACHE, 0L, 0, MYF(MY_WME))) + { + sql_print_error("Failed to create a cache on relay log info file (\ +file '%s')", fname); + error= 1; + } + if (error) + { + if (info_fd >= 0) + my_close(info_fd, MYF(0)); + rli->info_fd= -1; + rli->relay_log.close(1); + pthread_mutex_unlock(&rli->data_lock); + DBUG_RETURN(1); + } } - + rli->info_fd = info_fd; int relay_log_pos, master_log_pos; if (init_strvar_from_file(rli->relay_log_name, @@ -1292,7 +1329,12 @@ int init_relay_log_info(RELAY_LOG_INFO* rli, const char* info_fname) rli->relay_log_pos, 0 /* no data lock*/, &msg)) + { + char llbuf[22]; + sql_print_error("Failed to open the relay log (relay_log_name='%s', \ +relay_log_pos=%s", rli->relay_log_name, llstr(rli->relay_log_pos, llbuf)); goto err; + } } DBUG_ASSERT(rli->relay_log_pos >= BIN_LOG_HEADER_SIZE); DBUG_ASSERT(my_b_tell(rli->cur_log) == rli->relay_log_pos); @@ -1301,7 +1343,8 @@ int init_relay_log_info(RELAY_LOG_INFO* rli, const char* info_fname) before flush_relay_log_info */ reinit_io_cache(&rli->info_file, WRITE_CACHE,0L,0,1); - error= flush_relay_log_info(rli); + if ((error= flush_relay_log_info(rli))) + sql_print_error("Failed to flush relay log info file"); if (count_relay_log_space(rli)) { msg="Error counting relay log space"; @@ -1404,6 +1447,8 @@ int init_master_info(MASTER_INFO* mi, const char* master_info_fname, pthread_mutex_lock(&mi->data_lock); fd = mi->fd; + + /* does master.info exist ? */ if (access(fname,F_OK)) { @@ -1418,10 +1463,19 @@ int init_master_info(MASTER_INFO* mi, const char* master_info_fname, */ if (fd >= 0) my_close(fd, MYF(MY_WME)); - if ((fd = my_open(fname, O_CREAT|O_RDWR|O_BINARY, MYF(MY_WME))) < 0 || - init_io_cache(&mi->file, fd, IO_SIZE*2, READ_CACHE, 0L,0, - MYF(MY_WME))) + if ((fd = my_open(fname, O_CREAT|O_RDWR|O_BINARY, MYF(MY_WME))) < 0 ) + { + sql_print_error("Failed to create a new master info file (\ +file '%s', errno %d)", fname, my_errno); goto err; + } + if (init_io_cache(&mi->file, fd, IO_SIZE*2, READ_CACHE, 0L,0, + MYF(MY_WME))) + { + sql_print_error("Failed to create a cache on master info file (\ +file '%s')", fname); + goto err; + } mi->master_log_name[0] = 0; mi->master_log_pos = BIN_LOG_HEADER_SIZE; // skip magic number @@ -1440,10 +1494,22 @@ int init_master_info(MASTER_INFO* mi, const char* master_info_fname, { if (fd >= 0) reinit_io_cache(&mi->file, READ_CACHE, 0L,0,0); - else if ((fd = my_open(fname, O_RDWR|O_BINARY, MYF(MY_WME))) < 0 || - init_io_cache(&mi->file, fd, IO_SIZE*2, READ_CACHE, 0L, - 0, MYF(MY_WME))) - goto err; + else + { + if ((fd = my_open(fname, O_RDWR|O_BINARY, MYF(MY_WME))) < 0 ) + { + sql_print_error("Failed to open the existing master info file (\ +file '%s', errno %d)", fname, my_errno); + goto err; + } + if (init_io_cache(&mi->file, fd, IO_SIZE*2, READ_CACHE, 0L, + 0, MYF(MY_WME))) + { + sql_print_error("Failed to create a cache on master info file (\ +file '%s')", fname); + goto err; + } + } mi->fd = fd; int port, connect_retry, master_log_pos; @@ -1484,7 +1550,8 @@ int init_master_info(MASTER_INFO* mi, const char* master_info_fname, mi->inited = 1; // now change cache READ -> WRITE - must do this before flush_master_info reinit_io_cache(&mi->file, WRITE_CACHE,0L,0,1); - error=test(flush_master_info(mi)); + if ((error=test(flush_master_info(mi)))) + sql_print_error("Failed to flush master info file"); pthread_mutex_unlock(&mi->data_lock); DBUG_RETURN(error); diff --git a/sql/sql_repl.cc b/sql/sql_repl.cc index ca993c053a1..a651d8002fd 100644 --- a/sql/sql_repl.cc +++ b/sql/sql_repl.cc @@ -159,10 +159,18 @@ File open_binlog(IO_CACHE *log, const char *log_file_name, File file; DBUG_ENTER("open_binlog"); - if ((file = my_open(log_file_name, O_RDONLY | O_BINARY, MYF(MY_WME))) < 0 || - init_io_cache(log, file, IO_SIZE*2, READ_CACHE, 0, 0, + if ((file = my_open(log_file_name, O_RDONLY | O_BINARY, MYF(MY_WME))) < 0) + { + sql_print_error("Failed to open log (\ +file '%s', errno %d)", log_file_name, my_errno); + *errmsg = "Could not open log file"; // This will not be sent + goto err; + } + if (init_io_cache(log, file, IO_SIZE*2, READ_CACHE, 0, 0, MYF(MY_WME | MY_DONT_CHECK_FILESIZE))) { + sql_print_error("Failed to create a cache on log (\ +file '%s')", log_file_name); *errmsg = "Could not open log file"; // This will not be sent goto err; } @@ -743,6 +751,9 @@ int reset_slave(THD *thd, MASTER_INFO* mi) //Clear master's log coordinates (only for good display of SHOW SLAVE STATUS) mi->master_log_name[0]= 0; mi->master_log_pos= BIN_LOG_HEADER_SIZE; + //Clear the errors displayed by SHOW SLAVE STATUS + mi->rli.last_slave_error[0]=0; + mi->rli.last_slave_errno=0; //close master_info_file, relay_log_info_file, set mi->inited=rli->inited=0 end_master_info(mi); //and delete these two files From 90543684b4ad445562e267e430aa14cfffb477c7 Mon Sep 17 00:00:00 2001 From: "lenz@mysql.com" <> Date: Wed, 11 Jun 2003 13:38:02 +0200 Subject: [PATCH 22/40] - fixed a path to init script in RPM spec file (/sbin/init.d is obsolete) --- support-files/mysql.spec.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/support-files/mysql.spec.sh b/support-files/mysql.spec.sh index 06ba2d63f45..6cc33866efd 100644 --- a/support-files/mysql.spec.sh +++ b/support-files/mysql.spec.sh @@ -327,7 +327,7 @@ install -m755 $MBD/support-files/mysql.server $RBR/etc/init.d/mysql # Create a symlink "rcmysql", pointing to the init.script. SuSE users # will appreciate that, as all services usually offer this. -ln -s ../../sbin/init.d/mysql $RPM_BUILD_ROOT/usr/sbin/rcmysql +ln -s ../../etc/init.d/mysql $RPM_BUILD_ROOT/usr/sbin/rcmysql # Create symbolic compatibility link safe_mysqld -> mysqld_safe # (safe_mysqld will be gone in MySQL 4.1) From 2daa5643d3c9a5218f75a47476fa5624ad477cfe Mon Sep 17 00:00:00 2001 From: "lenz@mysql.com" <> Date: Wed, 11 Jun 2003 13:40:20 +0200 Subject: [PATCH 23/40] - applied patch to mysql_explain_log.sh provided by Dennis Haney to accept --socket option (Bug #592) --- scripts/mysql_explain_log.sh | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/scripts/mysql_explain_log.sh b/scripts/mysql_explain_log.sh index c4a4ef21568..973d9e8a363 100644 --- a/scripts/mysql_explain_log.sh +++ b/scripts/mysql_explain_log.sh @@ -14,12 +14,14 @@ $Param->{host}=''; $Param->{user}=''; $Param->{password}=''; $Param->{PrintError}=0; +$Param->{socket}=''; if (!GetOptions ('date|d:i' => \$Param->{ViewDate}, 'host|h:s' => \$Param->{host}, 'user|u:s' => \$Param->{user}, 'password|p:s' => \$Param->{password}, 'printerror|e:s' => \$Param->{PrintError}, + 'socket|s:s' => \$Param->{socket}, )) { ShowOptions(); } @@ -50,7 +52,7 @@ else { #print "Date=$Param->{ViewDate}, host=$Param->{host}, user=$Param->{user}, password=$Param->{password}\n"; - $Param->{dbh}=DBI->connect("DBI:mysql:host=$Param->{host}",$Param->{user},$Param->{password},{PrintError=>0}); + $Param->{dbh}=DBI->connect("DBI:mysql:host=$Param->{host}".($Param->{socket}?";mysql_socket=$Param->{socket}":""),$Param->{user},$Param->{password},{PrintError=>0}); if (DBI::err()) { print "Error: " . DBI::errstr() . "\n"; } @@ -313,6 +315,8 @@ Usage: $0 [OPTIONS] < LOGFILE -u=USERNAME --password=PASSWORD password of db-user -p=PASSWORD +--socket=SOCKET mysqld socket file to connect +-s=SOCKET Read logfile from STDIN an try to EXPLAIN all SELECT statements. All UPDATE statements are rewritten to an EXPLAIN SELECT statement. The results of the EXPLAIN statement are collected and counted. All results with type=ALL are collected in an separete list. Results are printed to STDOUT. @@ -344,7 +348,7 @@ Then add indices to avoid table scans and remove those which aren't used. =head1 USAGE -explain_log.pl [--date=YYMMDD] --host=dbhost] [--user=dbuser] [--password=dbpw] < logfile +explain_log.pl [--date=YYMMDD] --host=dbhost] [--user=dbuser] [--password=dbpw] [--socket=/path/to/socket] < logfile --date=YYMMDD select only entrys of date @@ -362,14 +366,19 @@ explain_log.pl [--date=YYMMDD] --host=dbhost] [--user=dbuser] [--password=dbpw] -p=PASSWORD +--socket=SOCKET change path to the socket + +-s=SOCKET + =head1 EXAMPLE explain_log.pl --host=localhost --user=foo --password=bar < /var/lib/mysql/mobile.log -=head1 AUTHOR +=head1 AUTHORS Stefan Nitz Jan Willamowius , http://www.mobile.de + Dennis Haney (Added socket support) =head1 RECRUITING From 8fea9b451abeb0fbcc6001e3823590e6bb57e245 Mon Sep 17 00:00:00 2001 From: "hf@deer.(none)" <> Date: Wed, 11 Jun 2003 20:09:37 +0500 Subject: [PATCH 24/40] Bugfix for #614 Item_extract needs special implementation for eq(). Item_func::eq doesn't work correctly because we have to compare Item_extract::int_type parameters also We need to propagate this to 4.1 --- BitKeeper/etc/logging_ok | 1 + sql/item_timefunc.cc | 16 ++++++++++++++++ sql/item_timefunc.h | 1 + 3 files changed, 18 insertions(+) diff --git a/BitKeeper/etc/logging_ok b/BitKeeper/etc/logging_ok index 4617e9d697b..092b6f3f2a5 100644 --- a/BitKeeper/etc/logging_ok +++ b/BitKeeper/etc/logging_ok @@ -24,6 +24,7 @@ heikki@donna.mysql.fi heikki@hundin.mysql.fi heikki@rescue. heikki@work.mysql.com +hf@deer.(none) hf@deer.mysql.r18.ru hf@genie.(none) igor@hundin.mysql.fi diff --git a/sql/item_timefunc.cc b/sql/item_timefunc.cc index 6a95c15a226..84e7a44ac61 100644 --- a/sql/item_timefunc.cc +++ b/sql/item_timefunc.cc @@ -1137,6 +1137,22 @@ longlong Item_extract::val_int() return 0; // Impossible } +bool Item_extract::eq(const Item *item, bool binary_cmp) const +{ + if (this == item) + return 1; + if (item->type() != FUNC_ITEM || + func_name() != ((Item_func*)item)->func_name()) + return 0; + + Item_extract* ie= (Item_extract*)item; + if (ie->int_type != int_type) + return 0; + + if (!args[0]->eq(ie->args[0], binary_cmp)) + return 0; + return 1; +} void Item_typecast::print(String *str) { diff --git a/sql/item_timefunc.h b/sql/item_timefunc.h index 0ca2a36609d..e04e24627d9 100644 --- a/sql/item_timefunc.h +++ b/sql/item_timefunc.h @@ -422,6 +422,7 @@ class Item_extract :public Item_int_func longlong val_int(); const char *func_name() const { return "extract"; } void fix_length_and_dec(); + bool eq(const Item *item, bool binary_cmp) const; unsigned int size_of() { return sizeof(*this);} }; From db088dc477d8ed59d7123a45f66da4c39dd57e57 Mon Sep 17 00:00:00 2001 From: "hf@deer.(none)" <> Date: Wed, 11 Jun 2003 22:07:23 +0500 Subject: [PATCH 25/40] test case for bug #614 --- mysql-test/r/func_time.result | 3 +++ mysql-test/t/func_time.test | 2 ++ 2 files changed, 5 insertions(+) diff --git a/mysql-test/r/func_time.result b/mysql-test/r/func_time.result index 2941352c776..06c0be86667 100644 --- a/mysql-test/r/func_time.result +++ b/mysql-test/r/func_time.result @@ -329,6 +329,9 @@ insert into t1 values ('2001-01-12 12:23:40'); select ctime, hour(ctime) from t1; ctime hour(ctime) 2001-01-12 12:23:40 12 +select ctime from t1 where extract(MONTH FROM ctime) = 1 AND extract(YEAR FROM ctime) = 2001; +ctime +2001-01-12 12:23:40 drop table t1; create table t1 (id int); create table t2 (id int, date date); diff --git a/mysql-test/t/func_time.test b/mysql-test/t/func_time.test index dd589ff2e66..3057729ab96 100644 --- a/mysql-test/t/func_time.test +++ b/mysql-test/t/func_time.test @@ -123,6 +123,8 @@ select extract(MONTH FROM "2001-02-00"); create table t1 (ctime varchar(20)); insert into t1 values ('2001-01-12 12:23:40'); select ctime, hour(ctime) from t1; +# test bug 614 (multiple extracts in where) +select ctime from t1 where extract(MONTH FROM ctime) = 1 AND extract(YEAR FROM ctime) = 2001; drop table t1; # From 63df5f7cb71ce01fa5f3dfc1a072cc57e8aab32b Mon Sep 17 00:00:00 2001 From: "lenz@mysql.com" <> Date: Thu, 12 Jun 2003 13:52:24 +0200 Subject: [PATCH 26/40] - applied patch for mysqld_safe from Christian Hammers to be able to define a different niceness level in my.cnf (Bug #627) --- scripts/mysqld_safe.sh | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/scripts/mysqld_safe.sh b/scripts/mysqld_safe.sh index e400c27b84c..626e04b1579 100644 --- a/scripts/mysqld_safe.sh +++ b/scripts/mysqld_safe.sh @@ -70,6 +70,7 @@ parse_arguments() { MYSQLD="mysqld" fi ;; + --nice=*) niceness=`echo "$arg" | sed -e "s;--nice=;;"` ;; *) if test -n "$pick_args" then @@ -110,6 +111,7 @@ fi MYSQL_UNIX_PORT=${MYSQL_UNIX_PORT:-@MYSQL_UNIX_ADDR@} MYSQL_TCP_PORT=${MYSQL_TCP_PORT:-@MYSQL_TCP_PORT@} user=@MYSQLD_USER@ +niceness=0 # Use the mysqld-max binary by default if the user doesn't specify a binary if test -x $ledir/mysqld-max @@ -167,7 +169,12 @@ export MYSQL_UNIX_PORT export MYSQL_TCP_PORT -NOHUP_NICENESS="nohup" +if test $niceness -eq 0 +then + NOHUP_NICENESS="nohup" +else + NOHUP_NICENESS="nohup nice -$niceness" +fi # Using nice with no args to get the niceness level is GNU-specific. # This check could be extended for other operating systems (e.g., @@ -198,8 +205,10 @@ then nice --$nice_value_diff echo testing > /dev/null 2>&1 then # nohup increases the priority (bad), and we are permitted - # to lower the priority - NOHUP_NICENESS="nice --$nice_value_diff nohup" + # to lower the priority with respect to the value the user + # might have been given + niceness=`expr $niceness - $nice_value_diff` + NOHUP_NICENESS="nice -$niceness nohup" fi fi else From a79bdf2ee75fa48b34cd4736b49ea91197afaeae Mon Sep 17 00:00:00 2001 From: "monty@narttu.mysql.fi" <> Date: Thu, 12 Jun 2003 16:25:26 +0300 Subject: [PATCH 27/40] Indentation --- mysys/safemalloc.c | 247 ++++++++++++++++++++++----------------------- 1 file changed, 121 insertions(+), 126 deletions(-) diff --git a/mysys/safemalloc.c b/mysys/safemalloc.c index 9cbb178edb4..42cff1e5069 100644 --- a/mysys/safemalloc.c +++ b/mysys/safemalloc.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2000 MySQL AB +/* Copyright (C) 2000-2003 MySQL AB This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -69,7 +69,7 @@ #include "my_static.h" #include "mysys_err.h" -ulonglong safemalloc_mem_limit = ~(ulonglong)0; +ulonglong safemalloc_mem_limit= ~(ulonglong)0; #define pNext tInt._pNext #define pPrev tInt._pPrev @@ -84,7 +84,7 @@ ulonglong safemalloc_mem_limit = ~(ulonglong)0; the linked list of blocks so that _sanity() will not fuss when it is not supposed to */ -static int sf_malloc_tampered = 0; +static int sf_malloc_tampered= 0; #endif @@ -92,11 +92,11 @@ static int sf_malloc_tampered = 0; static int check_ptr(const char *where, byte *ptr, const char *sFile, uint uLine); -static int _checkchunk(struct remember *pRec, const char *sFile, uint uLine); +static int _checkchunk(struct irem *pRec, const char *sFile, uint uLine); /* - Note: both these refer to the NEW'ed data only. They do not include - malloc() roundoff or the extra space required by the remember + Note: We only fill up the allocated block. This do not include + malloc() roundoff or the extra space required by the irem structures. */ @@ -127,90 +127,85 @@ static int _checkchunk(struct remember *pRec, const char *sFile, uint uLine); /* Allocate some memory. */ -gptr _mymalloc (uint uSize, const char *sFile, uint uLine, myf MyFlags) +gptr _mymalloc(uint uSize, const char *sFile, uint uLine, myf MyFlags) { - struct remember *pTmp; - DBUG_ENTER("_mymalloc"); - DBUG_PRINT("enter",("Size: %u",uSize)); + struct remember *pTmp; + DBUG_ENTER("_mymalloc"); + DBUG_PRINT("enter",("Size: %u",uSize)); + if (!sf_malloc_quick) + (void) _sanity (sFile, uLine); - if (!sf_malloc_quick) - (void) _sanity (sFile, uLine); - - if (uSize + lCurMemory > safemalloc_mem_limit) - pTmp = 0; - else + if (uSize + lCurMemory > safemalloc_mem_limit) + pTmp= 0; + else + { + /* Allocate the physical memory */ + pTmp= (struct remember *) malloc (ALIGN_SIZE(sizeof(struct irem)) + + sf_malloc_prehunc + + uSize + /* size requested */ + 4 + /* overrun mark */ + sf_malloc_endhunc + ); + } + /* Check if there isn't anymore memory avaiable */ + if (pTmp == NULL) + { + if (MyFlags & MY_FAE) + error_handler_hook=fatal_error_handler_hook; + if (MyFlags & (MY_FAE+MY_WME)) { - /* Allocate the physical memory */ - pTmp = (struct remember *) malloc ( - ALIGN_SIZE(sizeof(struct irem)) /* remember data */ - + sf_malloc_prehunc - + uSize /* size requested */ - + 4 /* overrun mark */ - + sf_malloc_endhunc - ); - } - /* Check if there isn't anymore memory avaiable */ - if (pTmp == NULL) - { - if (MyFlags & MY_FAE) - error_handler_hook=fatal_error_handler_hook; - if (MyFlags & (MY_FAE+MY_WME)) - { - char buff[SC_MAXWIDTH]; - my_errno=errno; - sprintf(buff,"Out of memory at line %d, '%s'", uLine, sFile); - my_message(EE_OUTOFMEMORY,buff,MYF(ME_BELL+ME_WAITTANG)); - sprintf(buff,"needed %d byte (%ldk), memory in use: %ld bytes (%ldk)", - uSize, (uSize + 1023L) / 1024L, - lMaxMemory, (lMaxMemory + 1023L) / 1024L); - my_message(EE_OUTOFMEMORY,buff,MYF(ME_BELL+ME_WAITTANG)); - } - DBUG_PRINT("error",("Out of memory, in use: %ld at line %d, '%s'", - lMaxMemory,uLine, sFile)); - if (MyFlags & MY_FAE) - exit(1); - DBUG_RETURN ((gptr) NULL); + char buff[SC_MAXWIDTH]; + my_errno=errno; + sprintf(buff,"Out of memory at line %d, '%s'", uLine, sFile); + my_message(EE_OUTOFMEMORY,buff,MYF(ME_BELL+ME_WAITTANG)); + sprintf(buff,"needed %d byte (%ldk), memory in use: %ld bytes (%ldk)", + uSize, (uSize + 1023L) / 1024L, + lMaxMemory, (lMaxMemory + 1023L) / 1024L); + my_message(EE_OUTOFMEMORY,buff,MYF(ME_BELL+ME_WAITTANG)); } + DBUG_PRINT("error",("Out of memory, in use: %ld at line %d, '%s'", + lMaxMemory,uLine, sFile)); + if (MyFlags & MY_FAE) + exit(1); + DBUG_RETURN ((gptr) NULL); + } - /* Fill up the structure */ - *((long*) ((char*) &pTmp -> lSpecialValue+sf_malloc_prehunc)) = MAGICKEY; - pTmp -> aData[uSize + sf_malloc_prehunc+0] = MAGICEND0; - pTmp -> aData[uSize + sf_malloc_prehunc+1] = MAGICEND1; - pTmp -> aData[uSize + sf_malloc_prehunc+2] = MAGICEND2; - pTmp -> aData[uSize + sf_malloc_prehunc+3] = MAGICEND3; - pTmp -> sFileName = (my_string) sFile; - pTmp -> uLineNum = uLine; - pTmp -> uDataSize = uSize; - pTmp -> pPrev = NULL; + /* Fill up the structure */ + *((uint32*) ((char*) &pTmp->lSpecialValue+sf_malloc_prehunc))= MAGICKEY; + pTmp->aData[uSize + sf_malloc_prehunc+0]= MAGICEND0; + pTmp->aData[uSize + sf_malloc_prehunc+1]= MAGICEND1; + pTmp->aData[uSize + sf_malloc_prehunc+2]= MAGICEND2; + pTmp->aData[uSize + sf_malloc_prehunc+3]= MAGICEND3; + pTmp->sFileName= (my_string) sFile; + pTmp->uLineNum= uLine; + pTmp->uDataSize= uSize; + pTmp->pPrev= NULL; - /* Add this remember structure to the linked list */ - pthread_mutex_lock(&THR_LOCK_malloc); - if ((pTmp->pNext=pRememberRoot)) - { - pRememberRoot -> pPrev = pTmp; - } - pRememberRoot = pTmp; + /* Add this remember structure to the linked list */ + pthread_mutex_lock(&THR_LOCK_malloc); + if ((pTmp->pNext= pRememberRoot)) + pRememberRoot->pPrev= pTmp; + pRememberRoot= pTmp; - /* Keep the statistics */ - lCurMemory += uSize; - if (lCurMemory > lMaxMemory) { - lMaxMemory = lCurMemory; - } - cNewCount++; - pthread_mutex_unlock(&THR_LOCK_malloc); + /* Keep the statistics */ + lCurMemory+= uSize; + if (lCurMemory > lMaxMemory) + lMaxMemory= lCurMemory; + cNewCount++; + pthread_mutex_unlock(&THR_LOCK_malloc); - /* Set the memory to the aribtrary wierd value */ - if ((MyFlags & MY_ZEROFILL) || !sf_malloc_quick) - bfill(&pTmp -> aData[sf_malloc_prehunc],uSize, - (char) (MyFlags & MY_ZEROFILL ? 0 : ALLOC_VAL)); - /* Return a pointer to the real data */ - DBUG_PRINT("exit",("ptr: %lx",&(pTmp -> aData[sf_malloc_prehunc]))); - if (sf_min_adress > &(pTmp -> aData[sf_malloc_prehunc])) - sf_min_adress = &(pTmp -> aData[sf_malloc_prehunc]); - if (sf_max_adress < &(pTmp -> aData[sf_malloc_prehunc])) - sf_max_adress = &(pTmp -> aData[sf_malloc_prehunc]); - DBUG_RETURN ((gptr) &(pTmp -> aData[sf_malloc_prehunc])); + /* Set the memory to the aribtrary wierd value */ + if ((MyFlags & MY_ZEROFILL) || !sf_malloc_quick) + bfill(&pTmp->aData[sf_malloc_prehunc],uSize, + (char) (MyFlags & MY_ZEROFILL ? 0 : ALLOC_VAL)); + /* Return a pointer to the real data */ + DBUG_PRINT("exit",("ptr: %lx",&(pTmp->aData[sf_malloc_prehunc]))); + if (sf_min_adress > &(pTmp->aData[sf_malloc_prehunc])) + sf_min_adress= &(pTmp->aData[sf_malloc_prehunc]); + if (sf_max_adress < &(pTmp->aData[sf_malloc_prehunc])) + sf_max_adress= &(pTmp->aData[sf_malloc_prehunc]); + DBUG_RETURN ((gptr) &(pTmp->aData[sf_malloc_prehunc])); } /* @@ -218,8 +213,8 @@ gptr _mymalloc (uint uSize, const char *sFile, uint uLine, myf MyFlags) Free then old memoryblock */ -gptr _myrealloc (register gptr pPtr, register uint uSize, - const char *sFile, uint uLine, myf MyFlags) +gptr _myrealloc(register gptr pPtr, register uint uSize, + const char *sFile, uint uLine, myf MyFlags) { struct remember *pRec; gptr ptr; @@ -234,9 +229,9 @@ gptr _myrealloc (register gptr pPtr, register uint uSize, if (check_ptr("Reallocating",(byte*) pPtr,sFile,uLine)) DBUG_RETURN((gptr) NULL); - pRec = (struct remember *) ((char*) pPtr - ALIGN_SIZE(sizeof(struct irem))- + pRec= (struct remember *) ((char*) pPtr - ALIGN_SIZE(sizeof(struct irem))- sf_malloc_prehunc); - if (*((long*) ((char*) &pRec -> lSpecialValue+sf_malloc_prehunc)) + if (*((uint32*) ((char*) &pRec->lSpecialValue+sf_malloc_prehunc)) != MAGICKEY) { fprintf(stderr, "Error: Reallocating unallocated data at line %d, '%s'\n", @@ -266,7 +261,7 @@ gptr _myrealloc (register gptr pPtr, register uint uSize, /* Deallocate some memory. */ -void _myfree (gptr pPtr, const char *sFile, uint uLine, myf myflags) +void _myfree(gptr pPtr, const char *sFile, uint uLine, myf myflags) { struct remember *pRec; DBUG_ENTER("_myfree"); @@ -280,19 +275,19 @@ void _myfree (gptr pPtr, const char *sFile, uint uLine, myf myflags) DBUG_VOID_RETURN; /* Calculate the address of the remember structure */ - pRec = (struct remember *) ((byte*) pPtr- ALIGN_SIZE(sizeof(struct irem))- + pRec= (struct remember *) ((byte*) pPtr- ALIGN_SIZE(sizeof(struct irem))- sf_malloc_prehunc); /* Check to make sure that we have a real remember structure. Note: this test could fail for four reasons: - (1) The memory was already free'ed - (2) The memory was never new'ed - (3) There was an underrun - (4) A stray pointer hit this location + (1) The memory was already free'ed + (2) The memory was never new'ed + (3) There was an underrun + (4) A stray pointer hit this location */ - if (*((long*) ((char*) &pRec -> lSpecialValue+sf_malloc_prehunc)) + if (*((uint32*) ((char*) &pRec->lSpecialValue+sf_malloc_prehunc)) != MAGICKEY) { fprintf(stderr, "Error: Freeing unallocated data at line %d, '%s'\n", @@ -304,16 +299,15 @@ void _myfree (gptr pPtr, const char *sFile, uint uLine, myf myflags) /* Remove this structure from the linked list */ pthread_mutex_lock(&THR_LOCK_malloc); - if (pRec -> pPrev) { - pRec -> pPrev -> pNext = pRec -> pNext; - } else { - pRememberRoot = pRec -> pNext; - } - if (pRec -> pNext) { - pRec -> pNext -> pPrev = pRec -> pPrev; - } + if (pRec->pPrev) + pRec->pPrev->pNext= pRec->pNext; + else + pRememberRoot= pRec->pNext; + + if (pRec->pNext) + pRec->pNext->pPrev= pRec->pPrev; /* Handle the statistics */ - lCurMemory -= pRec -> uDataSize; + lCurMemory -= pRec->uDataSize; cNewCount--; pthread_mutex_unlock(&THR_LOCK_malloc); @@ -322,7 +316,7 @@ void _myfree (gptr pPtr, const char *sFile, uint uLine, myf myflags) if (!sf_malloc_quick) bfill(&pRec->aData[sf_malloc_prehunc],pRec->uDataSize,(pchar) FREE_VAL); #endif - *((long*) ((char*) &pRec -> lSpecialValue+sf_malloc_prehunc)) = ~MAGICKEY; + *((uint32*) ((char*) &pRec->lSpecialValue+sf_malloc_prehunc))= ~MAGICKEY; /* Actually free the memory */ free ((my_string ) pRec); @@ -372,7 +366,7 @@ static int check_ptr(const char *where, byte *ptr, const char *sFile, free'ed as well as the statistics. */ -void TERMINATE (FILE *file) +void TERMINATE(FILE *file) { struct remember *pPtr; DBUG_ENTER("TERMINATE"); @@ -403,7 +397,8 @@ void TERMINATE (FILE *file) { if (file) { - fprintf(file, "Warning: Memory that was not free'ed (%ld bytes):\n",lCurMemory); + fprintf(file, "Warning: Memory that was not free'ed (%ld bytes):\n", + lCurMemory); (void) fflush(file); } DBUG_PRINT("safe",("Memory that was not free'ed (%ld bytes):",lCurMemory)); @@ -413,17 +408,17 @@ void TERMINATE (FILE *file) { fprintf(file, "\t%6u bytes at 0x%09lx, allocated at line %4u in '%s'", - pPtr -> uDataSize, - (ulong) &(pPtr -> aData[sf_malloc_prehunc]), - pPtr -> uLineNum, pPtr -> sFileName); + pPtr->uDataSize, + (ulong) &(pPtr->aData[sf_malloc_prehunc]), + pPtr->uLineNum, pPtr->sFileName); fprintf(file, "\n"); (void) fflush(file); } DBUG_PRINT("safe", ("%6u bytes at 0x%09lx, allocated at line %4d in '%s'", - pPtr -> uDataSize, &(pPtr -> aData[sf_malloc_prehunc]), - pPtr -> uLineNum, pPtr -> sFileName)); - pPtr = pPtr -> pNext; + pPtr->uDataSize, &(pPtr->aData[sf_malloc_prehunc]), + pPtr->uLineNum, pPtr->sFileName)); + pPtr= pPtr->pNext; } } /* Report the memory usage statistics */ @@ -442,44 +437,44 @@ void TERMINATE (FILE *file) /* Returns 0 if chunk is ok */ -static int _checkchunk (register struct remember *pRec, const char *sFile, - uint uLine) +static int _checkchunk(register struct remember *pRec, const char *sFile, + uint uLine) { reg1 uint uSize; reg2 my_string magicp; reg3 int flag=0; /* Check for a possible underrun */ - if (*((long*) ((char*) &pRec -> lSpecialValue+sf_malloc_prehunc)) + if (*((uint32*) ((char*) &pRec->lSpecialValue+sf_malloc_prehunc)) != MAGICKEY) { fprintf(stderr, "Error: Memory allocated at %s:%d was underrun,", - pRec -> sFileName, pRec -> uLineNum); + pRec->sFileName, pRec->uLineNum); fprintf(stderr, " discovered at %s:%d\n", sFile, uLine); (void) fflush(stderr); DBUG_PRINT("safe",("Underrun at %lx, allocated at %s:%d", - &(pRec -> aData[sf_malloc_prehunc]), - pRec -> sFileName, - pRec -> uLineNum)); + &(pRec->aData[sf_malloc_prehunc]), + pRec->sFileName, + pRec->uLineNum)); flag=1; } /* Check for a possible overrun */ - uSize = pRec -> uDataSize; - magicp = &(pRec -> aData[uSize+sf_malloc_prehunc]); + uSize= pRec->uDataSize; + magicp= &(pRec->aData[uSize+sf_malloc_prehunc]); if (*magicp++ != MAGICEND0 || *magicp++ != MAGICEND1 || *magicp++ != MAGICEND2 || *magicp++ != MAGICEND3) { fprintf(stderr, "Error: Memory allocated at %s:%d was overrun,", - pRec -> sFileName, pRec -> uLineNum); + pRec->sFileName, pRec->uLineNum); fprintf(stderr, " discovered at '%s:%d'\n", sFile, uLine); (void) fflush(stderr); DBUG_PRINT("safe",("Overrun at %lx, allocated at %s:%d", - &(pRec -> aData[sf_malloc_prehunc]), - pRec -> sFileName, - pRec -> uLineNum)); + &(pRec->aData[sf_malloc_prehunc]), + pRec->sFileName, + pRec->uLineNum)); flag=1; } return(flag); @@ -488,7 +483,7 @@ static int _checkchunk (register struct remember *pRec, const char *sFile, /* Returns how many wrong chunks */ -int _sanity (const char *sFile, uint uLine) +int _sanity(const char *sFile, uint uLine) { reg1 struct remember *pTmp; reg2 int flag=0; @@ -500,8 +495,8 @@ int _sanity (const char *sFile, uint uLine) cNewCount=0; #endif count=cNewCount; - for (pTmp = pRememberRoot; pTmp != NULL && count-- ; pTmp = pTmp -> pNext) - flag+=_checkchunk (pTmp, sFile, uLine); + for (pTmp= pRememberRoot; pTmp != NULL && count-- ; pTmp= pTmp->pNext) + flag+= _checkchunk (pTmp, sFile, uLine); pthread_mutex_unlock(&THR_LOCK_malloc); if (count || pTmp) { From 4302bfdb7de91da30ef4a2a7bcaa09954062d606 Mon Sep 17 00:00:00 2001 From: "serg@serg.mylan" <> Date: Thu, 12 Jun 2003 17:05:45 +0200 Subject: [PATCH 28/40] typed moved to a proper place --- include/my_global.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/my_global.h b/include/my_global.h index 1026e8e3940..e5cb75365b4 100644 --- a/include/my_global.h +++ b/include/my_global.h @@ -640,9 +640,6 @@ typedef long my_ptrdiff_t; typedef long long my_ptrdiff_t; #endif -/* typedef used for length of string; Should be unsigned! */ -typedef ulong size_str; - #define MY_ALIGN(A,L) (((A) + (L) - 1) & ~((L) - 1)) #define ALIGN_SIZE(A) MY_ALIGN((A),sizeof(double)) /* Size to make adressable obj. */ @@ -711,6 +708,9 @@ typedef long longlong; #endif #endif +/* typedef used for length of string; Should be unsigned! */ +typedef ulong size_str; + #ifdef USE_RAID /* The following is done with a if to not get problems with pre-processors From 36aae14f1aca76ff8dc84f7ae5c7db93ba6536cc Mon Sep 17 00:00:00 2001 From: "serg@serg.mylan" <> Date: Thu, 12 Jun 2003 17:38:15 +0200 Subject: [PATCH 29/40] HANDLER priv check fixed --- sql/sql_parse.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index 7447ba44e76..9524c832856 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -2476,8 +2476,10 @@ mysql_execute_command(void) res = mysql_ha_close(thd, tables); break; case SQLCOM_HA_READ: - if (check_db_used(thd,tables) || - check_table_access(thd,SELECT_ACL, tables)) + /* there is no need to check for table permissions here, because + if a user has no permissions to read a table, he won't be + able to open it (with SQLCOM_HA_OPEN) in the first place. */ + if (check_db_used(thd,tables)) goto error; res = mysql_ha_read(thd, tables, lex->ha_read_mode, lex->backup_dir, lex->insert_list, lex->ha_rkey_mode, select_lex->where, From 977c66b4e7262d2ddfb721d9cd828c5824b6a024 Mon Sep 17 00:00:00 2001 From: "serg@serg.mylan" <> Date: Thu, 12 Jun 2003 18:46:12 +0200 Subject: [PATCH 30/40] removed a wrong cast that limited ulonglong options to max. ulong value. --- mysys/my_getopt.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mysys/my_getopt.c b/mysys/my_getopt.c index 759c96462f6..e18c5a0b9eb 100644 --- a/mysys/my_getopt.c +++ b/mysys/my_getopt.c @@ -609,9 +609,9 @@ static ulonglong getopt_ull(char *arg, const struct my_option *optp, int *err) ulonglong getopt_ull_limit_value(ulonglong num, const struct my_option *optp) { - if ((ulonglong) num > (ulonglong) (ulong) optp->max_value && + if ((ulonglong) num > (ulonglong) optp->max_value && optp->max_value) /* if max value is not set -> no upper limit */ - num= (ulonglong) (ulong) optp->max_value; + num= (ulonglong) optp->max_value; if (optp->block_size > 1) { num/= (ulonglong) optp->block_size; From ecdb1c768ca856c8eef5389f02a2879c17688b21 Mon Sep 17 00:00:00 2001 From: "monty@narttu.mysql.fi" <> Date: Thu, 12 Jun 2003 22:39:45 +0300 Subject: [PATCH 31/40] Changed safemalloc structure to not have to be 8 byte aligned. (Portability problem) --- include/my_sys.h | 4 +- mysys/default.c | 2 - mysys/my_static.c | 11 +- mysys/my_static.h | 32 +++-- mysys/safemalloc.c | 305 ++++++++++++++++++++++----------------------- sql/mysqld.cc | 2 +- sql/sql_parse.cc | 5 +- 7 files changed, 173 insertions(+), 188 deletions(-) diff --git a/include/my_sys.h b/include/my_sys.h index 603b3bad6bd..7f8b8a80a1c 100644 --- a/include/my_sys.h +++ b/include/my_sys.h @@ -138,7 +138,7 @@ extern int NEAR my_errno; /* Last error in mysys */ #define QUICK_SAFEMALLOC sf_malloc_quick=1 #define NORMAL_SAFEMALLOC sf_malloc_quick=0 extern uint sf_malloc_prehunc,sf_malloc_endhunc,sf_malloc_quick; -extern ulonglong safemalloc_mem_limit; +extern ulonglong sf_malloc_mem_limit; #define CALLER_INFO_PROTO , const char *sFile, uint uLine #define CALLER_INFO , __FILE__, __LINE__ @@ -239,7 +239,7 @@ extern int NEAR my_umask, /* Default creation mask */ NEAR my_safe_to_handle_signal, /* Set when allowed to SIGTSTP */ NEAR my_dont_interrupt; /* call remember_intr when set */ extern my_bool NEAR mysys_uses_curses, my_use_symdir; -extern long lCurMemory,lMaxMemory; /* from safemalloc */ +extern ulong sf_malloc_cur_memory, sf_malloc_max_memory; extern ulong my_default_record_cache_size; extern my_bool NEAR my_disable_locking,NEAR my_disable_async_io, diff --git a/mysys/default.c b/mysys/default.c index 3ff240da3a1..cdacc8bee2b 100644 --- a/mysys/default.c +++ b/mysys/default.c @@ -33,8 +33,6 @@ ** --print-defaults ; Print the modified command line and exit ****************************************************************************/ -#undef SAFEMALLOC /* safe_malloc is not yet initailized */ - #include "mysys_priv.h" #include "m_string.h" #include "m_ctype.h" diff --git a/mysys/my_static.c b/mysys/my_static.c index bbf7582a454..b24ef28b7b1 100644 --- a/mysys/my_static.c +++ b/mysys/my_static.c @@ -69,14 +69,13 @@ uint sf_malloc_prehunc=0, /* If you have problem with core- */ sf_malloc_endhunc=0, /* dump when malloc-message.... */ /* set theese to 64 or 128 */ sf_malloc_quick=0; /* set if no calls to sanity */ -long lCurMemory = 0L; /* Current memory usage */ -long lMaxMemory = 0L; /* Maximum memory usage */ -uint cNewCount = 0; /* Number of times NEW() was called */ +ulong sf_malloc_cur_memory= 0L; /* Current memory usage */ +ulong sf_malloc_max_memory= 0L; /* Maximum memory usage */ +uint sf_malloc_count= 0; /* Number of times NEW() was called */ byte *sf_min_adress= (byte*) ~(unsigned long) 0L, *sf_max_adress= (byte*) 0L; - -/* Root of the linked list of remembers */ -struct remember *pRememberRoot = NULL; +/* Root of the linked list of struct st_irem */ +struct st_irem *sf_malloc_root = NULL; /* from my_alarm */ int volatile my_have_got_alarm=0; /* declare variable to reset */ diff --git a/mysys/my_static.h b/mysys/my_static.h index c1893f4074f..1a33bcf21f3 100644 --- a/mysys/my_static.h +++ b/mysys/my_static.h @@ -33,27 +33,23 @@ struct st_remember { }; /* - The size of the following structure MUST be dividable by 8 to not cause - alignment problems on some cpu's + Structure that stores information of a allocated memory block + The data is at &struct_adr+sizeof(ALIGN_SIZE(sizeof(struct irem))) + The lspecialvalue is at the previous 4 bytes from this, which may not + necessarily be in the struct if the struct size isn't aligned at a 8 byte + boundary. */ -struct irem +struct st_irem { - struct remember *_pNext; /* Linked list of structures */ - struct remember *_pPrev; /* Other link */ - char *_sFileName; /* File in which memory was new'ed */ - uint32 _uLineNum; /* Line number in above file */ - uint32 _uDataSize; /* Size requested */ -#if SIZEOF_CHARP == 8 - long _filler; /* For alignment */ -#endif - long _lSpecialValue; /* Underrun marker value */ + struct st_irem *next; /* Linked list of structures */ + struct st_irem *prev; /* Other link */ + char *filename; /* File in which memory was new'ed */ + uint32 linenum; /* Line number in above file */ + uint32 datasize; /* Size requested */ + uint32 SpecialValue; /* Underrun marker value */ }; -struct remember { - struct irem tInt; - char aData[1]; -}; extern char NEAR curr_dir[FN_REFLEN],NEAR home_dir_buff[FN_REFLEN]; @@ -70,8 +66,8 @@ extern int _my_tempnam_used; #endif extern byte *sf_min_adress,*sf_max_adress; -extern uint cNewCount; -extern struct remember *pRememberRoot; +extern uint sf_malloc_count; +extern struct st_irem *sf_malloc_root; #if defined(THREAD) && !defined(__WIN__) extern sigset_t my_signals; /* signals blocked by mf_brkhant */ diff --git a/mysys/safemalloc.c b/mysys/safemalloc.c index 42cff1e5069..bd77b4821ff 100644 --- a/mysys/safemalloc.c +++ b/mysys/safemalloc.c @@ -69,18 +69,11 @@ #include "my_static.h" #include "mysys_err.h" -ulonglong safemalloc_mem_limit= ~(ulonglong)0; - -#define pNext tInt._pNext -#define pPrev tInt._pPrev -#define sFileName tInt._sFileName -#define uLineNum tInt._uLineNum -#define uDataSize tInt._uDataSize -#define lSpecialValue tInt._lSpecialValue +ulonglong sf_malloc_mem_limit= ~(ulonglong)0; #ifndef PEDANTIC_SAFEMALLOC /* - Set to 1 after TERMINATE() if we had to fiddle with cNewCount and + Set to 1 after TERMINATE() if we had to fiddle with sf_malloc_count and the linked list of blocks so that _sanity() will not fuss when it is not supposed to */ @@ -92,7 +85,7 @@ static int sf_malloc_tampered= 0; static int check_ptr(const char *where, byte *ptr, const char *sFile, uint uLine); -static int _checkchunk(struct irem *pRec, const char *sFile, uint uLine); +static int _checkchunk(struct st_irem *pRec, const char *sFile, uint uLine); /* Note: We only fill up the allocated block. This do not include @@ -127,29 +120,29 @@ static int _checkchunk(struct irem *pRec, const char *sFile, uint uLine); /* Allocate some memory. */ -gptr _mymalloc(uint uSize, const char *sFile, uint uLine, myf MyFlags) +gptr _mymalloc(uint size, const char *filename, uint lineno, myf MyFlags) { - struct remember *pTmp; + struct st_irem *irem; + char *data; DBUG_ENTER("_mymalloc"); - DBUG_PRINT("enter",("Size: %u",uSize)); + DBUG_PRINT("enter",("Size: %u",size)); if (!sf_malloc_quick) - (void) _sanity (sFile, uLine); + (void) _sanity (filename, lineno); - if (uSize + lCurMemory > safemalloc_mem_limit) - pTmp= 0; + if (size + sf_malloc_cur_memory > sf_malloc_mem_limit) + irem= 0; else { /* Allocate the physical memory */ - pTmp= (struct remember *) malloc (ALIGN_SIZE(sizeof(struct irem)) + - sf_malloc_prehunc + - uSize + /* size requested */ - 4 + /* overrun mark */ - sf_malloc_endhunc - ); + irem= (struct st_irem *) malloc (ALIGN_SIZE(sizeof(struct st_irem)) + + sf_malloc_prehunc + + size + /* size requested */ + 4 + /* overrun mark */ + sf_malloc_endhunc); } /* Check if there isn't anymore memory avaiable */ - if (pTmp == NULL) + if (!irem) { if (MyFlags & MY_FAE) error_handler_hook=fatal_error_handler_hook; @@ -157,126 +150,127 @@ gptr _mymalloc(uint uSize, const char *sFile, uint uLine, myf MyFlags) { char buff[SC_MAXWIDTH]; my_errno=errno; - sprintf(buff,"Out of memory at line %d, '%s'", uLine, sFile); + sprintf(buff,"Out of memory at line %d, '%s'", lineno, filename); my_message(EE_OUTOFMEMORY,buff,MYF(ME_BELL+ME_WAITTANG)); sprintf(buff,"needed %d byte (%ldk), memory in use: %ld bytes (%ldk)", - uSize, (uSize + 1023L) / 1024L, - lMaxMemory, (lMaxMemory + 1023L) / 1024L); + size, (size + 1023L) / 1024L, + sf_malloc_max_memory, (sf_malloc_max_memory + 1023L) / 1024L); my_message(EE_OUTOFMEMORY,buff,MYF(ME_BELL+ME_WAITTANG)); } DBUG_PRINT("error",("Out of memory, in use: %ld at line %d, '%s'", - lMaxMemory,uLine, sFile)); + sf_malloc_max_memory,lineno, filename)); if (MyFlags & MY_FAE) exit(1); - DBUG_RETURN ((gptr) NULL); + DBUG_RETURN ((gptr) 0); } /* Fill up the structure */ - *((uint32*) ((char*) &pTmp->lSpecialValue+sf_malloc_prehunc))= MAGICKEY; - pTmp->aData[uSize + sf_malloc_prehunc+0]= MAGICEND0; - pTmp->aData[uSize + sf_malloc_prehunc+1]= MAGICEND1; - pTmp->aData[uSize + sf_malloc_prehunc+2]= MAGICEND2; - pTmp->aData[uSize + sf_malloc_prehunc+3]= MAGICEND3; - pTmp->sFileName= (my_string) sFile; - pTmp->uLineNum= uLine; - pTmp->uDataSize= uSize; - pTmp->pPrev= NULL; + data= (((char*) irem) + ALIGN_SIZE(sizeof(struct st_irem)) + + sf_malloc_prehunc); + *((uint32*) (data-sizeof(uint32)))= MAGICKEY; + data[size + 0]= MAGICEND0; + data[size + 1]= MAGICEND1; + data[size + 2]= MAGICEND2; + data[size + 3]= MAGICEND3; + irem->filename= (my_string) filename; + irem->linenum= lineno; + irem->datasize= size; + irem->prev= NULL; /* Add this remember structure to the linked list */ pthread_mutex_lock(&THR_LOCK_malloc); - if ((pTmp->pNext= pRememberRoot)) - pRememberRoot->pPrev= pTmp; - pRememberRoot= pTmp; + if ((irem->next= sf_malloc_root)) + sf_malloc_root->prev= irem; + sf_malloc_root= irem; /* Keep the statistics */ - lCurMemory+= uSize; - if (lCurMemory > lMaxMemory) - lMaxMemory= lCurMemory; - cNewCount++; + sf_malloc_cur_memory+= size; + if (sf_malloc_cur_memory > sf_malloc_max_memory) + sf_malloc_max_memory= sf_malloc_cur_memory; + sf_malloc_count++; pthread_mutex_unlock(&THR_LOCK_malloc); /* Set the memory to the aribtrary wierd value */ if ((MyFlags & MY_ZEROFILL) || !sf_malloc_quick) - bfill(&pTmp->aData[sf_malloc_prehunc],uSize, - (char) (MyFlags & MY_ZEROFILL ? 0 : ALLOC_VAL)); + bfill(data, size, (char) (MyFlags & MY_ZEROFILL ? 0 : ALLOC_VAL)); /* Return a pointer to the real data */ - DBUG_PRINT("exit",("ptr: %lx",&(pTmp->aData[sf_malloc_prehunc]))); - if (sf_min_adress > &(pTmp->aData[sf_malloc_prehunc])) - sf_min_adress= &(pTmp->aData[sf_malloc_prehunc]); - if (sf_max_adress < &(pTmp->aData[sf_malloc_prehunc])) - sf_max_adress= &(pTmp->aData[sf_malloc_prehunc]); - DBUG_RETURN ((gptr) &(pTmp->aData[sf_malloc_prehunc])); + DBUG_PRINT("exit",("ptr: %lx", data)); + if (sf_min_adress > data) + sf_min_adress= data; + if (sf_max_adress < data) + sf_max_adress= data; + DBUG_RETURN ((gptr) data); } + /* Allocate some new memory and move old memoryblock there. Free then old memoryblock */ -gptr _myrealloc(register gptr pPtr, register uint uSize, - const char *sFile, uint uLine, myf MyFlags) +gptr _myrealloc(register gptr ptr, register uint size, + const char *filename, uint lineno, myf MyFlags) { - struct remember *pRec; - gptr ptr; + struct st_irem *irem; + char *data; DBUG_ENTER("_myrealloc"); - if (!pPtr && (MyFlags & MY_ALLOW_ZERO_PTR)) - DBUG_RETURN(_mymalloc(uSize,sFile,uLine,MyFlags)); + if (!ptr && (MyFlags & MY_ALLOW_ZERO_PTR)) + DBUG_RETURN(_mymalloc(size, filename, lineno, MyFlags)); if (!sf_malloc_quick) - (void) _sanity (sFile, uLine); + (void) _sanity (filename, lineno); - if (check_ptr("Reallocating",(byte*) pPtr,sFile,uLine)) + if (check_ptr("Reallocating", (byte*) ptr, filename, lineno)) DBUG_RETURN((gptr) NULL); - pRec= (struct remember *) ((char*) pPtr - ALIGN_SIZE(sizeof(struct irem))- - sf_malloc_prehunc); - if (*((uint32*) ((char*) &pRec->lSpecialValue+sf_malloc_prehunc)) - != MAGICKEY) + irem= (struct st_irem *) (((char*) ptr) - ALIGN_SIZE(sizeof(struct st_irem))- + sf_malloc_prehunc); + if (*((uint32*) (((char*) ptr)- sizeof(uint32))) != MAGICKEY) { fprintf(stderr, "Error: Reallocating unallocated data at line %d, '%s'\n", - uLine, sFile); + lineno, filename); DBUG_PRINT("safe",("Reallocating unallocated data at line %d, '%s'", - uLine, sFile)); + lineno, filename)); (void) fflush(stderr); DBUG_RETURN((gptr) NULL); } - if ((ptr=_mymalloc(uSize,sFile,uLine,MyFlags))) /* Allocate new area */ + if ((data= _mymalloc(size,filename,lineno,MyFlags))) /* Allocate new area */ { - uSize=min(uSize,pRec-> uDataSize); /* Move as much as possibly */ - memcpy((byte*) ptr,pPtr,(size_t) uSize); /* Copy old data */ - _myfree(pPtr,sFile,uLine,0); /* Free not needed area */ + size=min(size, irem->datasize); /* Move as much as possibly */ + memcpy((byte*) data, ptr, (size_t) size); /* Copy old data */ + _myfree(ptr, filename, lineno, 0); /* Free not needed area */ } else { if (MyFlags & MY_HOLD_ON_ERROR) - DBUG_RETURN(pPtr); + DBUG_RETURN(ptr); if (MyFlags & MY_FREE_ON_ERROR) - _myfree(pPtr,sFile,uLine,0); + _myfree(ptr, filename, lineno, 0); } - DBUG_RETURN(ptr); + DBUG_RETURN(data); } /* _myrealloc */ /* Deallocate some memory. */ -void _myfree(gptr pPtr, const char *sFile, uint uLine, myf myflags) +void _myfree(gptr ptr, const char *filename, uint lineno, myf myflags) { - struct remember *pRec; + struct st_irem *irem; DBUG_ENTER("_myfree"); - DBUG_PRINT("enter",("ptr: %lx",pPtr)); + DBUG_PRINT("enter",("ptr: %lx", ptr)); if (!sf_malloc_quick) - (void) _sanity (sFile, uLine); + (void) _sanity (filename, lineno); - if ((!pPtr && (myflags & MY_ALLOW_ZERO_PTR)) || - check_ptr("Freeing",(byte*) pPtr,sFile,uLine)) + if ((!ptr && (myflags & MY_ALLOW_ZERO_PTR)) || + check_ptr("Freeing",(byte*) ptr,filename,lineno)) DBUG_VOID_RETURN; /* Calculate the address of the remember structure */ - pRec= (struct remember *) ((byte*) pPtr- ALIGN_SIZE(sizeof(struct irem))- - sf_malloc_prehunc); + irem= (struct st_irem *) ((char*) ptr- ALIGN_SIZE(sizeof(struct st_irem))- + sf_malloc_prehunc); /* Check to make sure that we have a real remember structure. @@ -287,52 +281,50 @@ void _myfree(gptr pPtr, const char *sFile, uint uLine, myf myflags) (4) A stray pointer hit this location */ - if (*((uint32*) ((char*) &pRec->lSpecialValue+sf_malloc_prehunc)) - != MAGICKEY) + if (*((uint32*) ((char*) ptr- sizeof(uint32))) != MAGICKEY) { fprintf(stderr, "Error: Freeing unallocated data at line %d, '%s'\n", - uLine, sFile); - DBUG_PRINT("safe",("Unallocated data at line %d, '%s'",uLine,sFile)); + lineno, filename); + DBUG_PRINT("safe",("Unallocated data at line %d, '%s'",lineno,filename)); (void) fflush(stderr); DBUG_VOID_RETURN; } /* Remove this structure from the linked list */ pthread_mutex_lock(&THR_LOCK_malloc); - if (pRec->pPrev) - pRec->pPrev->pNext= pRec->pNext; + if (irem->prev) + irem->prev->next= irem->next; else - pRememberRoot= pRec->pNext; + sf_malloc_root= irem->next; - if (pRec->pNext) - pRec->pNext->pPrev= pRec->pPrev; + if (irem->next) + irem->next->prev= irem->prev; /* Handle the statistics */ - lCurMemory -= pRec->uDataSize; - cNewCount--; + sf_malloc_cur_memory-= irem->datasize; + sf_malloc_count--; pthread_mutex_unlock(&THR_LOCK_malloc); #ifndef HAVE_purify /* Mark this data as free'ed */ if (!sf_malloc_quick) - bfill(&pRec->aData[sf_malloc_prehunc],pRec->uDataSize,(pchar) FREE_VAL); + bfill(ptr, irem->datasize, (pchar) FREE_VAL); #endif - *((uint32*) ((char*) &pRec->lSpecialValue+sf_malloc_prehunc))= ~MAGICKEY; - + *((uint32*) ((char*) ptr- sizeof(uint32)))= ~MAGICKEY; /* Actually free the memory */ - free ((my_string ) pRec); + free((char*) irem); DBUG_VOID_RETURN; } /* Check if we have a wrong pointer */ -static int check_ptr(const char *where, byte *ptr, const char *sFile, - uint uLine) +static int check_ptr(const char *where, byte *ptr, const char *filename, + uint lineno) { if (!ptr) { fprintf(stderr, "Error: %s NULL pointer at line %d, '%s'\n", - where,uLine, sFile); - DBUG_PRINT("safe",("Null pointer at line %d '%s'", uLine, sFile)); + where,lineno, filename); + DBUG_PRINT("safe",("Null pointer at line %d '%s'", lineno, filename)); (void) fflush(stderr); return 1; } @@ -340,9 +332,9 @@ static int check_ptr(const char *where, byte *ptr, const char *sFile, if ((long) ptr & (ALIGN_SIZE(1)-1)) { fprintf(stderr, "Error: %s wrong aligned pointer at line %d, '%s'\n", - where,uLine, sFile); + where,lineno, filename); DBUG_PRINT("safe",("Wrong aligned pointer at line %d, '%s'", - uLine,sFile)); + lineno,filename)); (void) fflush(stderr); return 1; } @@ -350,9 +342,9 @@ static int check_ptr(const char *where, byte *ptr, const char *sFile, if (ptr < sf_min_adress || ptr > sf_max_adress) { fprintf(stderr, "Error: %s pointer out of range at line %d, '%s'\n", - where,uLine, sFile); + where,lineno, filename); DBUG_PRINT("safe",("Pointer out of range at line %d '%s'", - uLine,sFile)); + lineno,filename)); (void) fflush(stderr); return 1; } @@ -368,7 +360,7 @@ static int check_ptr(const char *where, byte *ptr, const char *sFile, void TERMINATE(FILE *file) { - struct remember *pPtr; + struct st_irem *irem; DBUG_ENTER("TERMINATE"); pthread_mutex_lock(&THR_LOCK_malloc); @@ -378,14 +370,15 @@ void TERMINATE(FILE *file) NEWs than FREEs. <0, etc. */ - if (cNewCount) + if (sf_malloc_count) { if (file) { - fprintf(file, "Warning: Not freed memory segments: %d\n", cNewCount); + fprintf(file, "Warning: Not freed memory segments: %d\n", + sf_malloc_count); (void) fflush(file); } - DBUG_PRINT("safe",("cNewCount: %d",cNewCount)); + DBUG_PRINT("safe",("sf_malloc_count: %d", sf_malloc_count)); } /* @@ -393,43 +386,44 @@ void TERMINATE(FILE *file) but not free'ed with FREE. */ - if ((pPtr=pRememberRoot)) + if ((irem= sf_malloc_root)) { if (file) { fprintf(file, "Warning: Memory that was not free'ed (%ld bytes):\n", - lCurMemory); + sf_malloc_cur_memory); (void) fflush(file); } - DBUG_PRINT("safe",("Memory that was not free'ed (%ld bytes):",lCurMemory)); - while (pPtr) + DBUG_PRINT("safe",("Memory that was not free'ed (%ld bytes):", + sf_malloc_cur_memory)); + while (irem) { + char *data= (((char*) irem) + ALIGN_SIZE(sizeof(struct st_irem)) + + sf_malloc_prehunc); if (file) { fprintf(file, "\t%6u bytes at 0x%09lx, allocated at line %4u in '%s'", - pPtr->uDataSize, - (ulong) &(pPtr->aData[sf_malloc_prehunc]), - pPtr->uLineNum, pPtr->sFileName); + irem->datasize, (long) data, irem->linenum, irem->filename); fprintf(file, "\n"); (void) fflush(file); } DBUG_PRINT("safe", ("%6u bytes at 0x%09lx, allocated at line %4d in '%s'", - pPtr->uDataSize, &(pPtr->aData[sf_malloc_prehunc]), - pPtr->uLineNum, pPtr->sFileName)); - pPtr= pPtr->pNext; + irem->datasize, data, irem->linenum, irem->filename)); + irem= irem->next; } } /* Report the memory usage statistics */ if (file) { fprintf(file, "Maximum memory usage: %ld bytes (%ldk)\n", - lMaxMemory, (lMaxMemory + 1023L) / 1024L); + sf_malloc_max_memory, (sf_malloc_max_memory + 1023L) / 1024L); (void) fflush(file); } DBUG_PRINT("safe",("Maximum memory usage: %ld bytes (%ldk)", - lMaxMemory, (lMaxMemory + 1023L) / 1024L)); + sf_malloc_max_memory, (sf_malloc_max_memory + 1023L) / + 1024L)); pthread_mutex_unlock(&THR_LOCK_malloc); DBUG_VOID_RETURN; } @@ -437,44 +431,41 @@ void TERMINATE(FILE *file) /* Returns 0 if chunk is ok */ -static int _checkchunk(register struct remember *pRec, const char *sFile, - uint uLine) +static int _checkchunk(register struct st_irem *irem, const char *filename, + uint lineno) { - reg1 uint uSize; - reg2 my_string magicp; - reg3 int flag=0; + int flag=0; + char *magicp, *data; + data= (((char*) irem) + ALIGN_SIZE(sizeof(struct st_irem)) + + sf_malloc_prehunc); /* Check for a possible underrun */ - if (*((uint32*) ((char*) &pRec->lSpecialValue+sf_malloc_prehunc)) - != MAGICKEY) + if (*((uint32*) (data- sizeof(uint32))) != MAGICKEY) { fprintf(stderr, "Error: Memory allocated at %s:%d was underrun,", - pRec->sFileName, pRec->uLineNum); - fprintf(stderr, " discovered at %s:%d\n", sFile, uLine); + irem->filename, irem->linenum); + fprintf(stderr, " discovered at %s:%d\n", filename, lineno); (void) fflush(stderr); DBUG_PRINT("safe",("Underrun at %lx, allocated at %s:%d", - &(pRec->aData[sf_malloc_prehunc]), - pRec->sFileName, - pRec->uLineNum)); + data, irem->filename, irem->linenum)); flag=1; } /* Check for a possible overrun */ - uSize= pRec->uDataSize; - magicp= &(pRec->aData[uSize+sf_malloc_prehunc]); + magicp= data + irem->datasize; if (*magicp++ != MAGICEND0 || *magicp++ != MAGICEND1 || *magicp++ != MAGICEND2 || *magicp++ != MAGICEND3) { fprintf(stderr, "Error: Memory allocated at %s:%d was overrun,", - pRec->sFileName, pRec->uLineNum); - fprintf(stderr, " discovered at '%s:%d'\n", sFile, uLine); + irem->filename, irem->linenum); + fprintf(stderr, " discovered at '%s:%d'\n", filename, lineno); (void) fflush(stderr); DBUG_PRINT("safe",("Overrun at %lx, allocated at %s:%d", - &(pRec->aData[sf_malloc_prehunc]), - pRec->sFileName, - pRec->uLineNum)); + data, + irem->filename, + irem->linenum)); flag=1; } return(flag); @@ -483,28 +474,28 @@ static int _checkchunk(register struct remember *pRec, const char *sFile, /* Returns how many wrong chunks */ -int _sanity(const char *sFile, uint uLine) +int _sanity(const char *filename, uint lineno) { - reg1 struct remember *pTmp; + reg1 struct st_irem *irem; reg2 int flag=0; uint count=0; pthread_mutex_lock(&THR_LOCK_malloc); #ifndef PEDANTIC_SAFEMALLOC - if (sf_malloc_tampered && cNewCount < 0) - cNewCount=0; + if (sf_malloc_tampered && sf_malloc_count < 0) + sf_malloc_count=0; #endif - count=cNewCount; - for (pTmp= pRememberRoot; pTmp != NULL && count-- ; pTmp= pTmp->pNext) - flag+= _checkchunk (pTmp, sFile, uLine); + count=sf_malloc_count; + for (irem= sf_malloc_root; irem != NULL && count-- ; irem= irem->next) + flag+= _checkchunk (irem, filename, lineno); pthread_mutex_unlock(&THR_LOCK_malloc); - if (count || pTmp) + if (count || irem) { const char *format="Error: Safemalloc link list destroyed, discovered at '%s:%d'"; - fprintf(stderr, format, sFile, uLine); fputc('\n',stderr); - fprintf(stderr, "root=%p,count=%d,pTmp=%p\n", pRememberRoot,count,pTmp); + fprintf(stderr, format, filename, lineno); fputc('\n',stderr); + fprintf(stderr, "root=%p,count=%d,irem=%p\n", sf_malloc_root,count,irem); (void) fflush(stderr); - DBUG_PRINT("safe",(format, sFile, uLine)); + DBUG_PRINT("safe",(format, filename, lineno)); flag=1; } return flag; @@ -513,33 +504,33 @@ int _sanity(const char *sFile, uint uLine) /* malloc and copy */ -gptr _my_memdup(const byte *from, uint length, const char *sFile, uint uLine, - myf MyFlags) +gptr _my_memdup(const byte *from, uint length, const char *filename, + uint lineno, myf MyFlags) { gptr ptr; - if ((ptr=_mymalloc(length,sFile,uLine,MyFlags)) != 0) + if ((ptr=_mymalloc(length,filename,lineno,MyFlags)) != 0) memcpy((byte*) ptr, (byte*) from,(size_t) length); return(ptr); } /*_my_memdup */ -char *_my_strdup(const char *from, const char *sFile, uint uLine, +char *_my_strdup(const char *from, const char *filename, uint lineno, myf MyFlags) { gptr ptr; uint length=(uint) strlen(from)+1; - if ((ptr=_mymalloc(length,sFile,uLine,MyFlags)) != 0) + if ((ptr=_mymalloc(length,filename,lineno,MyFlags)) != 0) memcpy((byte*) ptr, (byte*) from,(size_t) length); return((char*) ptr); } /* _my_strdup */ char *_my_strdup_with_length(const byte *from, uint length, - const char *sFile, uint uLine, + const char *filename, uint lineno, myf MyFlags) { gptr ptr; - if ((ptr=_mymalloc(length+1,sFile,uLine,MyFlags)) != 0) + if ((ptr=_mymalloc(length+1,filename,lineno,MyFlags)) != 0) { memcpy((byte*) ptr, (byte*) from,(size_t) length); ptr[length]=0; diff --git a/sql/mysqld.cc b/sql/mysqld.cc index ca98ab96710..0f3500248c0 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -4292,7 +4292,7 @@ get_one_option(int optid, const struct my_option *opt __attribute__((unused)), break; case OPT_SAFEMALLOC_MEM_LIMIT: #if !defined(DBUG_OFF) && defined(SAFEMALLOC) - safemalloc_mem_limit = atoi(argument); + sf_malloc_mem_limit = atoi(argument); #endif break; #ifdef EMBEDDED_LIBRARY diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index 7447ba44e76..3e72bf1d919 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -1208,9 +1208,10 @@ bool dispatch_command(enum enum_server_command command, THD *thd, opened_tables,refresh_version, cached_tables(), uptime ? (float)thd->query_id/(float)uptime : 0); #ifdef SAFEMALLOC - if (lCurMemory) // Using SAFEMALLOC + if (sf_malloc_cur_memory) // Using SAFEMALLOC sprintf(strend(buff), " Memory in use: %ldK Max memory used: %ldK", - (lCurMemory+1023L)/1024L,(lMaxMemory+1023L)/1024L); + (sf_malloc_cur_memory+1023L)/1024L, + (sf_malloc_max_memory+1023L)/1024L); #endif VOID(my_net_write(net, buff,(uint) strlen(buff))); VOID(net_flush(net)); From 8a52c2d20bdb23ef154aa5c6a5980a2d763c9c4e Mon Sep 17 00:00:00 2001 From: "jani@rhols221.adsl.netsonic.fi" <> Date: Sat, 14 Jun 2003 12:29:42 +0300 Subject: [PATCH 32/40] Added option --skip-kill-mysqld to mysqld_safe. This can be useful, if one is running many mysqlds through mysqld_multi, for example. Without this option, on Linux one mysqld_safe process may kill other mysqlds as well, if started using the same binary and path. --- scripts/mysqld_safe.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/mysqld_safe.sh b/scripts/mysqld_safe.sh index 626e04b1579..fcd8e26c901 100644 --- a/scripts/mysqld_safe.sh +++ b/scripts/mysqld_safe.sh @@ -10,6 +10,8 @@ # mysql.server works by first doing a cd to the base directory and from there # executing mysqld_safe +KILL_MYSQLD=1; + trap '' 1 2 3 15 # we shouldn't let anyone kill us umask 007 @@ -34,6 +36,9 @@ parse_arguments() { for arg do case "$arg" in + --skip-kill-mysqld*) + KILL_MYSQLD=0; + ;; # these get passed explicitly to mysqld --basedir=*) MY_BASEDIR_VERSION=`echo "$arg" | sed -e "s;--basedir=;;"` ;; --datadir=*) DATADIR=`echo "$arg" | sed -e "s;--datadir=;;"` ;; @@ -83,6 +88,7 @@ parse_arguments() { done } + MY_PWD=`pwd` # Check if we are starting this relative (for the binary release) if test -d $MY_PWD/data/mysql -a -f ./share/mysql/english/errmsg.sys -a \ @@ -298,7 +304,7 @@ do break fi - if @IS_LINUX@ + if test @IS_LINUX@ -a $KILL_MYSQLD -eq 1 then # Test if one process was hanging. # This is only a fix for Linux (running as base 3 mysqld processes) From 28d3c3d76f7b573cb70838411a5a361907ae2953 Mon Sep 17 00:00:00 2001 From: "guilhem@mysql.com" <> Date: Sat, 14 Jun 2003 16:40:00 +0200 Subject: [PATCH 33/40] - Fix for bug 651: now a dying SQL slave threads wakes up any waiting MASTER_POS_WAIT(). Could not add a testcase for this: if the test goes into a MASTER_POS_WAIT, it waits until this terminates (even doing "connection other_con" to launch "stop slave" is blocked). - In MASTER_POS_WAIT() don't test if the I/O slave is running, but if the SQL thread is running. - Some DBUG info for this bugfix. --- sql/slave.cc | 36 ++++++++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/sql/slave.cc b/sql/slave.cc index d2bc5b2f758..74005c65672 100644 --- a/sql/slave.cc +++ b/sql/slave.cc @@ -289,6 +289,9 @@ err: pthread_cond_broadcast(&rli->data_cond); if (need_data_lock) pthread_mutex_unlock(&rli->data_lock); + + /* Isn't this strange: if !need_data_lock, we broadcast with no lock ?? */ + pthread_mutex_unlock(log_lock); DBUG_RETURN ((*errmsg) ? 1 : 0); } @@ -1778,6 +1781,13 @@ int st_relay_log_info::wait_for_pos(THD* thd, String* log_name, the master info. To catch this, these commands modify abort_pos_wait ; we just monitor abort_pos_wait and see if it has changed. + Why do we have this mechanism instead of simply monitoring slave_running in + the loop (we do this too), as CHANGE MASTER/RESET SLAVE require that the + SQL thread be stopped? This is in case + STOP SLAVE;CHANGE MASTER/RESET SLAVE; START SLAVE; + happens very quickly between the moment pthread_cond_wait() wakes up and + the while() is evaluated: in that case slave_running is again 1 when the + while() is evaluated. */ init_abort_pos_wait= abort_pos_wait; @@ -1814,7 +1824,12 @@ int st_relay_log_info::wait_for_pos(THD* thd, String* log_name, //"compare and wait" main loop while (!thd->killed && init_abort_pos_wait == abort_pos_wait && - mi->slave_running) + /* + formerly we tested mi->slave_running, but what we care about is + rli->slave_running (because this concerns the SQL thread, while + mi->slave_running concerns the I/O thread). + */ + slave_running) { bool pos_reached; int cmp_result= 0; @@ -1852,6 +1867,10 @@ int st_relay_log_info::wait_for_pos(THD* thd, String* log_name, DBUG_PRINT("info",("Waiting for master update")); const char* msg = thd->enter_cond(&data_cond, &data_lock, "Waiting for master update"); + /* + We are going to pthread_cond_(timed)wait(); if the SQL thread stops it + will wake us up. + */ if (timeout > 0) { /* @@ -1869,6 +1888,7 @@ int st_relay_log_info::wait_for_pos(THD* thd, String* log_name, } else pthread_cond_wait(&data_cond, &data_lock); + DBUG_PRINT("info",("Got signal of master update")); thd->exit_cond(msg); if (error == ETIMEDOUT || error == ETIME) { @@ -1877,6 +1897,7 @@ int st_relay_log_info::wait_for_pos(THD* thd, String* log_name, } error=0; event_count++; + DBUG_PRINT("info",("Testing if killed or SQL thread not running")); } err: @@ -1885,11 +1906,11 @@ err: improper_arguments: %d timed_out: %d", (int) thd->killed, (int) (init_abort_pos_wait != abort_pos_wait), - (int) mi->slave_running, + (int) slave_running, (int) (error == -2), (int) (error == -1))); if (thd->killed || init_abort_pos_wait != abort_pos_wait || - !mi->slave_running) + !slave_running) { error= -2; } @@ -2596,8 +2617,15 @@ the slave SQL thread with \"SLAVE START\". We stopped at log \ VOID(pthread_mutex_unlock(&LOCK_thread_count)); thd->proc_info = "Waiting for slave mutex on exit"; pthread_mutex_lock(&rli->run_lock); + /* We need data_lock, at least to wake up any waiting master_pos_wait() */ + pthread_mutex_lock(&rli->data_lock); DBUG_ASSERT(rli->slave_running == 1); // tracking buffer overrun - rli->slave_running = 0; + /* When master_pos_wait() wakes up it will check this and terminate */ + rli->slave_running= 0; + /* Wake up master_pos_wait() */ + pthread_mutex_unlock(&rli->data_lock); + DBUG_PRINT("info",("Signaling possibly waiting master_pos_wait() functions")); + pthread_cond_broadcast(&rli->data_cond); rli->save_temporary_tables = thd->temporary_tables; /* From 4da7f485b75867dbc2ed02054459d1e8050144f3 Mon Sep 17 00:00:00 2001 From: "heikki@hundin.mysql.fi" <> Date: Sun, 15 Jun 2003 01:04:28 +0300 Subject: [PATCH 34/40] Many files: Merge InnoDB-4.0.14: SAVEPOINT now implemented; InnoDB now accepts also column prefix keys; crashing bug in ON UPDATE CASCADE fixed; page checksum formula fixed --- include/my_base.h | 1 + innobase/btr/btr0cur.c | 66 ++--- innobase/btr/btr0pcur.c | 2 + innobase/buf/buf0buf.c | 112 ++++++--- innobase/buf/buf0flu.c | 26 +- innobase/data/data0data.c | 30 ++- innobase/data/data0type.c | 2 - innobase/dict/dict0boot.c | 16 +- innobase/dict/dict0crea.c | 28 ++- innobase/dict/dict0dict.c | 162 +++++++----- innobase/dict/dict0load.c | 28 ++- innobase/dict/dict0mem.c | 13 +- innobase/fil/fil0fil.c | 6 +- innobase/fsp/fsp0fsp.c | 4 +- innobase/ha/ha0ha.c | 8 +- innobase/ibuf/ibuf0ibuf.c | 8 +- innobase/include/btr0cur.h | 8 +- innobase/include/buf0buf.h | 17 +- innobase/include/data0data.h | 2 - innobase/include/data0type.h | 66 +++-- innobase/include/data0type.ic | 35 ++- innobase/include/db0err.h | 4 +- innobase/include/dict0dict.h | 13 + innobase/include/dict0mem.h | 25 +- innobase/include/fil0fil.h | 7 +- innobase/include/lock0lock.h | 12 + innobase/include/os0file.h | 23 ++ innobase/include/page0page.h | 9 + innobase/include/rem0cmp.h | 16 ++ innobase/include/row0mysql.ic | 4 +- innobase/include/row0row.h | 7 +- innobase/include/row0sel.h | 9 +- innobase/include/row0upd.h | 30 +-- innobase/include/srv0srv.h | 1 + innobase/include/trx0roll.h | 64 +++++ innobase/include/trx0sys.ic | 10 + innobase/include/trx0trx.h | 7 +- innobase/include/trx0types.h | 1 + innobase/include/ut0dbg.h | 31 +++ innobase/include/ut0mem.h | 2 +- innobase/lock/lock0lock.c | 86 ++++++- innobase/log/log0log.c | 45 +++- innobase/log/log0recv.c | 14 +- innobase/mem/mem0pool.c | 28 ++- innobase/os/os0file.c | 267 ++++++++++++++++++-- innobase/os/os0thread.c | 4 +- innobase/page/page0cur.c | 19 +- innobase/page/page0page.c | 54 +++- innobase/pars/pars0opt.c | 7 +- innobase/pars/pars0pars.c | 8 +- innobase/rem/rem0cmp.c | 117 ++++++--- innobase/row/row0ins.c | 134 ++++++++-- innobase/row/row0mysql.c | 60 ++--- innobase/row/row0row.c | 92 +++---- innobase/row/row0sel.c | 214 +++++++++++----- innobase/row/row0umod.c | 5 +- innobase/row/row0upd.c | 127 +++++----- innobase/row/row0vers.c | 12 +- innobase/srv/srv0srv.c | 16 +- innobase/srv/srv0start.c | 8 +- innobase/trx/trx0rec.c | 24 +- innobase/trx/trx0roll.c | 224 ++++++++++++++--- innobase/trx/trx0sys.c | 2 +- innobase/trx/trx0trx.c | 5 + innobase/ut/ut0mem.c | 14 +- innobase/ut/ut0ut.c | 2 + mysql-test/r/innodb.result | 10 - mysql-test/t/innodb.test | 9 - sql/ha_innodb.cc | 456 +++++++++++++++++++++++++++------- sql/ha_innodb.h | 19 +- sql/handler.cc | 65 ++++- sql/handler.h | 2 + sql/sql_lex.h | 5 +- sql/sql_parse.cc | 16 +- sql/sql_yacc.yy | 3 +- 75 files changed, 2300 insertions(+), 788 deletions(-) diff --git a/include/my_base.h b/include/my_base.h index cd04ab971db..91a248cd401 100644 --- a/include/my_base.h +++ b/include/my_base.h @@ -255,6 +255,7 @@ enum ha_base_keytype { #define HA_ERR_CANNOT_ADD_FOREIGN 150 /* Cannot add a foreign key constr. */ #define HA_ERR_NO_REFERENCED_ROW 151 /* Cannot add a child row */ #define HA_ERR_ROW_IS_REFERENCED 152 /* Cannot delete a parent row */ +#define HA_ERR_NO_SAVEPOINT 153 /* No savepoint with that name */ /* Other constants */ diff --git a/innobase/btr/btr0cur.c b/innobase/btr/btr0cur.c index e61dcf4ecee..8402993e971 100644 --- a/innobase/btr/btr0cur.c +++ b/innobase/btr/btr0cur.c @@ -1364,7 +1364,8 @@ btr_cur_update_sec_rec_in_place( } /***************************************************************** -Updates a record when the update causes no size changes in its fields. */ +Updates a record when the update causes no size changes in its fields. +We assume here that the ordering fields of the record do not change. */ ulint btr_cur_update_in_place( @@ -1455,7 +1456,8 @@ btr_cur_update_in_place( Tries to update a record on a page in an index tree. It is assumed that mtr holds an x-latch on the page. The operation does not succeed if there is too little space on the page or if the update would result in too empty a page, -so that tree compression is recommended. */ +so that tree compression is recommended. We assume here that the ordering +fields of the record do not change. */ ulint btr_cur_optimistic_update( @@ -1507,10 +1509,11 @@ btr_cur_optimistic_update( ut_ad(mtr_memo_contains(mtr, buf_block_align(page), MTR_MEMO_PAGE_X_FIX)); - if (!row_upd_changes_field_size(rec, index, update)) { + if (!row_upd_changes_field_size_or_external(rec, index, update)) { - /* The simplest and most common case: the update does not - change the size of any field */ + /* The simplest and the most common case: the update does not + change the size of any field and none of the updated fields is + externally stored in rec or update */ return(btr_cur_update_in_place(flags, cursor, update, cmpl_info, thr, mtr)); @@ -1539,7 +1542,7 @@ btr_cur_optimistic_update( new_entry = row_rec_to_index_entry(ROW_COPY_DATA, index, rec, heap); - row_upd_clust_index_replace_new_col_vals(new_entry, update); + row_upd_index_replace_new_col_vals(new_entry, index, update, NULL); old_rec_size = rec_get_size(rec); new_rec_size = rec_get_converted_size(new_entry); @@ -1669,54 +1672,13 @@ btr_cur_pess_upd_restore_supremum( lock_rec_reset_and_inherit_gap_locks(page_get_supremum_rec(prev_page), rec); } - -/*************************************************************** -Replaces and copies the data in the new column values stored in the -update vector to the clustered index entry given. */ -static -void -btr_cur_copy_new_col_vals( -/*======================*/ - dtuple_t* entry, /* in/out: index entry where replaced */ - upd_t* update, /* in: update vector */ - mem_heap_t* heap) /* in: heap where data is copied */ -{ - upd_field_t* upd_field; - dfield_t* dfield; - dfield_t* new_val; - ulint field_no; - byte* data; - ulint i; - - dtuple_set_info_bits(entry, update->info_bits); - - for (i = 0; i < upd_get_n_fields(update); i++) { - - upd_field = upd_get_nth_field(update, i); - - field_no = upd_field->field_no; - - dfield = dtuple_get_nth_field(entry, field_no); - - new_val = &(upd_field->new_val); - - if (new_val->len == UNIV_SQL_NULL) { - data = NULL; - } else { - data = mem_heap_alloc(heap, new_val->len); - - ut_memcpy(data, new_val->data, new_val->len); - } - - dfield_set_data(dfield, data, new_val->len); - } -} /***************************************************************** Performs an update of a record on a page of a tree. It is assumed that mtr holds an x-latch on the tree and on the cursor page. If the update is made on the leaf level, to avoid deadlocks, mtr must also -own x-latches to brothers of page, if those brothers exist. */ +own x-latches to brothers of page, if those brothers exist. We assume +here that the ordering fields of the record do not change. */ ulint btr_cur_pessimistic_update( @@ -1813,7 +1775,7 @@ btr_cur_pessimistic_update( new_entry = row_rec_to_index_entry(ROW_COPY_DATA, index, rec, heap); - btr_cur_copy_new_col_vals(new_entry, update, heap); + row_upd_index_replace_new_col_vals(new_entry, index, update, heap); if (!(flags & BTR_KEEP_SYS_FLAG)) { row_upd_index_entry_sys_field(new_entry, index, DATA_ROLL_PTR, @@ -3369,8 +3331,8 @@ btr_free_externally_stored_field( page_no = mach_read_from_4(data + local_len + BTR_EXTERN_PAGE_NO); - offset = mach_read_from_4(data + local_len + BTR_EXTERN_OFFSET); - + offset = mach_read_from_4(data + local_len + + BTR_EXTERN_OFFSET); extern_len = mach_read_from_4(data + local_len + BTR_EXTERN_LEN + 4); diff --git a/innobase/btr/btr0pcur.c b/innobase/btr/btr0pcur.c index 7b817d8263d..63e7763ef87 100644 --- a/innobase/btr/btr0pcur.c +++ b/innobase/btr/btr0pcur.c @@ -364,6 +364,8 @@ btr_pcur_move_to_next_page( btr_leaf_page_release(page, cursor->latch_mode, mtr); page_cur_set_before_first(next_page, btr_pcur_get_page_cur(cursor)); + + page_check_dir(next_page); } /************************************************************* diff --git a/innobase/buf/buf0buf.c b/innobase/buf/buf0buf.c index e000d862403..246a60a61cd 100644 --- a/innobase/buf/buf0buf.c +++ b/innobase/buf/buf0buf.c @@ -209,12 +209,12 @@ ibool buf_debug_prints = FALSE; /* If this is set TRUE, /************************************************************************ Calculates a page checksum which is stored to the page when it is written -to a file. Note that we must be careful to calculate the same value -on 32-bit and 64-bit architectures. */ +to a file. Note that we must be careful to calculate the same value on +32-bit and 64-bit architectures. */ ulint -buf_calc_page_checksum( -/*===================*/ +buf_calc_page_new_checksum( +/*=======================*/ /* out: checksum */ byte* page) /* in: buffer page */ { @@ -222,12 +222,39 @@ buf_calc_page_checksum( /* Since the fields FIL_PAGE_FILE_FLUSH_LSN and ..._ARCH_LOG_NO are written outside the buffer pool to the first pages of data - files, we have to skip them in page checksum calculation */ + files, we have to skip them in the page checksum calculation. + We must also skip the field FIL_PAGE_SPACE_OR_CHKSUM where the + checksum is stored, and also the last 8 bytes of page because + there we store the old formula checksum. */ + + checksum = ut_fold_binary(page + FIL_PAGE_OFFSET, + FIL_PAGE_FILE_FLUSH_LSN - FIL_PAGE_OFFSET) + + ut_fold_binary(page + FIL_PAGE_DATA, + UNIV_PAGE_SIZE - FIL_PAGE_DATA + - FIL_PAGE_END_LSN_OLD_CHKSUM); + checksum = checksum & 0xFFFFFFFF; + + return(checksum); +} + +/************************************************************************ +In versions < 4.0.14 and < 4.1.1 there was a bug that the checksum only +looked at the first few bytes of the page. This calculates that old +checksum. +NOTE: we must first store the new formula checksum to +FIL_PAGE_SPACE_OR_CHKSUM before calculating and storing this old checksum +because this takes that field as an input! */ + +ulint +buf_calc_page_old_checksum( +/*=======================*/ + /* out: checksum */ + byte* page) /* in: buffer page */ +{ + ulint checksum; checksum = ut_fold_binary(page, FIL_PAGE_FILE_FLUSH_LSN); - + ut_fold_binary(page + FIL_PAGE_DATA, - UNIV_PAGE_SIZE - FIL_PAGE_DATA - - FIL_PAGE_END_LSN); + checksum = checksum & 0xFFFFFFFF; return(checksum); @@ -243,27 +270,47 @@ buf_page_is_corrupted( byte* read_buf) /* in: a database page */ { ulint checksum; + ulint old_checksum; + ulint checksum_field; + ulint old_checksum_field; - checksum = buf_calc_page_checksum(read_buf); + if (mach_read_from_4(read_buf + FIL_PAGE_LSN + 4) + != mach_read_from_4(read_buf + UNIV_PAGE_SIZE + - FIL_PAGE_END_LSN_OLD_CHKSUM + 4)) { + + /* Stored log sequence numbers at the start and the end + of page do not match */ - /* Note that InnoDB initializes empty pages to zero, and - early versions of InnoDB did not store page checksum to - the 4 most significant bytes of the page lsn field at the - end of a page: */ - - if ((mach_read_from_4(read_buf + FIL_PAGE_LSN + 4) - != mach_read_from_4(read_buf + UNIV_PAGE_SIZE - - FIL_PAGE_END_LSN + 4)) - || (checksum != mach_read_from_4(read_buf - + UNIV_PAGE_SIZE - - FIL_PAGE_END_LSN) - && mach_read_from_4(read_buf + FIL_PAGE_LSN) - != mach_read_from_4(read_buf - + UNIV_PAGE_SIZE - - FIL_PAGE_END_LSN))) { return(TRUE); } + old_checksum = buf_calc_page_old_checksum(read_buf); + + old_checksum_field = mach_read_from_4(read_buf + UNIV_PAGE_SIZE + - FIL_PAGE_END_LSN_OLD_CHKSUM); + + /* There are 2 valid formulas for old_checksum_field: + 1. Very old versions of InnoDB only stored 8 byte lsn to the start + and the end of the page. + 2. Newer InnoDB versions store the old formula checksum there. */ + + if (old_checksum_field != mach_read_from_4(read_buf + FIL_PAGE_LSN) + && old_checksum_field != old_checksum) { + + return(TRUE); + } + + checksum = buf_calc_page_new_checksum(read_buf); + checksum_field = mach_read_from_4(read_buf + FIL_PAGE_SPACE_OR_CHKSUM); + + /* InnoDB versions < 4.0.14 and < 4.1.1 stored the space id + (always equal to 0), to FIL_PAGE_SPACE_SPACE_OR_CHKSUM */ + + if (checksum_field != 0 && checksum_field != checksum) { + + return(TRUE); + } + return(FALSE); } @@ -277,6 +324,7 @@ buf_page_print( { dict_index_t* index; ulint checksum; + ulint old_checksum; char* buf; buf = mem_alloc(4 * UNIV_PAGE_SIZE); @@ -291,19 +339,23 @@ buf_page_print( mem_free(buf); - checksum = buf_calc_page_checksum(read_buf); + checksum = buf_calc_page_new_checksum(read_buf); + old_checksum = buf_calc_page_old_checksum(read_buf); ut_print_timestamp(stderr); - fprintf(stderr, " InnoDB: Page checksum %lu stored checksum %lu\n", - checksum, mach_read_from_4(read_buf - + UNIV_PAGE_SIZE - - FIL_PAGE_END_LSN)); + fprintf(stderr, +" InnoDB: Page checksum %lu, prior-to-4.0.14-form checksum %lu\n" +"InnoDB: stored checksum %lu, prior-to-4.0.14-form stored checksum %lu\n", + checksum, old_checksum, + mach_read_from_4(read_buf + FIL_PAGE_SPACE_OR_CHKSUM), + mach_read_from_4(read_buf + UNIV_PAGE_SIZE + - FIL_PAGE_END_LSN_OLD_CHKSUM)); fprintf(stderr, "InnoDB: Page lsn %lu %lu, low 4 bytes of lsn at page end %lu\n", mach_read_from_4(read_buf + FIL_PAGE_LSN), mach_read_from_4(read_buf + FIL_PAGE_LSN + 4), mach_read_from_4(read_buf + UNIV_PAGE_SIZE - - FIL_PAGE_END_LSN + 4)); + - FIL_PAGE_END_LSN_OLD_CHKSUM + 4)); if (mach_read_from_2(read_buf + TRX_UNDO_PAGE_HDR + TRX_UNDO_PAGE_TYPE) == TRX_UNDO_INSERT) { fprintf(stderr, diff --git a/innobase/buf/buf0flu.c b/innobase/buf/buf0flu.c index 735966c28c5..c0999ee4841 100644 --- a/innobase/buf/buf0flu.c +++ b/innobase/buf/buf0flu.c @@ -361,21 +361,29 @@ buf_flush_init_for_writing( ulint space, /* in: space id */ ulint page_no) /* in: page number */ { - /* Write the newest modification lsn to the page */ + UT_NOT_USED(space); + + /* Write the newest modification lsn to the page header and trailer */ mach_write_to_8(page + FIL_PAGE_LSN, newest_lsn); - mach_write_to_8(page + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN, newest_lsn); + mach_write_to_8(page + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM, + newest_lsn); + /* Write the page number */ - /* Write to the page the space id and page number */ - - mach_write_to_4(page + FIL_PAGE_SPACE, space); mach_write_to_4(page + FIL_PAGE_OFFSET, page_no); - /* We overwrite the first 4 bytes of the end lsn field to store - a page checksum */ + /* Store the new formula checksum */ - mach_write_to_4(page + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN, - buf_calc_page_checksum(page)); + mach_write_to_4(page + FIL_PAGE_SPACE_OR_CHKSUM, + buf_calc_page_new_checksum(page)); + + /* We overwrite the first 4 bytes of the end lsn field to store + the old formula checksum. Since it depends also on the field + FIL_PAGE_SPACE_OR_CHKSUM, it has to be calculated after storing the + new formula checksum. */ + + mach_write_to_4(page + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM, + buf_calc_page_old_checksum(page)); } /************************************************************************ diff --git a/innobase/data/data0data.c b/innobase/data/data0data.c index 8ab5acb4da7..f2f94cc47ce 100644 --- a/innobase/data/data0data.c +++ b/innobase/data/data0data.c @@ -584,8 +584,7 @@ dtuple_convert_big_rec( * sizeof(big_rec_field_t)); /* Decide which fields to shorten: the algorithm is to look for - the longest field which does not occur in the ordering part - of any index on the table */ + the longest field whose type is DATA_BLOB */ n_fields = 0; @@ -610,12 +609,9 @@ dtuple_convert_big_rec( } } - /* Skip over fields which are ordering in some index */ - - if (!is_externally_stored && - dict_field_get_col( - dict_index_get_nth_field(index, i)) - ->ord_part == 0) { + if (!is_externally_stored + && dict_index_get_nth_type(index, i)->mtype + == DATA_BLOB) { dfield = dtuple_get_nth_field(entry, i); @@ -629,9 +625,13 @@ dtuple_convert_big_rec( } } - if (longest < BTR_EXTERN_FIELD_REF_SIZE + 10 - + REC_1BYTE_OFFS_LIMIT) { + /* We do not store externally fields which are smaller than + DICT_MAX_COL_PREFIX_LEN */ + ut_a(DICT_MAX_COL_PREFIX_LEN > REC_1BYTE_OFFS_LIMIT); + + if (longest < BTR_EXTERN_FIELD_REF_SIZE + 10 + + DICT_MAX_COL_PREFIX_LEN) { /* Cannot shorten more */ mem_heap_free(heap); @@ -644,13 +644,19 @@ dtuple_convert_big_rec( drop below 128 which is the limit for the 2-byte offset storage format in a physical record. This we accomplish by storing 128 bytes of data in entry - itself, and only the remaining part to big rec vec. */ + itself, and only the remaining part to big rec vec. + + We store the first bytes locally to the record. Then + we can calculate all ordering fields in all indexes + from locally stored data. */ dfield = dtuple_get_nth_field(entry, longest_i); vector->fields[n_fields].field_no = longest_i; + ut_a(dfield->len > DICT_MAX_COL_PREFIX_LEN); + vector->fields[n_fields].len = dfield->len - - REC_1BYTE_OFFS_LIMIT; + - DICT_MAX_COL_PREFIX_LEN; vector->fields[n_fields].data = mem_heap_alloc(heap, vector->fields[n_fields].len); diff --git a/innobase/data/data0type.c b/innobase/data/data0type.c index 5d0ddf3e887..df430f06bcb 100644 --- a/innobase/data/data0type.c +++ b/innobase/data/data0type.c @@ -85,8 +85,6 @@ dtype_print( printf("DATA_MIX_ID"); } else if (prtype == DATA_ENGLISH) { printf("DATA_ENGLISH"); - } else if (prtype == DATA_FINNISH) { - printf("DATA_FINNISH"); } else { printf("prtype %lu", mtype); } diff --git a/innobase/dict/dict0boot.c b/innobase/dict/dict0boot.c index 374c567c3ca..0bf2ace3324 100644 --- a/innobase/dict/dict0boot.c +++ b/innobase/dict/dict0boot.c @@ -276,7 +276,7 @@ dict_boot(void) DICT_HDR_SPACE, DICT_UNIQUE | DICT_CLUSTERED, 1); - dict_mem_index_add_field(index, (char *) "NAME", 0); + dict_mem_index_add_field(index, (char *) "NAME", 0, 0); index->page_no = mtr_read_ulint(dict_hdr + DICT_HDR_TABLES, MLOG_4BYTES, &mtr); @@ -287,7 +287,7 @@ dict_boot(void) index = dict_mem_index_create((char *) "SYS_TABLES", (char *) "ID_IND", DICT_HDR_SPACE, DICT_UNIQUE, 1); - dict_mem_index_add_field(index, (char *) "ID", 0); + dict_mem_index_add_field(index, (char *) "ID", 0, 0); index->page_no = mtr_read_ulint(dict_hdr + DICT_HDR_TABLE_IDS, MLOG_4BYTES, &mtr); @@ -313,8 +313,8 @@ dict_boot(void) (char *) "CLUST_IND", DICT_HDR_SPACE, DICT_UNIQUE | DICT_CLUSTERED, 2); - dict_mem_index_add_field(index, (char *) "TABLE_ID", 0); - dict_mem_index_add_field(index, (char *) "POS", 0); + dict_mem_index_add_field(index, (char *) "TABLE_ID", 0, 0); + dict_mem_index_add_field(index, (char *) "POS", 0, 0); index->page_no = mtr_read_ulint(dict_hdr + DICT_HDR_COLUMNS, MLOG_4BYTES, &mtr); @@ -343,8 +343,8 @@ dict_boot(void) (char *) "CLUST_IND", DICT_HDR_SPACE, DICT_UNIQUE | DICT_CLUSTERED, 2); - dict_mem_index_add_field(index, (char *) "TABLE_ID", 0); - dict_mem_index_add_field(index, (char *) "ID", 0); + dict_mem_index_add_field(index, (char *) "TABLE_ID", 0, 0); + dict_mem_index_add_field(index, (char *) "ID", 0, 0); index->page_no = mtr_read_ulint(dict_hdr + DICT_HDR_INDEXES, MLOG_4BYTES, &mtr); @@ -365,8 +365,8 @@ dict_boot(void) (char *) "CLUST_IND", DICT_HDR_SPACE, DICT_UNIQUE | DICT_CLUSTERED, 2); - dict_mem_index_add_field(index, (char *) "INDEX_ID", 0); - dict_mem_index_add_field(index, (char *) "POS", 0); + dict_mem_index_add_field(index, (char *) "INDEX_ID", 0, 0); + dict_mem_index_add_field(index, (char *) "POS", 0, 0); index->page_no = mtr_read_ulint(dict_hdr + DICT_HDR_FIELDS, MLOG_4BYTES, &mtr); diff --git a/innobase/dict/dict0crea.c b/innobase/dict/dict0crea.c index 3619ac02f4d..9139e589a0a 100644 --- a/innobase/dict/dict0crea.c +++ b/innobase/dict/dict0crea.c @@ -337,7 +337,7 @@ dict_create_index_for_cluster_step( for (i = 0; i < table->n_cols; i++) { col = dict_table_get_nth_col(table, i); - dict_mem_index_add_field(index, col->name, 0); + dict_mem_index_add_field(index, col->name, 0, 0); } (node->cluster)->index = index; @@ -450,9 +450,17 @@ dict_create_sys_fields_tuple( dict_field_t* field; dfield_t* dfield; byte* ptr; + ibool index_contains_column_prefix_field = FALSE; + ulint j; ut_ad(index && heap); + for (j = 0; j < index->n_fields; j++) { + if (dict_index_get_nth_field(index, j)->prefix_len > 0) { + index_contains_column_prefix_field = TRUE; + } + } + field = dict_index_get_nth_field(index, i); sys_fields = dict_sys->sys_fields; @@ -466,11 +474,25 @@ dict_create_sys_fields_tuple( mach_write_to_8(ptr, index->id); dfield_set_data(dfield, ptr, 8); - /* 1: POS ----------------------------*/ + /* 1: POS + PREFIX LENGTH ----------------------------*/ + dfield = dtuple_get_nth_field(entry, 1); ptr = mem_heap_alloc(heap, 4); - mach_write_to_4(ptr, i); + + if (index_contains_column_prefix_field) { + /* If there are column prefix fields in the index, then + we store the number of the field to the 2 HIGH bytes + and the prefix length to the 2 low bytes, */ + + mach_write_to_4(ptr, (i << 16) + field->prefix_len); + } else { + /* Else we store the number of the field to the 2 LOW bytes. + This is to keep the storage format compatible with + InnoDB versions < 4.0.14. */ + + mach_write_to_4(ptr, i); + } dfield_set_data(dfield, ptr, 4); /* 4: COL_NAME -------------------------*/ diff --git a/innobase/dict/dict0dict.c b/innobase/dict/dict0dict.c index c11a5f76d94..2fc05b1923f 100644 --- a/innobase/dict/dict0dict.c +++ b/innobase/dict/dict0dict.c @@ -88,15 +88,6 @@ dict_index_remove_from_cache( dict_table_t* table, /* in: table */ dict_index_t* index); /* in, own: index */ /*********************************************************************** -Adds a column to index. */ -UNIV_INLINE -void -dict_index_add_col( -/*===============*/ - dict_index_t* index, /* in: index */ - dict_col_t* col, /* in: column */ - ulint order); /* in: order criterion */ -/*********************************************************************** Copies fields contained in index2 to index1. */ static void @@ -482,8 +473,9 @@ dict_index_get_nth_col_pos( ut_ad(index); ut_ad(index->magic_n == DICT_INDEX_MAGIC_N); + col = dict_table_get_nth_col(index->table, n); + if (index->type & DICT_CLUSTERED) { - col = dict_table_get_nth_col(index->table, n); return(col->clust_pos); } @@ -492,9 +484,8 @@ dict_index_get_nth_col_pos( for (pos = 0; pos < n_fields; pos++) { field = dict_index_get_nth_field(index, pos); - col = field->col; - if (dict_col_get_no(col) == n) { + if (col == field->col && field->prefix_len == 0) { return(pos); } @@ -502,7 +493,46 @@ dict_index_get_nth_col_pos( return(ULINT_UNDEFINED); } + +/************************************************************************ +Looks for a matching field in an index. The column and the prefix len have +to be the same. */ + +ulint +dict_index_get_nth_field_pos( +/*=========================*/ + /* out: position in internal representation + of the index; if not contained, returns + ULINT_UNDEFINED */ + dict_index_t* index, /* in: index from which to search */ + dict_index_t* index2, /* in: index */ + ulint n) /* in: field number in index2 */ +{ + dict_field_t* field; + dict_field_t* field2; + ulint n_fields; + ulint pos; + ut_ad(index); + ut_ad(index->magic_n == DICT_INDEX_MAGIC_N); + + field2 = dict_index_get_nth_field(index2, n); + + n_fields = dict_index_get_n_fields(index); + + for (pos = 0; pos < n_fields; pos++) { + field = dict_index_get_nth_field(index, pos); + + if (field->col == field2->col + && field->prefix_len == field2->prefix_len) { + + return(pos); + } + } + + return(ULINT_UNDEFINED); +} + /************************************************************************** Returns a table object, based on table id, and memoryfixes it. */ @@ -622,8 +652,7 @@ dict_table_get( } /************************************************************************** -Returns a table object and increments MySQL open handle count on the table. -*/ +Returns a table object and increments MySQL open handle count on the table. */ dict_table_t* dict_table_get_and_increment_handle_count( @@ -732,11 +761,12 @@ dict_table_add_to_cache( } /* Add table to hash table of tables */ - HASH_INSERT(dict_table_t, name_hash, dict_sys->table_hash, fold, table); + HASH_INSERT(dict_table_t, name_hash, dict_sys->table_hash, fold, + table); /* Add table to hash table of tables based on table id */ HASH_INSERT(dict_table_t, id_hash, dict_sys->table_id_hash, id_fold, - table); + table); /* Add table to LRU list of tables */ UT_LIST_ADD_FIRST(table_LRU, dict_sys->table_LRU, table); @@ -828,7 +858,7 @@ dict_table_rename_in_cache( /* Remove table from the hash tables of tables */ HASH_DELETE(dict_table_t, name_hash, dict_sys->table_hash, - ut_fold_string(table->name), table); + ut_fold_string(table->name), table); name_buf = mem_heap_alloc(table->heap, ut_strlen(new_name) + 1); @@ -837,7 +867,8 @@ dict_table_rename_in_cache( table->name = name_buf; /* Add table to hash table of tables */ - HASH_INSERT(dict_table_t, name_hash, dict_sys->table_hash, fold, table); + HASH_INSERT(dict_table_t, name_hash, dict_sys->table_hash, fold, + table); dict_sys->size += (mem_heap_get_size(table->heap) - old_size); @@ -1128,7 +1159,6 @@ dict_index_add_to_cache( ulint n_ord; ibool success; ulint i; - ulint j; ut_ad(index); ut_ad(mutex_own(&(dict_sys->mutex))); @@ -1158,28 +1188,6 @@ dict_index_add_to_cache( return(FALSE); } - - /* Check that the same column does not appear twice in the index. - InnoDB assumes this in its algorithms, e.g., update of an index - entry */ - - for (i = 0; i < dict_index_get_n_fields(index); i++) { - - for (j = 0; j < i; j++) { - if (dict_index_get_nth_field(index, j)->col - == dict_index_get_nth_field(index, i)->col) { - - ut_print_timestamp(stderr); - - fprintf(stderr, -" InnoDB: Error: column %s appears twice in index %s of table %s\n" -"InnoDB: This is not allowed in InnoDB.\n" -"InnoDB: UPDATE can cause such an index to become corrupt in InnoDB.\n", - dict_index_get_nth_field(index, i)->col->name, - index->name, table->name); - } - } - } /* Build the cache internal representation of the index, containing also the added system fields */ @@ -1223,8 +1231,8 @@ dict_index_add_to_cache( cluster = dict_table_get_low(table->cluster_name); - tree = dict_index_get_tree(UT_LIST_GET_FIRST(cluster->indexes)); - + tree = dict_index_get_tree( + UT_LIST_GET_FIRST(cluster->indexes)); new_index->tree = tree; new_index->page_no = tree->page; } else { @@ -1352,13 +1360,14 @@ UNIV_INLINE void dict_index_add_col( /*===============*/ - dict_index_t* index, /* in: index */ - dict_col_t* col, /* in: column */ - ulint order) /* in: order criterion */ + dict_index_t* index, /* in: index */ + dict_col_t* col, /* in: column */ + ulint order, /* in: order criterion */ + ulint prefix_len) /* in: column prefix length */ { dict_field_t* field; - dict_mem_index_add_field(index, col->name, order); + dict_mem_index_add_field(index, col->name, order, prefix_len); field = dict_index_get_nth_field(index, index->n_def - 1); @@ -1384,7 +1393,8 @@ dict_index_copy( for (i = start; i < end; i++) { field = dict_index_get_nth_field(index2, i); - dict_index_add_col(index1, field->col, field->order); + dict_index_add_col(index1, field->col, field->order, + field->prefix_len); } } @@ -1487,7 +1497,7 @@ dict_index_build_internal_clust( /* Add the mix id column */ dict_index_add_col(new_index, - dict_table_get_sys_col(table, DATA_MIX_ID), 0); + dict_table_get_sys_col(table, DATA_MIX_ID), 0, 0); /* Copy the rest of fields */ dict_index_copy(new_index, index, table->mix_len, @@ -1525,14 +1535,15 @@ dict_index_build_internal_clust( if (!(index->type & DICT_UNIQUE)) { dict_index_add_col(new_index, - dict_table_get_sys_col(table, DATA_ROW_ID), 0); + dict_table_get_sys_col(table, DATA_ROW_ID), 0, 0); trx_id_pos++; } dict_index_add_col(new_index, - dict_table_get_sys_col(table, DATA_TRX_ID), 0); + dict_table_get_sys_col(table, DATA_TRX_ID), 0, 0); + dict_index_add_col(new_index, - dict_table_get_sys_col(table, DATA_ROLL_PTR), 0); + dict_table_get_sys_col(table, DATA_ROLL_PTR), 0, 0); for (i = 0; i < trx_id_pos; i++) { @@ -1561,7 +1572,14 @@ dict_index_build_internal_clust( for (i = 0; i < new_index->n_def; i++) { field = dict_index_get_nth_field(new_index, i); - (field->col)->aux = 0; + + /* If there is only a prefix of the column in the index + field, do not mark the column as contained in the index */ + + if (field->prefix_len == 0) { + + field->col->aux = 0; + } } /* Add to new_index non-system columns of table not yet included @@ -1572,7 +1590,7 @@ dict_index_build_internal_clust( ut_ad(col->type.mtype != DATA_SYS); if (col->aux == ULINT_UNDEFINED) { - dict_index_add_col(new_index, col, 0); + dict_index_add_col(new_index, col, 0, 0); } } @@ -1584,7 +1602,11 @@ dict_index_build_internal_clust( for (i = 0; i < new_index->n_def; i++) { field = dict_index_get_nth_field(new_index, i); - (field->col)->clust_pos = i; + + if (field->prefix_len == 0) { + + field->col->clust_pos = i; + } } new_index->cached = TRUE; @@ -1646,25 +1668,33 @@ dict_index_build_internal_non_clust( for (i = 0; i < clust_index->n_uniq; i++) { field = dict_index_get_nth_field(clust_index, i); - (field->col)->aux = ULINT_UNDEFINED; + field->col->aux = ULINT_UNDEFINED; } /* Mark with 0 table columns already contained in new_index */ for (i = 0; i < new_index->n_def; i++) { field = dict_index_get_nth_field(new_index, i); - (field->col)->aux = 0; + + /* If there is only a prefix of the column in the index + field, do not mark the column as contained in the index */ + + if (field->prefix_len == 0) { + + field->col->aux = 0; + } } - /* Add to new_index columns necessary to determine the clustered + /* Add to new_index the columns necessary to determine the clustered index entry uniquely */ for (i = 0; i < clust_index->n_uniq; i++) { field = dict_index_get_nth_field(clust_index, i); - if ((field->col)->aux == ULINT_UNDEFINED) { - dict_index_add_col(new_index, field->col, 0); + if (field->col->aux == ULINT_UNDEFINED) { + dict_index_add_col(new_index, field->col, 0, + field->prefix_len); } } @@ -1787,6 +1817,14 @@ dict_foreign_find_index( for (i = 0; i < n_cols; i++) { col_name = dict_index_get_nth_field(index, i) ->col->name; + if (dict_index_get_nth_field(index, i) + ->prefix_len != 0) { + /* We do not accept column prefix + indexes here */ + + break; + } + if (ut_strlen(columns[i]) != ut_strlen(col_name) || 0 != ut_cmp_in_lower_case(columns[i], @@ -3776,6 +3814,10 @@ dict_field_print_low( ut_ad(mutex_own(&(dict_sys->mutex))); printf(" %s", field->name); + + if (field->prefix_len != 0) { + printf("(%lu)", field->prefix_len); + } } /************************************************************************** diff --git a/innobase/dict/dict0load.c b/innobase/dict/dict0load.c index 8f39605e493..48c445fa0c9 100644 --- a/innobase/dict/dict0load.c +++ b/innobase/dict/dict0load.c @@ -301,6 +301,8 @@ dict_load_fields( dtuple_t* tuple; dfield_t* dfield; char* col_name; + ulint pos_and_prefix_len; + ulint prefix_len; rec_t* rec; byte* field; ulint len; @@ -345,8 +347,28 @@ dict_load_fields( ut_a(ut_memcmp(buf, field, len) == 0); field = rec_get_nth_field(rec, 1, &len); - ut_ad(len == 4); - ut_a(i == mach_read_from_4(field)); + ut_a(len == 4); + + /* The next field stores the field position in the index + and a possible column prefix length if the index field + does not contain the whole column. The storage format is + like this: if there is at least one prefix field in the index, + then the HIGH 2 bytes contain the field number (== i) and the + low 2 bytes the prefix length for the field. Otherwise the + field number (== i) is contained in the 2 LOW bytes. */ + + pos_and_prefix_len = mach_read_from_4(field); + + ut_a((pos_and_prefix_len & 0xFFFF) == i + || (pos_and_prefix_len & 0xFFFF0000) == (i << 16)); + + if ((i == 0 && pos_and_prefix_len > 0) + || (pos_and_prefix_len & 0xFFFF0000) > 0) { + + prefix_len = pos_and_prefix_len & 0xFFFF; + } else { + prefix_len = 0; + } ut_a(0 == ut_strcmp((char*) "COL_NAME", dict_field_get_col( @@ -359,7 +381,7 @@ dict_load_fields( ut_memcpy(col_name, field, len); col_name[len] = '\0'; - dict_mem_index_add_field(index, col_name, 0); + dict_mem_index_add_field(index, col_name, 0, prefix_len); btr_pcur_move_to_next_user_rec(&pcur, &mtr); } diff --git a/innobase/dict/dict0mem.c b/innobase/dict/dict0mem.c index e5918c6aeb6..56efc0a0117 100644 --- a/innobase/dict/dict0mem.c +++ b/innobase/dict/dict0mem.c @@ -266,10 +266,13 @@ by the column name may be released only after publishing the index. */ void dict_mem_index_add_field( /*=====================*/ - dict_index_t* index, /* in: index */ - char* name, /* in: column name */ - ulint order) /* in: order criterion; 0 means an ascending - order */ + dict_index_t* index, /* in: index */ + char* name, /* in: column name */ + ulint order, /* in: order criterion; 0 means an + ascending order */ + ulint prefix_len) /* in: 0 or the column prefix length + in a MySQL index like + INDEX (textcol(25)) */ { dict_field_t* field; @@ -282,6 +285,8 @@ dict_mem_index_add_field( field->name = name; field->order = order; + + field->prefix_len = prefix_len; } /************************************************************************** diff --git a/innobase/fil/fil0fil.c b/innobase/fil/fil0fil.c index 98980f6c337..a8dc357749c 100644 --- a/innobase/fil/fil0fil.c +++ b/innobase/fil/fil0fil.c @@ -632,7 +632,7 @@ fil_space_create( /* Spaces with an odd id number are reserved to replicate spaces used in log debugging */ - ut_a((purpose == FIL_LOG) || (id % 2 == 0)); + ut_anp((purpose == FIL_LOG) || (id % 2 == 0)); #endif mutex_enter(&(system->mutex)); @@ -1202,8 +1202,8 @@ loop: /* Do aio */ - ut_a(byte_offset % OS_FILE_LOG_BLOCK_SIZE == 0); - ut_a((len % OS_FILE_LOG_BLOCK_SIZE) == 0); + ut_anp(byte_offset % OS_FILE_LOG_BLOCK_SIZE == 0); + ut_anp((len % OS_FILE_LOG_BLOCK_SIZE) == 0); /* Queue the aio request */ ret = os_aio(type, mode | wake_later, node->name, node->handle, buf, diff --git a/innobase/fsp/fsp0fsp.c b/innobase/fsp/fsp0fsp.c index ee48288b875..b6941d80e90 100644 --- a/innobase/fsp/fsp0fsp.c +++ b/innobase/fsp/fsp0fsp.c @@ -778,7 +778,7 @@ fsp_init_file_page_low( page[i] = 0xFF; } #endif - mach_write_to_8(page + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN, + mach_write_to_8(page + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM, ut_dulint_zero); mach_write_to_8(page + FIL_PAGE_LSN, ut_dulint_zero); } @@ -2875,7 +2875,7 @@ fseg_free_step( freed yet */ ut_a(descr); - ut_a(xdes_get_bit(descr, XDES_FREE_BIT, buf_frame_get_page_no(header) + ut_anp(xdes_get_bit(descr, XDES_FREE_BIT, buf_frame_get_page_no(header) % FSP_EXTENT_SIZE, mtr) == FALSE); inode = fseg_inode_get(header, mtr); diff --git a/innobase/ha/ha0ha.c b/innobase/ha/ha0ha.c index 4489b25ec2b..eb28e15215d 100644 --- a/innobase/ha/ha0ha.c +++ b/innobase/ha/ha0ha.c @@ -293,11 +293,13 @@ ha_print_info( hash_table_t* table) /* in: hash table */ { hash_cell_t* cell; -/* ha_node_t* node; */ - ulint nodes = 0; - ulint cells = 0; +/* + ha_node_t* node; ulint len = 0; ulint max_len = 0; + ulint nodes = 0; +*/ + ulint cells = 0; ulint n_bufs; ulint i; diff --git a/innobase/ibuf/ibuf0ibuf.c b/innobase/ibuf/ibuf0ibuf.c index 187afa17047..c07756ab308 100644 --- a/innobase/ibuf/ibuf0ibuf.c +++ b/innobase/ibuf/ibuf0ibuf.c @@ -170,7 +170,7 @@ dropped! So, there seems to be no problem. */ /********************************************************************** Validates the ibuf data structures when the caller owns ibuf_mutex. */ -static + ibool ibuf_validate_low(void); /*===================*/ @@ -484,8 +484,8 @@ ibuf_data_init_for_space( index = dict_mem_index_create(buf, (char *) "CLUST_IND", space, DICT_CLUSTERED | DICT_UNIVERSAL | DICT_IBUF,2); - dict_mem_index_add_field(index, (char *) "PAGE_NO", 0); - dict_mem_index_add_field(index, (char *) "TYPES", 0); + dict_mem_index_add_field(index, (char *) "PAGE_NO", 0, 0); + dict_mem_index_add_field(index, (char *) "TYPES", 0, 0); index->page_no = FSP_IBUF_TREE_ROOT_PAGE_NO; @@ -2727,7 +2727,7 @@ reset_bit: /********************************************************************** Validates the ibuf data structures when the caller owns ibuf_mutex. */ -static + ibool ibuf_validate_low(void) /*===================*/ diff --git a/innobase/include/btr0cur.h b/innobase/include/btr0cur.h index 1d17c0e952d..506877333c3 100644 --- a/innobase/include/btr0cur.h +++ b/innobase/include/btr0cur.h @@ -690,7 +690,13 @@ and sleep this many microseconds in between */ #define BTR_CUR_RETRY_DELETE_N_TIMES 100 #define BTR_CUR_RETRY_SLEEP_TIME 50000 -/* The reference in a field of which data is stored on a different page */ +/* The reference in a field for which data is stored on a different page. +The reference is at the end of the 'locally' stored part of the field. +'Locally' means storage in the index record. +We store locally a long enough prefix of each column so that we can determine +the ordering parts of each index record without looking into the externally +stored part. */ + /*--------------------------------------*/ #define BTR_EXTERN_SPACE_ID 0 /* space id where stored */ #define BTR_EXTERN_PAGE_NO 4 /* page no where stored */ diff --git a/innobase/include/buf0buf.h b/innobase/include/buf0buf.h index e4d3671586d..2963efd6396 100644 --- a/innobase/include/buf0buf.h +++ b/innobase/include/buf0buf.h @@ -364,11 +364,24 @@ to a file. Note that we must be careful to calculate the same value on 32-bit and 64-bit architectures. */ ulint -buf_calc_page_checksum( -/*===================*/ +buf_calc_page_new_checksum( +/*=======================*/ /* out: checksum */ byte* page); /* in: buffer page */ /************************************************************************ +In versions < 4.0.14 and < 4.1.1 there was a bug that the checksum only +looked at the first few bytes of the page. This calculates that old +checksum. +NOTE: we must first store the new formula checksum to +FIL_PAGE_SPACE_OR_CHKSUM before calculating and storing this old checksum +because this takes that field as an input! */ + +ulint +buf_calc_page_old_checksum( +/*=======================*/ + /* out: checksum */ + byte* page); /* in: buffer page */ +/************************************************************************ Checks if a page is corrupt. */ ibool diff --git a/innobase/include/data0data.h b/innobase/include/data0data.h index e0fb06e5018..889d148d3fe 100644 --- a/innobase/include/data0data.h +++ b/innobase/include/data0data.h @@ -453,8 +453,6 @@ struct dfield_struct{ void* data; /* pointer to data */ ulint len; /* data length; UNIV_SQL_NULL if SQL null; */ dtype_t type; /* type of data */ - ulint col_no; /* when building index entries, the column - number can be stored here */ }; struct dtuple_struct { diff --git a/innobase/include/data0type.h b/innobase/include/data0type.h index b53a70a8909..5e28f657f0c 100644 --- a/innobase/include/data0type.h +++ b/innobase/include/data0type.h @@ -18,14 +18,16 @@ typedef struct dtype_struct dtype_t; data type */ extern dtype_t* dtype_binary; -/* Data main types of SQL data; NOTE! character data types requiring -collation transformation must have the smallest codes! All codes must be -less than 256! */ +/* Data main types of SQL data */ #define DATA_VARCHAR 1 /* character varying */ #define DATA_CHAR 2 /* fixed length character */ #define DATA_FIXBINARY 3 /* binary string of fixed length */ #define DATA_BINARY 4 /* binary string */ -#define DATA_BLOB 5 /* binary large object */ +#define DATA_BLOB 5 /* binary large object, or a TEXT type; if + prtype & DATA_NONLATIN1 != 0 the data must + be compared by MySQL as a whole field; if + prtype & DATA_BINARY_TYPE == 0, then this is + actually a TEXT column */ #define DATA_INT 6 /* integer: can be any size 1 - 8 bytes */ #define DATA_SYS_CHILD 7 /* address of the child page in node pointer */ #define DATA_SYS 8 /* system column */ @@ -34,35 +36,55 @@ binary strings */ #define DATA_FLOAT 9 #define DATA_DOUBLE 10 #define DATA_DECIMAL 11 /* decimal number stored as an ASCII string */ -#define DATA_VARMYSQL 12 /* data types for which comparisons must be */ -#define DATA_MYSQL 13 /* made by MySQL */ -#define DATA_ERROR 111 /* error value */ -#define DATA_MTYPE_MAX 255 +#define DATA_VARMYSQL 12 /* non-latin1 varying length char */ +#define DATA_MYSQL 13 /* non-latin1 fixed length char */ +#define DATA_MTYPE_MAX 63 /* dtype_store_for_order_and_null_size() + requires the values are <= 63 */ /*-------------------------------------------*/ -/* Precise data types for system columns; NOTE: the values must run -from 0 up in the order given! All codes must be less than 256! */ +/* In the lowest byte in the precise type we store the MySQL type code +(not applicable for system columns). */ + +#define DATA_ENGLISH 4 /* English language character string: this + is a relic from pre-MySQL time and only used + for InnoDB's own system tables */ +#define DATA_ERROR 111 /* another relic from pre-MySQL time */ + +#define DATA_MYSQL_TYPE_MASK 255 /* AND with this mask to extract the MySQL + type from the precise type */ + +/* Precise data types for system columns and the length of those columns; +NOTE: the values must run from 0 up in the order given! All codes must +be less than 256 */ #define DATA_ROW_ID 0 /* row id: a dulint */ #define DATA_ROW_ID_LEN 6 /* stored length for row id */ + #define DATA_TRX_ID 1 /* transaction id: 6 bytes */ #define DATA_TRX_ID_LEN 6 + #define DATA_ROLL_PTR 2 /* rollback data pointer: 7 bytes */ #define DATA_ROLL_PTR_LEN 7 + #define DATA_MIX_ID 3 /* mixed index label: a dulint, stored in a row in a compressed form */ #define DATA_MIX_ID_LEN 9 /* maximum stored length for mix id (in a compressed dulint form) */ #define DATA_N_SYS_COLS 4 /* number of system columns defined above */ +/*-------------------------------------------*/ +/* Flags ORed to the precise data type */ #define DATA_NOT_NULL 256 /* this is ORed to the precise type when the column is declared as NOT NULL */ #define DATA_UNSIGNED 512 /* this id ORed to the precise type when we have an unsigned integer type */ +#define DATA_BINARY_TYPE 1024 /* if the data type is a binary character + string, this is ORed to the precise type: + this only holds for tables created with + >= MySQL-4.0.14 */ +#define DATA_NONLATIN1 2048 /* if the data type is a character string + of a non-latin1 type, this is ORed to the + precise type: this only holds for tables + created with >= MySQL-4.0.14 */ /*-------------------------------------------*/ -/* Precise types of a char or varchar data. All codes must be less than 256! */ -#define DATA_ENGLISH 4 /* English language character string */ -#define DATA_FINNISH 5 /* Finnish */ -#define DATA_PRTYPE_MAX 255 - /* This many bytes we need to store the type information affecting the alphabetical order for a single field and decide the storage size of an SQL null*/ @@ -123,7 +145,7 @@ dtype_get_pad_char( /*===============*/ /* out: padding character code, or ULINT_UNDEFINED if no padding specified */ - dtype_t* type); /* in: typeumn */ + dtype_t* type); /* in: type */ /*************************************************************************** Returns the size of a fixed size data type, 0 if not a fixed size type. */ UNIV_INLINE @@ -150,24 +172,24 @@ dtype_is_fixed_size( /* out: TRUE if fixed size */ dtype_t* type); /* in: type */ /************************************************************************** -Stores to a type the information which determines its alphabetical -ordering. */ +Stores for a type the information which determines its alphabetical ordering +and the storage size of an SQL NULL value. */ UNIV_INLINE void dtype_store_for_order_and_null_size( /*================================*/ byte* buf, /* in: buffer for DATA_ORDER_NULL_TYPE_BUF_SIZE - bytes */ + bytes where we store the info */ dtype_t* type); /* in: type struct */ /************************************************************************** -Reads of a type the stored information which determines its alphabetical -ordering. */ +Reads to a type the stored information which determines its alphabetical +ordering and the storage size of an SQL NULL value. */ UNIV_INLINE void dtype_read_for_order_and_null_size( /*===============================*/ dtype_t* type, /* in: type struct */ - byte* buf); /* in: buffer for type order info */ + byte* buf); /* in: buffer for the stored order info */ /************************************************************************* Validates a data type structure. */ diff --git a/innobase/include/data0type.ic b/innobase/include/data0type.ic index d82d976d076..ddd0b0ae8cc 100644 --- a/innobase/include/data0type.ic +++ b/innobase/include/data0type.ic @@ -110,7 +110,9 @@ dtype_get_pad_char( if (type->mtype == DATA_CHAR || type->mtype == DATA_VARCHAR || type->mtype == DATA_BINARY - || type->mtype == DATA_FIXBINARY) { + || type->mtype == DATA_FIXBINARY + || type->mtype == DATA_MYSQL + || type->mtype == DATA_VARMYSQL) { /* Space is the padding character for all char and binary strings */ @@ -124,39 +126,56 @@ dtype_get_pad_char( } /************************************************************************** -Stores to a type the information which determines its alphabetical -ordering. */ +Stores for a type the information which determines its alphabetical ordering +and the storage size of an SQL NULL value. */ UNIV_INLINE void dtype_store_for_order_and_null_size( /*================================*/ byte* buf, /* in: buffer for DATA_ORDER_NULL_TYPE_BUF_SIZE - bytes */ + bytes where we store the info */ dtype_t* type) /* in: type struct */ { ut_ad(4 == DATA_ORDER_NULL_TYPE_BUF_SIZE); buf[0] = (byte)(type->mtype & 0xFF); + + if (type->prtype & DATA_BINARY_TYPE) { + buf[0] = buf[0] | 128; + } + + if (type->prtype & DATA_NONLATIN1) { + buf[0] = buf[0] | 64; + } + buf[1] = (byte)(type->prtype & 0xFF); mach_write_to_2(buf + 2, type->len & 0xFFFF); } /************************************************************************** -Reads of a type the stored information which determines its alphabetical -ordering. */ +Reads to a type the stored information which determines its alphabetical +ordering and the storage size of an SQL NULL value. */ UNIV_INLINE void dtype_read_for_order_and_null_size( /*===============================*/ dtype_t* type, /* in: type struct */ - byte* buf) /* in: buffer for type order info */ + byte* buf) /* in: buffer for stored type order info */ { ut_ad(4 == DATA_ORDER_NULL_TYPE_BUF_SIZE); - type->mtype = buf[0]; + type->mtype = buf[0] & 63; type->prtype = buf[1]; + if (buf[0] & 128) { + type->prtype = type->prtype | DATA_BINARY_TYPE; + } + + if (buf[0] & 64) { + type->prtype = type->prtype | DATA_NONLATIN1; + } + type->len = mach_read_from_2(buf + 2); } diff --git a/innobase/include/db0err.h b/innobase/include/db0err.h index ab7d0caa35c..854b9794c00 100644 --- a/innobase/include/db0err.h +++ b/innobase/include/db0err.h @@ -44,8 +44,10 @@ Created 5/24/1996 Heikki Tuuri #define DB_CORRUPTION 39 /* data structure corruption noticed */ #define DB_COL_APPEARS_TWICE_IN_INDEX 40 /* InnoDB cannot handle an index where same column appears twice */ -#define DB_CANNOT_DROP_CONSTRAINT 40 /* dropping a foreign key constraint +#define DB_CANNOT_DROP_CONSTRAINT 41 /* dropping a foreign key constraint from a table failed */ +#define DB_NO_SAVEPOINT 42 /* no savepoint exists with the given + name */ /* The following are partial failure codes */ #define DB_FAIL 1000 diff --git a/innobase/include/dict0dict.h b/innobase/include/dict0dict.h index 97486a7c2f6..e88c6a52bcb 100644 --- a/innobase/include/dict0dict.h +++ b/innobase/include/dict0dict.h @@ -569,6 +569,19 @@ dict_index_get_nth_col_pos( dict_index_t* index, /* in: index */ ulint n); /* in: column number */ /************************************************************************ +Looks for a matching field in an index. The column and the prefix len has +to be the same. */ + +ulint +dict_index_get_nth_field_pos( +/*=========================*/ + /* out: position in internal representation + of the index; if not contained, returns + ULINT_UNDEFINED */ + dict_index_t* index, /* in: index from which to search */ + dict_index_t* index2, /* in: index */ + ulint n); /* in: field number in index2 */ +/************************************************************************ Looks for column n position in the clustered index. */ ulint diff --git a/innobase/include/dict0mem.h b/innobase/include/dict0mem.h index 0798541cfe0..03dc913a7c9 100644 --- a/innobase/include/dict0mem.h +++ b/innobase/include/dict0mem.h @@ -111,10 +111,13 @@ by the column name may be released only after publishing the index. */ void dict_mem_index_add_field( /*=====================*/ - dict_index_t* index, /* in: index */ - char* name, /* in: column name */ - ulint order); /* in: order criterion; 0 means an ascending - order */ + dict_index_t* index, /* in: index */ + char* name, /* in: column name */ + ulint order, /* in: order criterion; 0 means an + ascending order */ + ulint prefix_len); /* in: 0 or the column prefix length + in a MySQL index like + INDEX (textcol(25)) */ /************************************************************************** Frees an index memory object. */ @@ -158,12 +161,18 @@ struct dict_col_struct{ in some of the functions below */ }; +#define DICT_MAX_COL_PREFIX_LEN 512 + /* Data structure for a field in an index */ struct dict_field_struct{ - dict_col_t* col; /* pointer to the table column */ - char* name; /* name of the column */ - ulint order; /* flags for ordering this field: - DICT_DESCEND, ... */ + dict_col_t* col; /* pointer to the table column */ + char* name; /* name of the column */ + ulint order; /* flags for ordering this field: + DICT_DESCEND, ... */ + ulint prefix_len; /* 0 or the length of the column + prefix in a MySQL index of type, e.g., + INDEX (textcol(25)); must be smaller + than DICT_MAX_COL_PREFIX_LEN */ }; /* Data structure for an index tree */ diff --git a/innobase/include/fil0fil.h b/innobase/include/fil0fil.h index 23ef0304b2d..4f78fdb2fd7 100644 --- a/innobase/include/fil0fil.h +++ b/innobase/include/fil0fil.h @@ -43,7 +43,10 @@ struct fil_addr_struct{ extern fil_addr_t fil_addr_null; /* The byte offsets on a file page for various variables */ -#define FIL_PAGE_SPACE 0 /* space id the page belongs to */ +#define FIL_PAGE_SPACE_OR_CHKSUM 0 /* in < MySQL-4.0.14 space id the + page belongs to (== 0) but in later + versions the 'new' checksum of the + page */ #define FIL_PAGE_OFFSET 4 /* page offset inside space */ #define FIL_PAGE_PREV 8 /* if there is a 'natural' predecessor of the page, its offset */ @@ -64,7 +67,7 @@ extern fil_addr_t fil_addr_null; #define FIL_PAGE_DATA 38 /* start of the data on the page */ /* File page trailer */ -#define FIL_PAGE_END_LSN 8 /* the low 4 bytes of this are used +#define FIL_PAGE_END_LSN_OLD_CHKSUM 8 /* the low 4 bytes of this are used to store the page checksum, the last 4 bytes should be identical to the last 4 bytes of FIL_PAGE_LSN */ diff --git a/innobase/include/lock0lock.h b/innobase/include/lock0lock.h index d3b3d55d015..5608ba020b7 100644 --- a/innobase/include/lock0lock.h +++ b/innobase/include/lock0lock.h @@ -450,6 +450,18 @@ lock_rec_get_mutex_for_addr( ulint space, /* in: space id */ ulint page_no);/* in: page number */ /************************************************************************* +Checks that a transaction id is sensible, i.e., not in the future. */ + +ibool +lock_check_trx_id_sanity( +/*=====================*/ + /* out: TRUE if ok */ + dulint trx_id, /* in: trx id */ + rec_t* rec, /* in: user record */ + dict_index_t* index, /* in: clustered index */ + ibool has_kernel_mutex);/* in: TRUE if the caller owns the + kernel mutex */ +/************************************************************************* Validates the lock queue on a single record. */ ibool diff --git a/innobase/include/os0file.h b/innobase/include/os0file.h index 86f27a2d3eb..1ec8b71d069 100644 --- a/innobase/include/os0file.h +++ b/innobase/include/os0file.h @@ -146,6 +146,21 @@ os_file_create_simple( ulint access_type,/* in: OS_FILE_READ_ONLY or OS_FILE_READ_WRITE */ ibool* success);/* out: TRUE if succeed, FALSE if error */ /******************************************************************** +A simple function to open or create a file. */ + +os_file_t +os_file_create_simple_no_error_handling( +/*====================================*/ + /* out, own: handle to the file, not defined if error, + error number can be retrieved with os_get_last_error */ + char* name, /* in: name of the file or path as a null-terminated + string */ + ulint create_mode,/* in: OS_FILE_OPEN if an existing file is opened + (if does not exist, error), or OS_FILE_CREATE if a new + file is created (if exists, error) */ + ulint access_type,/* in: OS_FILE_READ_ONLY or OS_FILE_READ_WRITE */ + ibool* success);/* out: TRUE if succeed, FALSE if error */ +/******************************************************************** Opens an existing file or creates a new. */ os_file_t @@ -173,6 +188,14 @@ os_file_close( /* out: TRUE if success */ os_file_t file); /* in, own: handle to a file */ /*************************************************************************** +Closes a file handle. */ + +ibool +os_file_close_no_error_handling( +/*============================*/ + /* out: TRUE if success */ + os_file_t file); /* in, own: handle to a file */ +/*************************************************************************** Gets a file size. */ ibool diff --git a/innobase/include/page0page.h b/innobase/include/page0page.h index b5e33af5bc0..04f771c3abd 100644 --- a/innobase/include/page0page.h +++ b/innobase/include/page0page.h @@ -666,6 +666,15 @@ page_rec_validate( /* out: TRUE if ok */ rec_t* rec); /* in: record on the page */ /******************************************************************* +Checks that the first directory slot points to the infimum record and +the last to the supremum. This function is intended to track if the +bug fixed in 4.0.14 has caused corruption to users' databases. */ + +void +page_check_dir( +/*===========*/ + page_t* page); /* in: index page */ +/******************************************************************* This function checks the consistency of an index page when we do not know the index. This is also resilient so that this should never crash even if the page is total garbage. */ diff --git a/innobase/include/rem0cmp.h b/innobase/include/rem0cmp.h index 6f2a99fc8c2..712e263350e 100644 --- a/innobase/include/rem0cmp.h +++ b/innobase/include/rem0cmp.h @@ -42,6 +42,22 @@ cmp_data_data( buffer) */ ulint len2); /* in: data field length or UNIV_SQL_NULL */ /***************************************************************** +This function is used to compare two data fields for which we know the +data type. */ + +int +cmp_data_data_slow( +/*===============*/ + /* out: 1, 0, -1, if data1 is greater, equal, + less than data2, respectively */ + dtype_t* cur_type,/* in: data type of the fields */ + byte* data1, /* in: data field (== a pointer to a memory + buffer) */ + ulint len1, /* in: data field length or UNIV_SQL_NULL */ + byte* data2, /* in: data field (== a pointer to a memory + buffer) */ + ulint len2); /* in: data field length or UNIV_SQL_NULL */ +/***************************************************************** This function is used to compare two dfields where at least the first has its data type field set. */ UNIV_INLINE diff --git a/innobase/include/row0mysql.ic b/innobase/include/row0mysql.ic index e9d493da8b5..4ecd66e06ec 100644 --- a/innobase/include/row0mysql.ic +++ b/innobase/include/row0mysql.ic @@ -58,7 +58,8 @@ row_mysql_store_col_in_innobase_format( /*===================================*/ dfield_t* dfield, /* in/out: dfield */ byte* buf, /* in/out: buffer for the converted - value */ + value; this must be at least col_len + long! */ byte* mysql_data, /* in: MySQL column value, not SQL NULL; NOTE that dfield may also get a pointer to mysql_data, @@ -96,7 +97,6 @@ row_mysql_store_col_in_innobase_format( while (col_len > 0 && ptr[col_len - 1] == ' ') { col_len--; } - } else if (type == DATA_BLOB) { ptr = row_mysql_read_blob_ref(&col_len, mysql_data, col_len); } diff --git a/innobase/include/row0row.h b/innobase/include/row0row.h index 09a79e19fd7..d1befbbbad3 100644 --- a/innobase/include/row0row.h +++ b/innobase/include/row0row.h @@ -86,9 +86,10 @@ dtuple_t* row_build( /*======*/ /* out, own: row built; see the NOTE below! */ - ulint type, /* in: ROW_COPY_DATA, or ROW_COPY_POINTERS: - the former copies also the data fields to - heap as the latter only places pointers to + ulint type, /* in: ROW_COPY_POINTERS, ROW_COPY_DATA, or + ROW_COPY_ALSO_EXTERNALS, + the two last copy also the data fields to + heap as the first only places pointers to data fields on the index page, and thus is more efficient */ dict_index_t* index, /* in: clustered index */ diff --git a/innobase/include/row0sel.h b/innobase/include/row0sel.h index aa2da6fe5f6..5ef7ff9399a 100644 --- a/innobase/include/row0sel.h +++ b/innobase/include/row0sel.h @@ -87,9 +87,11 @@ row_printf_step( /* out: query thread to run next or NULL */ que_thr_t* thr); /* in: query thread */ /******************************************************************** -Converts a key value stored in MySQL format to an Innobase dtuple. -The last field of the key value may be just a prefix of a fixed length -field: hence the parameter key_len. */ +Converts a key value stored in MySQL format to an Innobase dtuple. The last +field of the key value may be just a prefix of a fixed length field: hence +the parameter key_len. But currently we do not allow search keys where the +last field is only a prefix of the full key field len and print a warning if +such appears. */ void row_sel_convert_mysql_key_to_innobase( @@ -100,6 +102,7 @@ row_sel_convert_mysql_key_to_innobase( to index! */ byte* buf, /* in: buffer to use in field conversions */ + ulint buf_len, /* in: buffer length */ dict_index_t* index, /* in: index of the key value */ byte* key_ptr, /* in: MySQL key value */ ulint key_len); /* in: MySQL key value length */ diff --git a/innobase/include/row0upd.h b/innobase/include/row0upd.h index 273ec6074eb..473c55c7ef9 100644 --- a/innobase/include/row0upd.h +++ b/innobase/include/row0upd.h @@ -114,13 +114,15 @@ row_upd_index_write_log( closed within this function */ mtr_t* mtr); /* in: mtr into whose log to write */ /*************************************************************** -Returns TRUE if row update changes size of some field in index. */ +Returns TRUE if row update changes size of some field in index or if some +field to be updated is stored externally in rec or update. */ ibool -row_upd_changes_field_size( -/*=======================*/ +row_upd_changes_field_size_or_external( +/*===================================*/ /* out: TRUE if the update changes the size of - some field in index */ + some field in index or the field is external + in rec or update */ rec_t* rec, /* in: record in clustered index */ dict_index_t* index, /* in: clustered index */ upd_t* update);/* in: update vector */ @@ -175,16 +177,10 @@ row_upd_index_replace_new_col_vals( dtuple_t* entry, /* in/out: index entry where replaced */ dict_index_t* index, /* in: index; NOTE that may also be a non-clustered index */ - upd_t* update); /* in: update vector */ -/*************************************************************** -Replaces the new column values stored in the update vector to the -clustered index entry given. */ - -void -row_upd_clust_index_replace_new_col_vals( -/*=====================================*/ - dtuple_t* entry, /* in/out: index entry where replaced */ - upd_t* update); /* in: update vector */ + upd_t* update, /* in: update vector */ + mem_heap_t* heap); /* in: memory heap to which we allocate and + copy the new values, set this as NULL if you + do not want allocation */ /*************************************************************** Checks if an update vector changes an ordering field of an index record. This function is fast if the update vector is short or the number of ordering @@ -358,9 +354,9 @@ struct upd_node_struct{ externally in the clustered index record of row */ ulint n_ext_vec;/* number of fields in ext_vec */ - mem_heap_t* heap; /* memory heap used as auxiliary storage for - row; this must be emptied after a successful - update if node->row != NULL */ + mem_heap_t* heap; /* memory heap used as auxiliary storage; + this must be emptied after a successful + update */ /*----------------------*/ sym_node_t* table_sym;/* table node in symbol table */ que_node_t* col_assign_list; diff --git a/innobase/include/srv0srv.h b/innobase/include/srv0srv.h index 24e692dedab..1e54d7bfc35 100644 --- a/innobase/include/srv0srv.h +++ b/innobase/include/srv0srv.h @@ -153,6 +153,7 @@ extern mutex_t* kernel_mutex_temp;/* mutex protecting the server, trx structs, /* Array of English strings describing the current state of an i/o handler thread */ extern char* srv_io_thread_op_info[]; +extern char* srv_io_thread_function[]; typedef struct srv_sys_struct srv_sys_t; diff --git a/innobase/include/trx0roll.h b/innobase/include/trx0roll.h index 820af4cd014..0d7126c9c57 100644 --- a/innobase/include/trx0roll.h +++ b/innobase/include/trx0roll.h @@ -177,6 +177,55 @@ trx_general_rollback_for_mysql( ibool partial,/* in: TRUE if partial rollback requested */ trx_savept_t* savept);/* in: pointer to savepoint undo number, if partial rollback requested */ +/*********************************************************************** +Rolls back a transaction back to a named savepoint. Modifications after the +savepoint are undone but InnoDB does NOT release the corresponding locks +which are stored in memory. If a lock is 'implicit', that is, a new inserted +row holds a lock where the lock information is carried by the trx id stored in +the row, these locks are naturally released in the rollback. Savepoints which +were set after this savepoint are deleted. */ + +ulint +trx_rollback_to_savepoint_for_mysql( +/*================================*/ + /* out: if no savepoint + of the name found then + DB_NO_SAVEPOINT, + otherwise DB_SUCCESS */ + trx_t* trx, /* in: transaction handle */ + char* savepoint_name, /* in: savepoint name */ + ib_longlong* mysql_binlog_cache_pos);/* out: the MySQL binlog cache + position corresponding to this + savepoint; MySQL needs this + information to remove the + binlog entries of the queries + executed after the savepoint */ +/*********************************************************************** +Creates a named savepoint. If the transaction is not yet started, starts it. +If there is already a savepoint of the same name, this call erases that old +savepoint and replaces it with a new. Savepoints are deleted in a transaction +commit or rollback. */ + +ulint +trx_savepoint_for_mysql( +/*====================*/ + /* out: always DB_SUCCESS */ + trx_t* trx, /* in: transaction handle */ + char* savepoint_name, /* in: savepoint name */ + ib_longlong binlog_cache_pos); /* in: MySQL binlog cache + position corresponding to this + connection at the time of the + savepoint */ +/*********************************************************************** +Frees savepoint structs. */ + +void +trx_roll_savepoints_free( +/*=====================*/ + trx_t* trx, /* in: transaction handle */ + trx_named_savept_t* savep); /* in: free all savepoints > this one; + if this is NULL, free all savepoints + of trx */ extern sess_t* trx_dummy_sess; @@ -207,6 +256,21 @@ struct roll_node_struct{ case of a partial rollback */ }; +/* A savepoint set with SQL's "SAVEPOINT savepoint_id" command */ +struct trx_named_savept_struct{ + char* name; /* savepoint name */ + trx_savept_t savept; /* the undo number corresponding to + the savepoint */ + ib_longlong mysql_binlog_cache_pos; + /* the MySQL binlog cache position + corresponding to this savepoint, not + defined if the MySQL binlogging is not + enabled */ + UT_LIST_NODE_T(trx_named_savept_t) + trx_savepoints; /* the list of savepoints of a + transaction */ +}; + /* Rollback node states */ #define ROLL_NODE_SEND 1 #define ROLL_NODE_WAIT 2 diff --git a/innobase/include/trx0sys.ic b/innobase/include/trx0sys.ic index ada2d8cb19c..343e6d7c2fa 100644 --- a/innobase/include/trx0sys.ic +++ b/innobase/include/trx0sys.ic @@ -296,6 +296,16 @@ trx_is_active( return(FALSE); } + if (ut_dulint_cmp(trx_id, trx_sys->max_trx_id) >= 0) { + + /* There must be corruption: we return TRUE because this + function is only called by lock_clust_rec_some_has_impl() + and row_vers_impl_x_locked_off_kernel() and they have + diagnostic prints in this case */ + + return(TRUE); + } + trx = trx_get_on_id(trx_id); if (trx && (trx->conc_state == TRX_ACTIVE)) { diff --git a/innobase/include/trx0trx.h b/innobase/include/trx0trx.h index 39229923375..6b08b674db8 100644 --- a/innobase/include/trx0trx.h +++ b/innobase/include/trx0trx.h @@ -381,7 +381,8 @@ struct trx_struct{ replication slave, we have here the master binlog name up to which replication has processed; otherwise - this is a pointer to a null character */ + this is a pointer to a null + character */ ib_longlong mysql_master_log_pos; /* if the database server is a MySQL replication slave, this is the @@ -501,6 +502,10 @@ struct trx_struct{ mem_heap_t* read_view_heap; /* memory heap for the read view */ read_view_t* read_view; /* consistent read view or NULL */ /*------------------------------*/ + UT_LIST_BASE_NODE_T(trx_named_savept_t) + trx_savepoints; /* savepoints set with SAVEPOINT ..., + oldest first */ + /*------------------------------*/ mutex_t undo_mutex; /* mutex protecting the fields in this section (down to undo_no_arr), EXCEPT last_sql_stat_start, which can be diff --git a/innobase/include/trx0types.h b/innobase/include/trx0types.h index b8befe7172f..2965eb4451f 100644 --- a/innobase/include/trx0types.h +++ b/innobase/include/trx0types.h @@ -24,6 +24,7 @@ typedef struct trx_undo_inf_struct trx_undo_inf_t; typedef struct trx_purge_struct trx_purge_t; typedef struct roll_node_struct roll_node_t; typedef struct commit_node_struct commit_node_t; +typedef struct trx_named_savept_struct trx_named_savept_t; /* Transaction savepoint */ typedef struct trx_savept_struct trx_savept_t; diff --git a/innobase/include/ut0dbg.h b/innobase/include/ut0dbg.h index e99dc8c09d6..802557099fc 100644 --- a/innobase/include/ut0dbg.h +++ b/innobase/include/ut0dbg.h @@ -50,6 +50,37 @@ extern ulint* ut_dbg_null_ptr; }\ } +/* This can be used if there are % characters in the assertion formula: +if we try to printf the formula gcc would complain of illegal print +format characters */ +#define ut_anp(EXPR)\ +{\ + ulint dbg_i;\ +\ + if (!((ulint)(EXPR) + ut_dbg_zero)) {\ + ut_print_timestamp(stderr);\ + fprintf(stderr,\ + " InnoDB: Assertion failure in thread %lu in file %s line %lu\n",\ + os_thread_pf(os_thread_get_curr_id()), IB__FILE__,\ + (ulint)__LINE__);\ + fprintf(stderr,\ + "\nInnoDB: We intentionally generate a memory trap.\n");\ + fprintf(stderr,\ + "InnoDB: Send a detailed bug report to mysql@lists.mysql.com\n");\ + ut_dbg_stop_threads = TRUE;\ + dbg_i = *(ut_dbg_null_ptr);\ + if (dbg_i) {\ + ut_dbg_null_ptr = NULL;\ + }\ + }\ + if (ut_dbg_stop_threads) {\ + fprintf(stderr,\ + "InnoDB: Thread %lu stopped in file %s line %lu\n",\ + os_thread_pf(os_thread_get_curr_id()), IB__FILE__, (ulint)__LINE__);\ + os_thread_sleep(1000000000);\ + }\ +} + #define ut_error {\ ulint dbg_i;\ ut_print_timestamp(stderr);\ diff --git a/innobase/include/ut0mem.h b/innobase/include/ut0mem.h index 09e0d800685..4e8566eba1b 100644 --- a/innobase/include/ut0mem.h +++ b/innobase/include/ut0mem.h @@ -57,7 +57,7 @@ ut_free( /*====*/ void* ptr); /* in, own: memory block */ /************************************************************************** -Frees all allocated memory not freed yet. */ +Frees in shutdown all allocated memory not freed yet. */ void ut_free_all_mem(void); diff --git a/innobase/lock/lock0lock.c b/innobase/lock/lock0lock.c index 4bb1d243ed4..fecb1f95c68 100644 --- a/innobase/lock/lock0lock.c +++ b/innobase/lock/lock0lock.c @@ -356,7 +356,7 @@ lock_mutex_enter_kernel(void) } /************************************************************************* -Releses the kernel mutex. This function is used in this module to allow +Releases the kernel mutex. This function is used in this module to allow monitoring the contention degree on the kernel mutex caused by the lock operations. */ UNIV_INLINE @@ -514,6 +514,53 @@ lock_rec_mutex_own_all(void) #endif +/************************************************************************* +Checks that a transaction id is sensible, i.e., not in the future. */ + +ibool +lock_check_trx_id_sanity( +/*=====================*/ + /* out: TRUE if ok */ + dulint trx_id, /* in: trx id */ + rec_t* rec, /* in: user record */ + dict_index_t* index, /* in: clustered index */ + ibool has_kernel_mutex)/* in: TRUE if the caller owns the + kernel mutex */ +{ + char err_buf[500]; + ibool is_ok = TRUE; + + if (!has_kernel_mutex) { + mutex_enter(&kernel_mutex); + } + + /* A sanity check: the trx_id in rec must be smaller than the global + trx id counter */ + + if (ut_dulint_cmp(trx_id, trx_sys->max_trx_id) >= 0) { + rec_sprintf(err_buf, 400, rec); + ut_print_timestamp(stderr); + fprintf(stderr, +"InnoDB: Error: transaction id associated with record\n%s\n" +"InnoDB: in table %s index %s\n" +"InnoDB: is %lu %lu which is higher than the global trx id counter %lu %lu!\n" +"InnoDB: The table is corrupt. You have to do dump + drop + reimport.\n", + err_buf, index->table_name, index->name, + ut_dulint_get_high(trx_id), + ut_dulint_get_low(trx_id), + ut_dulint_get_high(trx_sys->max_trx_id), + ut_dulint_get_low(trx_sys->max_trx_id)); + + is_ok = FALSE; + } + + if (!has_kernel_mutex) { + mutex_exit(&kernel_mutex); + } + + return(is_ok); +} + /************************************************************************* Checks that a record is seen in a consistent read. */ @@ -539,6 +586,15 @@ lock_clust_rec_cons_read_sees( return(TRUE); } + if (!lock_check_trx_id_sanity(trx_id, rec, index, FALSE)) { + /* Trying to get the 'history' of a corrupt record is bound + to fail: let us try to use the record itself in the query */ + fprintf(stderr, +"InnoDB: We try to access the corrupt record in the query anyway.\n"); + + return(TRUE); + } + return(FALSE); } @@ -562,7 +618,9 @@ lock_sec_rec_cons_read_sees( read_view_t* view) /* in: consistent read view */ { dulint max_trx_id; - + + UT_NOT_USED(index); + ut_ad(!(index->type & DICT_CLUSTERED)); ut_ad(page_rec_is_user_rec(rec)); @@ -575,6 +633,16 @@ lock_sec_rec_cons_read_sees( if (ut_dulint_cmp(max_trx_id, view->up_limit_id) >= 0) { + if (!lock_check_trx_id_sanity(max_trx_id, rec, index, FALSE)) { + /* Trying to get the 'history' of a corrupt record is + bound to fail: let us try to use the record itself in + the query */ + fprintf(stderr, +"InnoDB: We try to access the corrupt record in the query anyway.\n"); + + return(TRUE); + } + return(FALSE); } @@ -1569,6 +1637,15 @@ lock_sec_rec_some_has_impl_off_kernel( /* Ok, in this case it is possible that some transaction has an implicit x-lock. We have to look in the clustered index. */ + if (!lock_check_trx_id_sanity(page_get_max_trx_id(page), rec, index, + TRUE)) { + buf_page_print(page); + + /* The page is corrupt: try to avoid a crash by returning + NULL */ + return(NULL); + } + return(row_vers_impl_x_locked_off_kernel(rec, index)); } @@ -2565,7 +2642,7 @@ lock_move_rec_list_start( ulint heap_no; ulint type_mode; - ut_ad(new_page); + ut_a(new_page); lock_mutex_enter_kernel(); @@ -3028,7 +3105,7 @@ lock_deadlock_recursive( we return LOCK_VICTIM_IS_START */ { lock_t* lock; - ulint bit_no; + ulint bit_no = ULINT_UNDEFINED; trx_t* lock_trx; char* err_buf; ulint ret; @@ -3067,6 +3144,7 @@ lock_deadlock_recursive( lock = UT_LIST_GET_PREV(un_member.tab_lock.locks, lock); } else { ut_ad(lock_get_type(lock) == LOCK_REC); + ut_a(bit_no != ULINT_UNDEFINED); lock = lock_rec_get_prev(lock, bit_no); } diff --git a/innobase/log/log0log.c b/innobase/log/log0log.c index e15812e03af..b5ce1a3d97b 100644 --- a/innobase/log/log0log.c +++ b/innobase/log/log0log.c @@ -375,7 +375,7 @@ log_pad_current_log_block(void) log_close(); log_release(); - ut_ad((ut_dulint_get_low(lsn) % OS_FILE_LOG_BLOCK_SIZE) + ut_anp((ut_dulint_get_low(lsn) % OS_FILE_LOG_BLOCK_SIZE) == LOG_BLOCK_HDR_SIZE); } @@ -1070,8 +1070,8 @@ log_group_write_buf( ulint i; ut_ad(mutex_own(&(log_sys->mutex))); - ut_ad(len % OS_FILE_LOG_BLOCK_SIZE == 0); - ut_ad(ut_dulint_get_low(start_lsn) % OS_FILE_LOG_BLOCK_SIZE == 0); + ut_anp(len % OS_FILE_LOG_BLOCK_SIZE == 0); + ut_anp(ut_dulint_get_low(start_lsn) % OS_FILE_LOG_BLOCK_SIZE == 0); if (new_data_offset == 0) { write_header = TRUE; @@ -2123,11 +2123,11 @@ log_group_archive( start_lsn = log_sys->archived_lsn; - ut_ad(ut_dulint_get_low(start_lsn) % OS_FILE_LOG_BLOCK_SIZE == 0); + ut_anp(ut_dulint_get_low(start_lsn) % OS_FILE_LOG_BLOCK_SIZE == 0); end_lsn = log_sys->next_archived_lsn; - ut_ad(ut_dulint_get_low(end_lsn) % OS_FILE_LOG_BLOCK_SIZE == 0); + ut_anp(ut_dulint_get_low(end_lsn) % OS_FILE_LOG_BLOCK_SIZE == 0); buf = log_sys->archive_buf; @@ -2234,7 +2234,7 @@ loop: group->next_archived_file_no = group->archived_file_no + n_files; group->next_archived_offset = next_offset % group->file_size; - ut_ad(group->next_archived_offset % OS_FILE_LOG_BLOCK_SIZE == 0); + ut_anp(group->next_archived_offset % OS_FILE_LOG_BLOCK_SIZE == 0); } /********************************************************* @@ -2429,8 +2429,8 @@ loop: start_lsn = log_sys->archived_lsn; if (calc_new_limit) { - ut_ad(log_sys->archive_buf_size % OS_FILE_LOG_BLOCK_SIZE == 0); - + ut_anp(log_sys->archive_buf_size % OS_FILE_LOG_BLOCK_SIZE + == 0); limit_lsn = ut_dulint_add(start_lsn, log_sys->archive_buf_size); @@ -2916,6 +2916,7 @@ loop: mutex_enter(&kernel_mutex); + /* Check that there are no longer transactions */ if (trx_n_mysql_transactions > 0 || UT_LIST_GET_LEN(trx_sys->trx_list) > 0) { @@ -2924,6 +2925,8 @@ loop: goto loop; } + /* Check that the master thread is suspended */ + if (srv_n_threads_active[SRV_MASTER] != 0) { mutex_exit(&kernel_mutex); @@ -2952,7 +2955,6 @@ loop: } log_archive_all(); - log_make_checkpoint_at(ut_dulint_max, TRUE); mutex_enter(&(log_sys->mutex)); @@ -2961,8 +2963,9 @@ loop: if (ut_dulint_cmp(lsn, log_sys->last_checkpoint_lsn) != 0 || (srv_log_archive_on - && ut_dulint_cmp(lsn, - ut_dulint_add(log_sys->archived_lsn, LOG_BLOCK_HDR_SIZE)) != 0)) { + && ut_dulint_cmp(lsn, + ut_dulint_add(log_sys->archived_lsn, LOG_BLOCK_HDR_SIZE)) + != 0)) { mutex_exit(&(log_sys->mutex)); @@ -2981,10 +2984,22 @@ loop: mutex_exit(&(log_sys->mutex)); + mutex_enter(&kernel_mutex); + /* Check that the master thread has stayed suspended */ + if (srv_n_threads_active[SRV_MASTER] != 0) { + fprintf(stderr, +"InnoDB: Warning: the master thread woke up during shutdown\n"); + + mutex_exit(&kernel_mutex); + + goto loop; + } + mutex_exit(&kernel_mutex); + fil_flush_file_spaces(FIL_TABLESPACE); fil_flush_file_spaces(FIL_LOG); - /* The following fil_write_... will pass the buffer pool: therefore + /* The next fil_write_... will pass the buffer pool: therefore it is essential that the buffer pool has been completely flushed to disk! */ @@ -2993,12 +3008,14 @@ loop: goto loop; } + /* The lock timeout thread should now have exited */ + if (srv_lock_timeout_and_monitor_active) { goto loop; } - /* We now suspend also the InnoDB error monitor thread */ + /* We now let also the InnoDB error monitor thread to exit */ srv_shutdown_state = SRV_SHUTDOWN_LAST_PHASE; @@ -3008,6 +3025,7 @@ loop: } /* Make some checks that the server really is quiet */ + ut_a(srv_n_threads_active[SRV_MASTER] == 0); ut_a(buf_all_freed()); ut_a(0 == ut_dulint_cmp(lsn, log_sys->lsn)); @@ -3016,6 +3034,7 @@ loop: fil_flush_file_spaces(FIL_TABLESPACE); /* Make some checks that the server really is quiet */ + ut_a(srv_n_threads_active[SRV_MASTER] == 0); ut_a(buf_all_freed()); ut_a(0 == ut_dulint_cmp(lsn, log_sys->lsn)); } diff --git a/innobase/log/log0recv.c b/innobase/log/log0recv.c index 47833214d15..8e5fe819afb 100644 --- a/innobase/log/log0recv.c +++ b/innobase/log/log0recv.c @@ -973,7 +973,7 @@ recv_recover_page( ulint space, /* in: space id */ ulint page_no) /* in: page number */ { - buf_block_t* block; + buf_block_t* block = NULL; recv_addr_t* recv_addr; recv_t* recv; byte* buf; @@ -1085,7 +1085,7 @@ recv_recover_page( page_lsn = page_newest_lsn; mach_write_to_8(page + UNIV_PAGE_SIZE - - FIL_PAGE_END_LSN, ut_dulint_zero); + - FIL_PAGE_END_LSN_OLD_CHKSUM, ut_dulint_zero); mach_write_to_8(page + FIL_PAGE_LSN, ut_dulint_zero); } @@ -1107,7 +1107,7 @@ recv_recover_page( recv_parse_or_apply_log_rec_body(recv->type, buf, buf + recv->len, page, &mtr); mach_write_to_8(page + UNIV_PAGE_SIZE - - FIL_PAGE_END_LSN, + - FIL_PAGE_END_LSN_OLD_CHKSUM, ut_dulint_add(recv->start_lsn, recv->len)); mach_write_to_8(page + FIL_PAGE_LSN, @@ -1132,6 +1132,8 @@ recv_recover_page( mutex_exit(&(recv_sys->mutex)); if (!recover_backup && modification_to_page) { + ut_a(block); + buf_flush_recv_note_modification(block, start_lsn, end_lsn); } @@ -1339,6 +1341,7 @@ loop: mutex_exit(&(recv_sys->mutex)); } +#ifdef UNIV_HOTBACKUP /*********************************************************************** Applies log records in the hash table to a backup. */ @@ -1520,8 +1523,8 @@ recv_check_identical( for (i = 0; i < len; i++) { if (str1[i] != str2[i]) { - fprintf(stderr, "Strings do not match at offset %lu\n", i); - + fprintf(stderr, + "Strings do not match at offset %lu\n", i); ut_print_buf(str1 + i, 16); fprintf(stderr, "\n"); ut_print_buf(str2 + i, 16); @@ -1654,6 +1657,7 @@ recv_compare_spaces_low( recv_compare_spaces(space1, space2, n_pages); } +#endif /*********************************************************************** Tries to parse a single log record and returns its length. */ diff --git a/innobase/mem/mem0pool.c b/innobase/mem/mem0pool.c index 382e505b63f..b004a8c4df7 100644 --- a/innobase/mem/mem0pool.c +++ b/innobase/mem/mem0pool.c @@ -99,6 +99,12 @@ mem_pool_t* mem_comm_pool = NULL; ulint mem_out_of_mem_err_msg_count = 0; +/* We use this counter to check that the mem pool mutex does not leak; +this is to track a strange assertion failure reported at +mysql@lists.mysql.com */ + +ulint mem_n_threads_inside = 0; + /************************************************************************ Reserves the mem pool mutex. */ @@ -328,6 +334,9 @@ mem_area_alloc( n = ut_2_log(ut_max(size + MEM_AREA_EXTRA_SIZE, MEM_AREA_MIN_SIZE)); mutex_enter(&(pool->mutex)); + mem_n_threads_inside++; + + ut_a(mem_n_threads_inside == 1); area = UT_LIST_GET_FIRST(pool->free_list[n]); @@ -338,6 +347,7 @@ mem_area_alloc( /* Out of memory in memory pool: we try to allocate from the operating system with the regular malloc: */ + mem_n_threads_inside--; mutex_exit(&(pool->mutex)); return(ut_malloc(size)); @@ -353,6 +363,16 @@ mem_area_alloc( n); mem_analyze_corruption((byte*)area); + + /* Try to analyze a strange assertion failure reported at + mysql@lists.mysql.com where the free bit IS 1 in the + hex dump above */ + + if (mem_area_get_free(area)) { + fprintf(stderr, +"InnoDB: Probably a race condition because now the area is marked free!\n"); + } + ut_a(0); } @@ -374,6 +394,7 @@ mem_area_alloc( pool->reserved += mem_area_get_size(area); + mem_n_threads_inside--; mutex_exit(&(pool->mutex)); ut_ad(mem_pool_validate(pool)); @@ -495,6 +516,9 @@ mem_area_free( n = ut_2_log(size); mutex_enter(&(pool->mutex)); + mem_n_threads_inside++; + + ut_a(mem_n_threads_inside == 1); if (buddy && mem_area_get_free(buddy) && (size == mem_area_get_size(buddy))) { @@ -518,6 +542,7 @@ mem_area_free( pool->reserved += ut_2_exp(n); + mem_n_threads_inside--; mutex_exit(&(pool->mutex)); mem_area_free(new_ptr, pool); @@ -533,6 +558,7 @@ mem_area_free( pool->reserved -= size; } + mem_n_threads_inside--; mutex_exit(&(pool->mutex)); ut_ad(mem_pool_validate(pool)); @@ -577,7 +603,7 @@ mem_pool_validate( } } - ut_a(free + pool->reserved == pool->size + ut_anp(free + pool->reserved == pool->size - (pool->size % MEM_AREA_MIN_SIZE)); mutex_exit(&(pool->mutex)); diff --git a/innobase/os/os0file.c b/innobase/os/os0file.c index 2f32b9347dc..612bd534fd1 100644 --- a/innobase/os/os0file.c +++ b/innobase/os/os0file.c @@ -60,6 +60,7 @@ struct os_aio_slot_struct{ ulint pos; /* index of the slot in the aio array */ ibool reserved; /* TRUE if this slot is reserved */ + time_t reservation_time;/* time when reserved */ ulint len; /* length of the block to read or write */ byte* buf; /* buffer used in i/o */ @@ -147,6 +148,12 @@ time_t os_last_printout; ibool os_has_said_disk_full = FALSE; +/* The mutex protecting the following counts of pending pread and pwrite +operations */ +os_mutex_t os_file_count_mutex; +ulint os_file_n_pending_preads = 0; +ulint os_file_n_pending_pwrites = 0; + /*************************************************************************** Gets the operating system version. Currently works only on Windows. */ @@ -364,6 +371,8 @@ os_io_init_simple(void) { ulint i; + os_file_count_mutex = os_mutex_create(NULL); + for (i = 0; i < OS_FILE_N_SEEK_MUTEXES; i++) { os_file_seek_mutexes[i] = os_mutex_create(NULL); } @@ -415,9 +424,8 @@ try_again: file = CreateFile(name, access, - FILE_SHARE_READ | FILE_SHARE_WRITE, - /* file can be read and written - also by other processes */ + FILE_SHARE_READ,/* file can be read also by other + processes */ NULL, /* default security attributes */ create_flag, attributes, @@ -481,6 +489,101 @@ try_again: return(file); #endif } + +/******************************************************************** +A simple function to open or create a file. */ + +os_file_t +os_file_create_simple_no_error_handling( +/*====================================*/ + /* out, own: handle to the file, not defined if error, + error number can be retrieved with os_get_last_error */ + char* name, /* in: name of the file or path as a null-terminated + string */ + ulint create_mode,/* in: OS_FILE_OPEN if an existing file is opened + (if does not exist, error), or OS_FILE_CREATE if a new + file is created (if exists, error) */ + ulint access_type,/* in: OS_FILE_READ_ONLY or OS_FILE_READ_WRITE */ + ibool* success)/* out: TRUE if succeed, FALSE if error */ +{ +#ifdef __WIN__ + os_file_t file; + DWORD create_flag; + DWORD access; + DWORD attributes = 0; + + ut_a(name); + + if (create_mode == OS_FILE_OPEN) { + create_flag = OPEN_EXISTING; + } else if (create_mode == OS_FILE_CREATE) { + create_flag = CREATE_NEW; + } else { + create_flag = 0; + ut_error; + } + + if (access_type == OS_FILE_READ_ONLY) { + access = GENERIC_READ; + } else if (access_type == OS_FILE_READ_WRITE) { + access = GENERIC_READ | GENERIC_WRITE; + } else { + access = 0; + ut_error; + } + + file = CreateFile(name, + access, + FILE_SHARE_READ,/* file can be read also by other + processes */ + NULL, /* default security attributes */ + create_flag, + attributes, + NULL); /* no template file */ + + if (file == INVALID_HANDLE_VALUE) { + *success = FALSE; + } else { + *success = TRUE; + } + + return(file); +#else + os_file_t file; + int create_flag; + + ut_a(name); + + if (create_mode == OS_FILE_OPEN) { + if (access_type == OS_FILE_READ_ONLY) { + create_flag = O_RDONLY; + } else { + create_flag = O_RDWR; + } + } else if (create_mode == OS_FILE_CREATE) { + create_flag = O_RDWR | O_CREAT | O_EXCL; + } else { + create_flag = 0; + ut_error; + } + + if (create_mode == OS_FILE_CREATE) { + file = open(name, create_flag, S_IRUSR | S_IWUSR + | S_IRGRP | S_IWGRP); + } else { + file = open(name, create_flag); + } + + if (file == -1) { + *success = FALSE; + } else { + *success = TRUE; + } + + return(file); +#endif +} + /******************************************************************** Opens an existing file or creates a new. */ @@ -566,9 +669,14 @@ try_again: file = CreateFile(name, GENERIC_READ | GENERIC_WRITE, /* read and write access */ - FILE_SHARE_READ | FILE_SHARE_WRITE, - /* file can be read and written - also by other processes */ + FILE_SHARE_READ,/* File can be read also by other + processes; we must give the read + permission because of ibbackup. We do + not give the write permission to + others because if one would succeed to + start 2 instances of mysqld on the + SAME files, that could cause severe + database corruption! */ NULL, /* default security attributes */ create_flag, attributes, @@ -676,6 +784,41 @@ os_file_close( #endif } +/*************************************************************************** +Closes a file handle. */ + +ibool +os_file_close_no_error_handling( +/*============================*/ + /* out: TRUE if success */ + os_file_t file) /* in, own: handle to a file */ +{ +#ifdef __WIN__ + BOOL ret; + + ut_a(file); + + ret = CloseHandle(file); + + if (ret) { + return(TRUE); + } + + return(FALSE); +#else + int ret; + + ret = close(file); + + if (ret == -1) { + + return(FALSE); + } + + return(TRUE); +#endif +} + /*************************************************************************** Gets a file size. */ @@ -896,6 +1039,7 @@ os_file_pread( offset */ { off_t offs; + ssize_t n_bytes; ut_a((offset & 0xFFFFFFFF) == offset); @@ -917,7 +1061,17 @@ os_file_pread( os_n_file_reads++; #ifdef HAVE_PREAD - return(pread(file, buf, n, offs)); + os_mutex_enter(os_file_count_mutex); + os_file_n_pending_preads++; + os_mutex_exit(os_file_count_mutex); + + n_bytes = pread(file, buf, n, offs); + + os_mutex_enter(os_file_count_mutex); + os_file_n_pending_preads--; + os_mutex_exit(os_file_count_mutex); + + return(n_bytes); #else { ssize_t ret; @@ -982,8 +1136,16 @@ os_file_pwrite( os_n_file_writes++; #ifdef HAVE_PWRITE + os_mutex_enter(os_file_count_mutex); + os_file_n_pending_pwrites++; + os_mutex_exit(os_file_count_mutex); + ret = pwrite(file, buf, n, offs); + os_mutex_enter(os_file_count_mutex); + os_file_n_pending_pwrites--; + os_mutex_exit(os_file_count_mutex); + if (srv_unix_file_flush_method != SRV_UNIX_LITTLESYNC && srv_unix_file_flush_method != SRV_UNIX_NOSYNC && !os_do_not_call_flush_at_each_write) { @@ -1372,20 +1534,36 @@ os_aio_init( os_io_init_simple(); + for (i = 0; i < n_segments; i++) { + srv_io_thread_op_info[i] = (char*)"not started yet"; + } + n_per_seg = n / n_segments; n_write_segs = (n_segments - 2) / 2; n_read_segs = n_segments - 2 - n_write_segs; /* printf("Array n per seg %lu\n", n_per_seg); */ - os_aio_read_array = os_aio_array_create(n_read_segs * n_per_seg, - n_read_segs); - os_aio_write_array = os_aio_array_create(n_write_segs * n_per_seg, - n_write_segs); os_aio_ibuf_array = os_aio_array_create(n_per_seg, 1); + srv_io_thread_function[0] = (char*)"insert buffer thread"; + os_aio_log_array = os_aio_array_create(n_per_seg, 1); + srv_io_thread_function[1] = (char*)"log thread"; + + os_aio_read_array = os_aio_array_create(n_read_segs * n_per_seg, + n_read_segs); + for (i = 2; i < 2 + n_read_segs; i++) { + srv_io_thread_function[i] = (char*)"read thread"; + } + + os_aio_write_array = os_aio_array_create(n_write_segs * n_per_seg, + n_write_segs); + for (i = 2 + n_read_segs; i < n_segments; i++) { + srv_io_thread_function[i] = (char*)"write thread"; + } + os_aio_sync_array = os_aio_array_create(n_slots_sync, 1); os_aio_n_segments = n_segments; @@ -1677,6 +1855,7 @@ loop: } slot->reserved = TRUE; + slot->reservation_time = time(NULL); slot->message1 = message1; slot->message2 = message2; slot->file = file; @@ -2249,6 +2428,8 @@ os_aio_simulated_handle( ulint total_len; ulint offs; ulint lowest_offset; + ulint biggest_age; + ulint age; byte* combined_buf; byte* combined_buf2= 0; /* Remove warning */ ibool ret; @@ -2301,22 +2482,55 @@ restart: n_consecutive = 0; - /* Look for an i/o request at the lowest offset in the array - (we ignore the high 32 bits of the offset in these heuristics) */ + /* If there are at least 2 seconds old requests, then pick the oldest + one to prevent starvation. If several requests have the same age, + then pick the one at the lowest offset. */ + biggest_age = 0; lowest_offset = ULINT_MAX; - + for (i = 0; i < n; i++) { slot = os_aio_array_get_nth_slot(array, i + segment * n); - if (slot->reserved && slot->offset < lowest_offset) { + if (slot->reserved) { + age = (ulint)difftime(time(NULL), + slot->reservation_time); - /* Found an i/o request */ - consecutive_ios[0] = slot; + if ((age >= 2 && age > biggest_age) + || (age >= 2 && age == biggest_age + && slot->offset < lowest_offset)) { - n_consecutive = 1; + /* Found an i/o request */ + consecutive_ios[0] = slot; - lowest_offset = slot->offset; + n_consecutive = 1; + + biggest_age = age; + lowest_offset = slot->offset; + } + } + } + + if (n_consecutive == 0) { + /* There were no old requests. Look for an i/o request at the + lowest offset in the array (we ignore the high 32 bits of the + offset in these heuristics) */ + + lowest_offset = ULINT_MAX; + + for (i = 0; i < n; i++) { + slot = os_aio_array_get_nth_slot(array, + i + segment * n); + + if (slot->reserved && slot->offset < lowest_offset) { + + /* Found an i/o request */ + consecutive_ios[0] = slot; + + n_consecutive = 1; + + lowest_offset = slot->offset; + } } } @@ -2422,7 +2636,7 @@ consecutive_loop: + FIL_PAGE_LSN + 4) != mach_read_from_4(combined_buf + len2 + UNIV_PAGE_SIZE - - FIL_PAGE_END_LSN + 4)) { + - FIL_PAGE_END_LSN_OLD_CHKSUM + 4)) { ut_print_timestamp(stderr); fprintf(stderr, " InnoDB: ERROR: The page to be written seems corrupt!\n"); @@ -2583,14 +2797,15 @@ os_aio_print( double avg_bytes_read; ulint i; - if (buf_end - buf < 1000) { + if (buf_end - buf < 1200) { return; } for (i = 0; i < srv_n_file_io_threads; i++) { - buf += sprintf(buf, "I/O thread %lu state: %s\n", i, - srv_io_thread_op_info[i]); + buf += sprintf(buf, "I/O thread %lu state: %s (%s)\n", i, + srv_io_thread_op_info[i], + srv_io_thread_function[i]); } buf += sprintf(buf, "Pending normal aio reads:"); @@ -2665,6 +2880,12 @@ loop: "%lu OS file reads, %lu OS file writes, %lu OS fsyncs\n", os_n_file_reads, os_n_file_writes, os_n_fsyncs); + if (os_file_n_pending_preads != 0 || os_file_n_pending_pwrites != 0) { + buf += sprintf(buf, + "%lu pending preads, %lu pending pwrites\n", + os_file_n_pending_preads, os_file_n_pending_pwrites); + } + if (os_n_file_reads == os_n_file_reads_old) { avg_bytes_read = 0.0; } else { diff --git a/innobase/os/os0thread.c b/innobase/os/os0thread.c index 9af98760ad1..1252cc5e4b7 100644 --- a/innobase/os/os0thread.c +++ b/innobase/os/os0thread.c @@ -187,8 +187,8 @@ os_thread_exit( is cast as a DWORD */ { #ifdef UNIV_DEBUG_THREAD_CREATION - printf("A thread exits.\n"); - printf("Thread id %lu\n", os_thread_pf(os_thread_get_curr_id())); + printf("Thread exits, id %lu\n", + os_thread_pf(os_thread_get_curr_id())); #endif os_mutex_enter(os_sync_mutex); os_thread_count--; diff --git a/innobase/page/page0cur.c b/innobase/page/page0cur.c index d3a40668c4b..7e2fc19c00f 100644 --- a/innobase/page/page0cur.c +++ b/innobase/page/page0cur.c @@ -14,6 +14,7 @@ Created 10/4/1994 Heikki Tuuri #include "rem0cmp.h" #include "mtr0log.h" #include "log0recv.h" +#include "rem0cmp.h" ulint page_cur_short_succ = 0; @@ -218,6 +219,8 @@ page_cur_search_with_match( || (mode == PAGE_CUR_G) || (mode == PAGE_CUR_GE) || (mode == PAGE_CUR_LE_OR_EXTENDS) || (mode == PAGE_CUR_DBG)); + page_check_dir(page); + #ifdef PAGE_CUR_ADAPT if ((page_header_get_field(page, PAGE_LEVEL) == 0) && (mode == PAGE_CUR_LE) @@ -595,6 +598,7 @@ page_cur_parse_insert_rec( rec_t* cursor_rec; byte buf1[1024]; byte* buf; + byte* ptr2 = ptr; ulint info_bits = 0; /* remove warning */ page_cur_t cursor; @@ -697,7 +701,20 @@ page_cur_parse_insert_rec( /* Build the inserted record to buf */ - ut_a(mismatch_index < UNIV_PAGE_SIZE); + if (mismatch_index >= UNIV_PAGE_SIZE) { + printf("Is short %lu, info_bits %lu, offset %lu, o_offset %lu\n" + "mismatch index %lu, end_seg_len %lu\n" + "parsed len %lu\n", + is_short, info_bits, offset, origin_offset, + mismatch_index, end_seg_len, (ulint)(ptr - ptr2)); + + printf("Dump of 300 bytes of log:\n"); + ut_print_buf(ptr2, 300); + + buf_page_print(page); + + ut_a(0); + } ut_memcpy(buf, rec_get_start(cursor_rec), mismatch_index); ut_memcpy(buf + mismatch_index, ptr, end_seg_len); diff --git a/innobase/page/page0page.c b/innobase/page/page0page.c index 7d240bdd5b0..ef5dad60c08 100644 --- a/innobase/page/page0page.c +++ b/innobase/page/page0page.c @@ -353,7 +353,7 @@ page_create( infimum_rec = rec_convert_dtuple_to_rec(heap_top, tuple); - ut_ad(infimum_rec == page + PAGE_INFIMUM); + ut_a(infimum_rec == page + PAGE_INFIMUM); rec_set_n_owned(infimum_rec, 1); rec_set_heap_no(infimum_rec, 0); @@ -370,7 +370,7 @@ page_create( supremum_rec = rec_convert_dtuple_to_rec(heap_top, tuple); - ut_ad(supremum_rec == page + PAGE_SUPREMUM); + ut_a(supremum_rec == page + PAGE_SUPREMUM); rec_set_n_owned(supremum_rec, 1); rec_set_heap_no(supremum_rec, 1); @@ -389,6 +389,8 @@ page_create( page_header_set_ptr(page, PAGE_FREE, NULL); page_header_set_field(page, PAGE_GARBAGE, 0); page_header_set_ptr(page, PAGE_LAST_INSERT, NULL); + page_header_set_field(page, PAGE_DIRECTION, PAGE_NO_DIRECTION); + page_header_set_field(page, PAGE_N_DIRECTION, 0); page_header_set_field(page, PAGE_N_RECS, 0); page_set_max_trx_id(page, ut_dulint_zero); @@ -402,17 +404,22 @@ page_create( slot = page_dir_get_nth_slot(page, 1); page_dir_slot_set_rec(slot, supremum_rec); - /* Set next pointers in infimum and supremum */ + /* Set the next pointers in infimum and supremum */ rec_set_next_offs(infimum_rec, (ulint)(supremum_rec - page)); rec_set_next_offs(supremum_rec, 0); +#ifdef notdefined + /* Disable the use of page_template: there is a race condition here: + while one thread is creating page_template, another one can start + using it before the memcpy completes! */ + if (page_template == NULL) { page_template = mem_alloc(UNIV_PAGE_SIZE); ut_memcpy(page_template, page, UNIV_PAGE_SIZE); } - +#endif return(page); } @@ -439,6 +446,9 @@ page_copy_rec_list_end_no_locks( page_cur_move_to_next(&cur1); } + /* Track a memory corruption bug in Windows */ + ut_a(mach_read_from_2(new_page + UNIV_PAGE_SIZE - 10) == PAGE_INFIMUM); + page_cur_set_before_first(new_page, &cur2); /* Copy records from the original page to the new page */ @@ -449,6 +459,8 @@ page_copy_rec_list_end_no_locks( ut_a( page_cur_rec_insert(&cur2, page_cur_get_rec(&cur1), mtr)); + ut_a(mach_read_from_2(new_page + UNIV_PAGE_SIZE - 10) + == PAGE_INFIMUM); page_cur_move_to_next(&cur1); page_cur_move_to_next(&cur2); } @@ -1315,6 +1327,37 @@ page_rec_validate( return(TRUE); } + +/******************************************************************* +Checks that the first directory slot points to the infimum record and +the last to the supremum. This function is intended to track if the +bug fixed in 4.0.14 has caused corruption to users' databases. */ + +void +page_check_dir( +/*===========*/ + page_t* page) /* in: index page */ +{ + ulint n_slots; + + n_slots = page_dir_get_n_slots(page); + + if (page_dir_slot_get_rec(page_dir_get_nth_slot(page, 0)) + != page_get_infimum_rec(page)) { + + fprintf(stderr, +"InnoDB: Page directory corruption: supremum not pointed to\n"); + buf_page_print(page); + } + + if (page_dir_slot_get_rec(page_dir_get_nth_slot(page, n_slots - 1)) + != page_get_supremum_rec(page)) { + + fprintf(stderr, +"InnoDB: Page directory corruption: supremum not pointed to\n"); + buf_page_print(page); + } +} /******************************************************************* This function checks the consistency of an index page when we do not @@ -1598,7 +1641,8 @@ page_validate( "InnoDB: previous record %s\n", err_buf); rec_sprintf(err_buf, 900, rec); - fprintf(stderr, "InnoDB: record %s\n", err_buf); + fprintf(stderr, + "InnoDB: record %s\n", err_buf); goto func_exit; } diff --git a/innobase/pars/pars0opt.c b/innobase/pars/pars0opt.c index 91083e6fa16..4faf83b47a3 100644 --- a/innobase/pars/pars0opt.c +++ b/innobase/pars/pars0opt.c @@ -1058,7 +1058,6 @@ opt_clust_access( dfield_t* dfield; mem_heap_t* heap; ulint n_fields; - ulint col_no; ulint pos; ulint i; @@ -1093,8 +1092,7 @@ opt_clust_access( plan->clust_map = mem_heap_alloc(heap, n_fields * sizeof(ulint)); for (i = 0; i < n_fields; i++) { - col_no = dict_index_get_nth_col_no(clust_index, i); - pos = dict_index_get_nth_col_pos(index, col_no); + pos = dict_index_get_nth_field_pos(index, clust_index, i); *(plan->clust_map + i) = pos; @@ -1109,7 +1107,8 @@ opt_clust_access( dfield = dtuple_get_nth_field(plan->clust_ref, table->mix_len); - dfield_set_data(dfield, mem_heap_alloc(heap, table->mix_id_len), + dfield_set_data(dfield, mem_heap_alloc(heap, + table->mix_id_len), table->mix_id_len); ut_memcpy(dfield_get_data(dfield), table->mix_id_buf, table->mix_id_len); diff --git a/innobase/pars/pars0pars.c b/innobase/pars/pars0pars.c index 664f498ef3e..3e43b6ae262 100644 --- a/innobase/pars/pars0pars.c +++ b/innobase/pars/pars0pars.c @@ -244,13 +244,11 @@ pars_resolve_func_data_type( /* Inherit the data type from the first argument (which must not be the SQL null literal whose type is DATA_ERROR) */ - ut_a(dtype_get_mtype(que_node_get_data_type(arg)) - != DATA_ERROR); dtype_copy(que_node_get_data_type(node), que_node_get_data_type(arg)); - ut_a(dtype_get_mtype(que_node_get_data_type(node)) == DATA_INT); - + ut_a(dtype_get_mtype(que_node_get_data_type(node)) + == DATA_INT); } else if (func == PARS_COUNT_TOKEN) { ut_a(arg); dtype_set(que_node_get_data_type(node), DATA_INT, 0, 4, 0); @@ -1596,7 +1594,7 @@ pars_create_index( column = column_list; while (column) { - dict_mem_index_add_field(index, column->name, 0); + dict_mem_index_add_field(index, column->name, 0, 0); column->resolved = TRUE; column->token_type = SYM_COLUMN; diff --git a/innobase/rem/rem0cmp.c b/innobase/rem/rem0cmp.c index e9740d7ea78..2e18e68ec43 100644 --- a/innobase/rem/rem0cmp.c +++ b/innobase/rem/rem0cmp.c @@ -38,7 +38,7 @@ Used in debug checking of cmp_dtuple_... . This function is used to compare a data tuple to a physical record. If dtuple has n fields then rec must have either m >= n fields, or it must differ from dtuple in some of the m fields rec has. */ -static + int cmp_debug_dtuple_rec_with_match( /*============================*/ @@ -50,9 +50,10 @@ cmp_debug_dtuple_rec_with_match( dtuple in some of the common fields, or which has an equal number or more fields than dtuple */ - ulint* matched_fields);/* in/out: number of already completely - matched fields; when function returns, - contains the value for current comparison */ + ulint* matched_fields);/* in/out: number of already + completely matched fields; when function + returns, contains the value for current + comparison */ /***************************************************************** This function is used to compare two data fields for which the data type is such that we must use MySQL code to compare them. The prototype here @@ -79,17 +80,12 @@ UNIV_INLINE ulint cmp_collate( /*========*/ - /* out: collation order position */ - dtype_t* type __attribute__((unused)) , /* in: type */ - ulint code) /* in: code of a character stored in database - record */ -{ - ut_ad((type->mtype == DATA_CHAR) || (type->mtype == DATA_VARCHAR)); - + /* out: collation order position */ + ulint code) /* in: code of a character stored in database record */ +{ return((ulint) srv_latin1_ordering[code]); } - /***************************************************************** Returns TRUE if two types are equal for comparison purposes. */ @@ -118,7 +114,8 @@ cmp_types_are_equal( if (type1->mtype == DATA_INT && (type1->prtype & DATA_UNSIGNED) - != (type2->prtype & DATA_UNSIGNED)) { + != (type2->prtype & DATA_UNSIGNED)) { + /* The storage format of an unsigned integer is different from a signed integer: in a signed integer we OR 0x8000... to the value of positive integers. */ @@ -131,12 +128,17 @@ cmp_types_are_equal( return(FALSE); } + if (type1->mtype == DATA_BLOB && (type1->prtype & DATA_BINARY_TYPE) + != (type2->prtype & DATA_BINARY_TYPE)) { + return(FALSE); + } + return(TRUE); } /***************************************************************** -Innobase uses this function is to compare two data fields for which the -data type is such that we must compare whole fields. */ +Innobase uses this function to compare two data fields for which the data type +is such that we must compare whole fields or call MySQL to do the comparison */ static int cmp_whole_field( @@ -239,8 +241,34 @@ cmp_whole_field( return(0); case DATA_VARMYSQL: case DATA_MYSQL: + case DATA_BLOB: + if (data_type == DATA_BLOB + && 0 != (type->prtype & DATA_BINARY_TYPE)) { + + ut_print_timestamp(stderr); + fprintf(stderr, +" InnoDB: Error: comparing a binary BLOB with a character set sensitive\n" +"InnoDB: comparison!\n"); + } + + /* MySQL does not pad the ends of strings with spaces in a + comparison. That would cause a foreign key check to fail for + non-latin1 character sets if we have different length columns. + To prevent that we remove trailing spaces here before doing + the comparison. NOTE that if we in the future map more MySQL + types to DATA_MYSQL or DATA_VARMYSQL, we have to change this + code. */ + + while (a_length > 0 && a[a_length - 1] == ' ') { + a_length--; + } + + while (b_length > 0 && b[b_length - 1] == ' ') { + b_length--; + } + return(innobase_mysql_cmp( - (int)(type->prtype & ~DATA_NOT_NULL), + (int)(type->prtype & DATA_MYSQL_TYPE_MASK), a, a_length, b, b_length)); default: fprintf(stderr, @@ -291,7 +319,10 @@ cmp_data_data_slow( return(1); } - if (cur_type->mtype >= DATA_FLOAT) { + if (cur_type->mtype >= DATA_FLOAT + || (cur_type->mtype == DATA_BLOB + && (cur_type->prtype & DATA_NONLATIN1))) { + return(cmp_whole_field(cur_type, data1, len1, data2, len2)); } @@ -334,9 +365,12 @@ cmp_data_data_slow( goto next_byte; } - if (cur_type->mtype <= DATA_CHAR) { - data1_byte = cmp_collate(cur_type, data1_byte); - data2_byte = cmp_collate(cur_type, data2_byte); + if (cur_type->mtype <= DATA_CHAR + || (cur_type->mtype == DATA_BLOB + && 0 == (cur_type->prtype & DATA_BINARY_TYPE))) { + + data1_byte = cmp_collate(data1_byte); + data2_byte = cmp_collate(data2_byte); } if (data1_byte > data2_byte) { @@ -487,7 +521,9 @@ cmp_dtuple_rec_with_match( } } - if (cur_type->mtype >= DATA_FLOAT) { + if (cur_type->mtype >= DATA_FLOAT + || (cur_type->mtype == DATA_BLOB + && (cur_type->prtype & DATA_NONLATIN1))) { ret = cmp_whole_field(cur_type, dfield_get_data(dtuple_field), dtuple_f_len, @@ -547,10 +583,13 @@ cmp_dtuple_rec_with_match( goto next_byte; } - if (cur_type->mtype <= DATA_CHAR) { - rec_byte = cmp_collate(cur_type, rec_byte); - dtuple_byte = cmp_collate(cur_type, - dtuple_byte); + if (cur_type->mtype <= DATA_CHAR + || (cur_type->mtype == DATA_BLOB + && 0 == + (cur_type->prtype & DATA_BINARY_TYPE))) { + + rec_byte = cmp_collate(rec_byte); + dtuple_byte = cmp_collate(dtuple_byte); } if (dtuple_byte > rec_byte) { @@ -583,8 +622,8 @@ order_resolved: matched_fields)); ut_ad(*matched_fields == cur_field); /* In the debug version, the above cmp_debug_... sets - *matched_fields to a value */ - *matched_fields = cur_field; + *matched_fields to a value */ + *matched_fields = cur_field; *matched_bytes = cur_bytes; return(ret); @@ -804,7 +843,10 @@ cmp_rec_rec_with_match( } } - if (cur_type->mtype >= DATA_FLOAT) { + if (cur_type->mtype >= DATA_FLOAT + || (cur_type->mtype == DATA_BLOB + && (cur_type->prtype & DATA_NONLATIN1))) { + ret = cmp_whole_field(cur_type, rec1_b_ptr, rec1_f_len, rec2_b_ptr, rec2_f_len); @@ -861,9 +903,13 @@ cmp_rec_rec_with_match( goto next_byte; } - if (cur_type->mtype <= DATA_CHAR) { - rec1_byte = cmp_collate(cur_type, rec1_byte); - rec2_byte = cmp_collate(cur_type, rec2_byte); + if (cur_type->mtype <= DATA_CHAR + || (cur_type->mtype == DATA_BLOB + && 0 == + (cur_type->prtype & DATA_BINARY_TYPE))) { + + rec1_byte = cmp_collate(rec1_byte); + rec2_byte = cmp_collate(rec2_byte); } if (rec1_byte < rec2_byte) { @@ -906,7 +952,7 @@ This function is used to compare a data tuple to a physical record. If dtuple has n fields then rec must have either m >= n fields, or it must differ from dtuple in some of the m fields rec has. If encounters an externally stored field, returns 0. */ -static + int cmp_debug_dtuple_rec_with_match( /*============================*/ @@ -918,9 +964,10 @@ cmp_debug_dtuple_rec_with_match( dtuple in some of the common fields, or which has an equal number or more fields than dtuple */ - ulint* matched_fields) /* in/out: number of already completely - matched fields; when function returns, - contains the value for current comparison */ + ulint* matched_fields) /* in/out: number of already + completely matched fields; when function + returns, contains the value for current + comparison */ { dtype_t* cur_type; /* pointer to type of the current field in dtuple */ diff --git a/innobase/row/row0ins.c b/innobase/row/row0ins.c index e96c08a715b..23da0b9b93c 100644 --- a/innobase/row/row0ins.c +++ b/innobase/row/row0ins.c @@ -217,8 +217,8 @@ ins_node_set_new_row( } /*********************************************************************** -Does an insert operation by updating a delete marked existing record -in the index. This situation can occur if the delete marked record is +Does an insert operation by updating a delete-marked existing record +in the index. This situation can occur if the delete-marked record is kept in the index for consistent reads. */ static ulint @@ -240,9 +240,9 @@ row_ins_sec_index_entry_by_modify( ut_ad((cursor->index->type & DICT_CLUSTERED) == 0); ut_ad(rec_get_deleted_flag(rec)); - /* We know that in the ordering entry and rec are identified. - But in their binary form there may be differences if there - are char fields in them. Therefore we have to calculate the + /* We know that in the alphabetical ordering, entry and rec are + identical. But in their binary form there may be differences if + there are char fields in them. Therefore we have to calculate the difference and do an update-in-place if necessary. */ heap = mem_heap_create(1024); @@ -305,8 +305,8 @@ row_ins_clust_index_entry_by_modify( /* Try optimistic updating of the record, keeping changes within the page */ - err = btr_cur_optimistic_update(0, cursor, update, 0, thr, mtr); - + err = btr_cur_optimistic_update(0, cursor, update, 0, thr, + mtr); if (err == DB_OVERFLOW || err == DB_UNDERFLOW) { err = DB_FAIL; } @@ -364,11 +364,17 @@ row_ins_cascade_calc_update_vec( /* out: number of fields in the calculated update vector; the value can also be 0 if no foreign key - fields changed */ + fields changed; the returned value + is ULINT_UNDEFINED if the column + type in the child table is too short + to fit the new value in the parent + table: that means the update fails */ upd_node_t* node, /* in: update node of the parent table */ - dict_foreign_t* foreign) /* in: foreign key constraint whose + dict_foreign_t* foreign, /* in: foreign key constraint whose type is != 0 */ + mem_heap_t* heap) /* in: memory heap to use as + temporary storage */ { upd_node_t* cascade = node->cascade_node; dict_table_t* table = foreign->foreign_table; @@ -381,14 +387,16 @@ row_ins_cascade_calc_update_vec( upd_field_t* parent_ufield; ulint n_fields_updated; ulint parent_field_no; + dtype_t* type; ulint i; ulint j; ut_a(node && foreign && cascade && table && index); /* Calculate the appropriate update vector which will set the fields - in the child index record to the same value as the referenced index - record will get in the update. */ + in the child index record to the same value (possibly padded with + spaces if the column is a fixed length CHAR or FIXBINARY column) as + the referenced index record will get in the update. */ parent_table = node->table; ut_a(parent_table == foreign->referenced_table); @@ -424,7 +432,56 @@ row_ins_cascade_calc_update_vec( dict_table_get_nth_col_pos(table, dict_index_get_nth_col_no(index, i)); ufield->exp = NULL; + ufield->new_val = parent_ufield->new_val; + + type = dict_index_get_nth_type(index, i); + + /* Do not allow a NOT NULL column to be + updated as NULL */ + + if (ufield->new_val.len == UNIV_SQL_NULL + && (type->prtype & DATA_NOT_NULL)) { + + return(ULINT_UNDEFINED); + } + + /* If the new value would not fit in the + column, do not allow the update */ + + if (ufield->new_val.len != UNIV_SQL_NULL + && ufield->new_val.len + > dtype_get_len(type)) { + + return(ULINT_UNDEFINED); + } + + /* If the parent column type has a different + length than the child column type, we may + need to pad with spaces the new value of the + child column */ + + if (dtype_is_fixed_size(type) + && ufield->new_val.len != UNIV_SQL_NULL + && ufield->new_val.len + < dtype_get_fixed_size(type)) { + + ufield->new_val.data = + mem_heap_alloc(heap, + dtype_get_fixed_size(type)); + ufield->new_val.len = + dtype_get_fixed_size(type); + ut_a(dtype_get_pad_char(type) + != ULINT_UNDEFINED); + + memset(ufield->new_val.data, + (byte)dtype_get_pad_char(type), + dtype_get_fixed_size(type)); + ut_memcpy(ufield->new_val.data, + parent_ufield->new_val.data, + parent_ufield->new_val.len); + } + ufield->extern_storage = FALSE; n_fields_updated++; @@ -570,9 +627,11 @@ row_ins_foreign_check_on_constraint( dict_index_t* clust_index; dtuple_t* ref; mem_heap_t* tmp_heap; + mem_heap_t* upd_vec_heap = NULL; rec_t* rec; rec_t* clust_rec; upd_t* update; + ulint n_to_update; ulint err; ulint i; char* ptr; @@ -597,8 +656,10 @@ row_ins_foreign_check_on_constraint( *ptr = '\0'; /* We call a function in ha_innodb.cc */ +#ifndef UNIV_HOTBACKUP innobase_invalidate_query_cache(thr_get_trx(thr), table_name_buf, ut_strlen(table->name) + 1); +#endif node = thr->run_node; if (node->is_delete && 0 == (foreign->type & @@ -828,7 +889,21 @@ row_ins_foreign_check_on_constraint( /* Build the appropriate update vector which sets changing foreign->n_fields first fields in rec to new values */ - row_ins_cascade_calc_update_vec(node, foreign); + upd_vec_heap = mem_heap_create(256); + + n_to_update = row_ins_cascade_calc_update_vec(node, foreign, + upd_vec_heap); + if (n_to_update == ULINT_UNDEFINED) { + err = DB_ROW_IS_REFERENCED; + + row_ins_foreign_report_err( +(char*)"Trying a cascaded update where the updated value in the child\n" +"table would not fit in the length of the column, or the value would\n" +"be NULL and the column is declared as not NULL in the child table,", + thr, foreign, btr_pcur_get_rec(pcur), entry); + + goto nonstandard_exit_func; + } if (cascade->update->n_fields == 0) { @@ -867,10 +942,18 @@ row_ins_foreign_check_on_constraint( btr_pcur_restore_position(BTR_SEARCH_LEAF, pcur, mtr); + if (upd_vec_heap) { + mem_heap_free(upd_vec_heap); + } + return(err); nonstandard_exit_func: + if (upd_vec_heap) { + mem_heap_free(upd_vec_heap); + } + btr_pcur_store_position(pcur, mtr); mtr_commit(mtr); @@ -1275,6 +1358,11 @@ row_ins_unique_report_err( dtuple_t* entry, /* in: index entry to insert in the index */ dict_index_t* index) /* in: index */ { + UT_NOT_USED(thr); + UT_NOT_USED(rec); + UT_NOT_USED(entry); + UT_NOT_USED(index); + #ifdef notdefined /* Disable reporting to test if the slowdown of REPLACE in 4.0.13 was caused by this! */ @@ -1816,7 +1904,7 @@ row_ins_index_entry( /* Try first optimistic descent to the B-tree */ err = row_ins_index_entry_low(BTR_MODIFY_LEAF, index, entry, - ext_vec, n_ext_vec, thr); + ext_vec, n_ext_vec, thr); if (err != DB_FAIL) { return(err); @@ -1832,13 +1920,15 @@ row_ins_index_entry( /*************************************************************** Sets the values of the dtuple fields in entry from the values of appropriate columns in row. */ -UNIV_INLINE +static void row_ins_index_entry_set_vals( /*=========================*/ + dict_index_t* index, /* in: index */ dtuple_t* entry, /* in: index entry to make */ dtuple_t* row) /* in: row */ { + dict_field_t* ind_field; dfield_t* field; dfield_t* row_field; ulint n_fields; @@ -1850,11 +1940,21 @@ row_ins_index_entry_set_vals( for (i = 0; i < n_fields; i++) { field = dtuple_get_nth_field(entry, i); + ind_field = dict_index_get_nth_field(index, i); - row_field = dtuple_get_nth_field(row, field->col_no); + row_field = dtuple_get_nth_field(row, ind_field->col->ind); + + /* Check column prefix indexes */ + if (ind_field->prefix_len > 0 + && dfield_get_len(row_field) != UNIV_SQL_NULL + && dfield_get_len(row_field) > ind_field->prefix_len) { + + field->len = ind_field->prefix_len; + } else { + field->len = row_field->len; + } field->data = row_field->data; - field->len = row_field->len; } } @@ -1873,7 +1973,7 @@ row_ins_index_entry_step( ut_ad(dtuple_check_typed(node->row)); - row_ins_index_entry_set_vals(node->entry, node->row); + row_ins_index_entry_set_vals(node->index, node->entry, node->row); ut_ad(dtuple_check_typed(node->entry)); diff --git a/innobase/row/row0mysql.c b/innobase/row/row0mysql.c index 428e4d568f3..61ba9111b91 100644 --- a/innobase/row/row0mysql.c +++ b/innobase/row/row0mysql.c @@ -76,9 +76,6 @@ row_mysql_store_blob_ref( also to set the NULL bit in the MySQL record header! */ { - ulint sum = 0; - ulint i; - /* MySQL might assume the field is set to zero except the length and the pointer fields */ @@ -93,22 +90,6 @@ row_mysql_store_blob_ref( ut_a(col_len - 8 > 2 || len < 256 * 256); ut_a(col_len - 8 > 3 || len < 256 * 256 * 256); - /* We try to track an elusive bug which probably was fixed - May 9, 2002, but better be sure: we probe the data buffer - to make sure it is in valid allocated memory */ - - for (i = 0; i < len; i++) { - - sum += (ulint)(data + i); - } - - /* The variable below is identically false, we just fool the - compiler to not optimize away our loop */ - if (row_mysql_identically_false) { - - printf("Sum %lu\n", sum); - } - mach_write_to_n_little_endian(dest, col_len - 8, len); ut_memcpy(dest + col_len - 8, (byte*)&data, sizeof(byte*)); @@ -952,7 +933,8 @@ row_update_for_mysql( if (prebuilt->pcur->btr_cur.index == clust_index) { btr_pcur_copy_stored_position(node->pcur, prebuilt->pcur); } else { - btr_pcur_copy_stored_position(node->pcur, prebuilt->clust_pcur); + btr_pcur_copy_stored_position(node->pcur, + prebuilt->clust_pcur); } ut_a(node->pcur->rel_pos == BTR_PCUR_ON); @@ -1477,8 +1459,7 @@ row_create_index_for_mysql( ulint namelen; ulint keywordlen; ulint err; - ulint i; - ulint j; + ulint i, j; ut_ad(rw_lock_own(&dict_operation_lock, RW_LOCK_EX)); ut_ad(mutex_own(&(dict_sys->mutex))); @@ -1486,23 +1467,9 @@ row_create_index_for_mysql( trx->op_info = (char *) "creating index"; - trx_start_if_not_started(trx); - - namelen = ut_strlen(index->table_name); - - keywordlen = ut_strlen("_recover_innodb_tmp_table"); - - if (namelen >= keywordlen - && 0 == ut_memcmp( - index->table_name + namelen - keywordlen, - (char*)"_recover_innodb_tmp_table", keywordlen)) { - - return(DB_SUCCESS); - } - /* Check that the same column does not appear twice in the index. - InnoDB assumes this in its algorithms, e.g., update of an index - entry */ + Starting from 4.0.14 InnoDB should be able to cope with that, but + safer not to allow them. */ for (i = 0; i < dict_index_get_n_fields(index); i++) { for (j = 0; j < i; j++) { @@ -1525,6 +1492,20 @@ row_create_index_for_mysql( } } + trx_start_if_not_started(trx); + + namelen = ut_strlen(index->table_name); + + keywordlen = ut_strlen("_recover_innodb_tmp_table"); + + if (namelen >= keywordlen + && 0 == ut_memcmp( + index->table_name + namelen - keywordlen, + (char*)"_recover_innodb_tmp_table", keywordlen)) { + + return(DB_SUCCESS); + } + heap = mem_heap_create(512); trx->dict_operation = TRUE; @@ -1542,6 +1523,7 @@ row_create_index_for_mysql( que_graph_free((que_t*) que_node_get_parent(thr)); error_handling: + if (err != DB_SUCCESS) { /* We have special error handling here */ @@ -2541,7 +2523,7 @@ loop: prev_entry = row_rec_to_index_entry(ROW_COPY_DATA, index, rec, heap); - ret = row_search_for_mysql(buf, PAGE_CUR_G, prebuilt, 0, ROW_SEL_NEXT); + ret = row_search_for_mysql(buf, PAGE_CUR_G, prebuilt, 0, ROW_SEL_NEXT); goto loop; } diff --git a/innobase/row/row0row.c b/innobase/row/row0row.c index 40a775143f4..6c0c6c04cd5 100644 --- a/innobase/row/row0row.c +++ b/innobase/row/row0row.c @@ -136,7 +136,14 @@ row_build_index_entry( dfield2 = dtuple_get_nth_field(row, dict_col_get_no(col)); dfield_copy(dfield, dfield2); - dfield->col_no = dict_col_get_no(col); + + /* If a column prefix index, take only the prefix */ + if (ind_field->prefix_len > 0 + && dfield_get_len(dfield2) != UNIV_SQL_NULL + && dfield_get_len(dfield2) > ind_field->prefix_len) { + + dfield_set_len(dfield, ind_field->prefix_len); + } } ut_ad(dtuple_check_typed(entry)); @@ -146,8 +153,7 @@ row_build_index_entry( /*********************************************************************** An inverse function to dict_row_build_index_entry. Builds a row from a -record in a clustered index. NOTE that externally stored (often big) -fields are always copied to heap. */ +record in a clustered index. */ dtuple_t* row_build( @@ -172,6 +178,7 @@ row_build( { dtuple_t* row; dict_table_t* table; + dict_field_t* ind_field; dict_col_t* col; dfield_t* dfield; ulint n_fields; @@ -204,19 +211,24 @@ row_build( dict_table_copy_types(row, table); for (i = 0; i < n_fields; i++) { + ind_field = dict_index_get_nth_field(index, i); - col = dict_field_get_col(dict_index_get_nth_field(index, i)); - dfield = dtuple_get_nth_field(row, dict_col_get_no(col)); - field = rec_get_nth_field(rec, i, &len); + if (ind_field->prefix_len == 0) { - if (type == ROW_COPY_ALSO_EXTERNALS - && rec_get_nth_field_extern_bit(rec, i)) { + col = dict_field_get_col(ind_field); + dfield = dtuple_get_nth_field(row, + dict_col_get_no(col)); + field = rec_get_nth_field(rec, i, &len); - field = btr_rec_copy_externally_stored_field(rec, - i, &len, heap); + if (type == ROW_COPY_ALSO_EXTERNALS + && rec_get_nth_field_extern_bit(rec, i)) { + + field = btr_rec_copy_externally_stored_field( + rec, i, &len, heap); + } + + dfield_set_data(dfield, field, len); } - - dfield_set_data(dfield, field, len); } ut_ad(dtuple_check_typed(row)); @@ -371,7 +383,6 @@ row_build_row_ref( dict_table_t* table; dict_index_t* clust_index; dfield_t* dfield; - dict_col_t* col; dtuple_t* ref; byte* field; ulint len; @@ -403,24 +414,13 @@ row_build_row_ref( for (i = 0; i < ref_len; i++) { dfield = dtuple_get_nth_field(ref, i); - col = dict_field_get_col( - dict_index_get_nth_field(clust_index, i)); - pos = dict_index_get_nth_col_pos(index, dict_col_get_no(col)); + pos = dict_index_get_nth_field_pos(index, clust_index, i); - if (pos != ULINT_UNDEFINED) { - field = rec_get_nth_field(rec, pos, &len); + ut_a(pos != ULINT_UNDEFINED); + + field = rec_get_nth_field(rec, pos, &len); - dfield_set_data(dfield, field, len); - } else { - ut_ad(table->type == DICT_TABLE_CLUSTER_MEMBER); - ut_ad(i == table->mix_len); - - dfield_set_data(dfield, - mem_heap_alloc(heap, table->mix_id_len), - table->mix_id_len); - ut_memcpy(dfield_get_data(dfield), table->mix_id_buf, - table->mix_id_len); - } + dfield_set_data(dfield, field, len); } ut_ad(dtuple_check_typed(ref)); @@ -448,7 +448,6 @@ row_build_row_ref_in_tuple( dict_table_t* table; dict_index_t* clust_index; dfield_t* dfield; - dict_col_t* col; byte* field; ulint len; ulint ref_len; @@ -483,19 +482,13 @@ row_build_row_ref_in_tuple( for (i = 0; i < ref_len; i++) { dfield = dtuple_get_nth_field(ref, i); - col = dict_field_get_col( - dict_index_get_nth_field(clust_index, i)); - pos = dict_index_get_nth_col_pos(index, dict_col_get_no(col)); + pos = dict_index_get_nth_field_pos(index, clust_index, i); - if (pos != ULINT_UNDEFINED) { - field = rec_get_nth_field(rec, pos, &len); + ut_a(pos != ULINT_UNDEFINED); + + field = rec_get_nth_field(rec, pos, &len); - dfield_set_data(dfield, field, len); - } else { - ut_ad(table->type == DICT_TABLE_CLUSTER_MEMBER); - ut_ad(i == table->mix_len); - ut_a(0); - } + dfield_set_data(dfield, field, len); } ut_ad(dtuple_check_typed(ref)); @@ -517,6 +510,7 @@ row_build_row_ref_from_row( directly into data of this row */ { dict_index_t* clust_index; + dict_field_t* field; dfield_t* dfield; dfield_t* dfield2; dict_col_t* col; @@ -533,13 +527,21 @@ row_build_row_ref_from_row( for (i = 0; i < ref_len; i++) { dfield = dtuple_get_nth_field(ref, i); - - col = dict_field_get_col( - dict_index_get_nth_field(clust_index, i)); - + + field = dict_index_get_nth_field(clust_index, i); + + col = dict_field_get_col(field); + dfield2 = dtuple_get_nth_field(row, dict_col_get_no(col)); dfield_copy(dfield, dfield2); + + if (field->prefix_len > 0 + && dfield->len != UNIV_SQL_NULL + && dfield->len > field->prefix_len) { + + dfield->len = field->prefix_len; + } } ut_ad(dtuple_check_typed(ref)); diff --git a/innobase/row/row0sel.c b/innobase/row/row0sel.c index 4732472d805..114ebf870b0 100644 --- a/innobase/row/row0sel.c +++ b/innobase/row/row0sel.c @@ -65,41 +65,50 @@ row_sel_sec_rec_is_for_clust_rec( rec_t* sec_rec, /* in: secondary index record */ dict_index_t* sec_index, /* in: secondary index */ rec_t* clust_rec, /* in: clustered index record */ - dict_index_t* clust_index __attribute__((unused))) - /* in: clustered index */ + dict_index_t* clust_index) /* in: clustered index */ { - dict_col_t* col; - byte* sec_field; - ulint sec_len; - byte* clust_field; - ulint clust_len; - ulint n; - ulint i; + dict_field_t* ifield; + dict_col_t* col; + byte* sec_field; + ulint sec_len; + byte* clust_field; + ulint clust_len; + ulint n; + ulint i; - n = dict_index_get_n_ordering_defined_by_user(sec_index); + UT_NOT_USED(clust_index); - for (i = 0; i < n; i++) { - col = dict_field_get_col( - dict_index_get_nth_field(sec_index, i)); + n = dict_index_get_n_ordering_defined_by_user(sec_index); - clust_field = rec_get_nth_field(clust_rec, - dict_col_get_clust_pos(col), - &clust_len); - sec_field = rec_get_nth_field(sec_rec, i, &sec_len); + for (i = 0; i < n; i++) { + ifield = dict_index_get_nth_field(sec_index, i); + col = dict_field_get_col(ifield); + + clust_field = rec_get_nth_field(clust_rec, + dict_col_get_clust_pos(col), + &clust_len); + sec_field = rec_get_nth_field(sec_rec, i, &sec_len); - if (sec_len != clust_len) { + if (ifield->prefix_len > 0 + && clust_len != UNIV_SQL_NULL + && clust_len > ifield->prefix_len) { - return(FALSE); + clust_len = ifield->prefix_len; } - if (0 != cmp_data_data(dict_col_get_type(col), - clust_field, clust_len, - sec_field, sec_len)) { - return(FALSE); - } - } + if (sec_len != clust_len) { - return(TRUE); + return(FALSE); + } + + if (0 != cmp_data_data(dict_col_get_type(col), + clust_field, clust_len, + sec_field, sec_len)) { + return(FALSE); + } + } + + return(TRUE); } /************************************************************************* @@ -606,7 +615,7 @@ row_sel_get_clust_rec( /* Try to place a lock on the index record */ err = lock_clust_rec_read_check_and_lock(0, clust_rec, index, - node->row_lock_mode, LOCK_ORDINARY, thr); + node->row_lock_mode, LOCK_ORDINARY, thr); if (err != DB_SUCCESS) { return(err); @@ -656,7 +665,7 @@ row_sel_get_clust_rec( *out_rec = clust_rec; return(DB_SUCCESS); - } + } } /* Fetch the columns needed in test conditions */ @@ -1850,9 +1859,11 @@ row_printf_step( } /******************************************************************** -Converts a key value stored in MySQL format to an Innobase dtuple. -The last field of the key value may be just a prefix of a fixed length -field: hence the parameter key_len. */ +Converts a key value stored in MySQL format to an Innobase dtuple. The last +field of the key value may be just a prefix of a fixed length field: hence +the parameter key_len. But currently we do not allow search keys where the +last field is only a prefix of the full key field len and print a warning if +such appears. */ void row_sel_convert_mysql_key_to_innobase( @@ -1863,17 +1874,24 @@ row_sel_convert_mysql_key_to_innobase( to index! */ byte* buf, /* in: buffer to use in field conversions */ + ulint buf_len, /* in: buffer length */ dict_index_t* index, /* in: index of the key value */ byte* key_ptr, /* in: MySQL key value */ ulint key_len) /* in: MySQL key value length */ { + byte* original_buf = buf; + dict_field_t* field; dfield_t* dfield; - ulint offset; - ulint len; + ulint data_offset; + ulint data_len; + ulint data_field_len; + ibool is_null; byte* key_end; ulint n_fields = 0; + ulint type; - UT_NOT_USED(index); + /* For documentation of the key value storage format in MySQL, see + ha_innobase::store_key_val_for_row() in ha_innodb.cc. */ key_end = key_ptr + key_len; @@ -1882,11 +1900,14 @@ row_sel_convert_mysql_key_to_innobase( dtuple_set_n_fields(tuple, ULINT_MAX); dfield = dtuple_get_nth_field(tuple, 0); + field = dict_index_get_nth_field(index, 0); if (dfield_get_type(dfield)->mtype == DATA_SYS) { - /* A special case: we are looking for a position in a - generated clustered index: the first and the only - ordering column is ROW_ID */ + /* A special case: we are looking for a position in the + generated clustered index which InnoDB automatically added + to a table with no primary key: the first and the only + ordering column is ROW_ID which InnoDB stored to the key_ptr + buffer. */ ut_a(key_len == DATA_ROW_ID_LEN); @@ -1897,70 +1918,114 @@ row_sel_convert_mysql_key_to_innobase( return; } - while (key_ptr < key_end) { - offset = 0; - len = dfield_get_type(dfield)->len; + while (key_ptr < key_end) { - n_fields++; + ut_a(dict_col_get_type(field->col)->mtype + == dfield_get_type(dfield)->mtype); + + data_offset = 0; + is_null = FALSE; if (!(dfield_get_type(dfield)->prtype & DATA_NOT_NULL)) { /* The first byte in the field tells if this is an SQL NULL value */ - offset = 1; + data_offset = 1; - if (*key_ptr != 0) { + if (*key_ptr != 0) { dfield_set_data(dfield, NULL, UNIV_SQL_NULL); - goto next_part; + is_null = TRUE; } } - row_mysql_store_col_in_innobase_format( - dfield, buf, key_ptr + offset, len, - dfield_get_type(dfield)->mtype, + type = dfield_get_type(dfield)->mtype; + + /* Calculate data length and data field total length */ + + if (type == DATA_BLOB) { + /* The key field is a column prefix of a BLOB or + TEXT type column */ + + ut_a(field->prefix_len > 0); + + /* MySQL stores the actual data length to the first 2 + bytes after the optional SQL NULL marker byte. The + storage format is little-endian. */ + + /* There are no key fields > 255 bytes currently in + MySQL */ + if (key_ptr[data_offset + 1] != 0) { + ut_print_timestamp(stderr); + fprintf(stderr, +" InnoDB: Error: BLOB or TEXT prefix > 255 bytes in query to table %s\n", + index->table_name); + } + + data_len = key_ptr[data_offset]; + data_field_len = data_offset + 2 + field->prefix_len; + data_offset += 2; + + type = DATA_CHAR; /* now that we know the length, we + store the column value like it would + be a fixed char field */ + } else if (field->prefix_len > 0) { + data_len = field->prefix_len; + data_field_len = data_offset + data_len; + } else { + data_len = dfield_get_type(dfield)->len; + data_field_len = data_offset + data_len; + } + + /* Storing may use at most data_len bytes of buf */ + + if (!is_null) { + row_mysql_store_col_in_innobase_format( + dfield, buf, key_ptr + data_offset, + data_len, type, dfield_get_type(dfield)->prtype & DATA_UNSIGNED); - next_part: - key_ptr += (offset + len); + buf += data_len; + } + + key_ptr += data_field_len; if (key_ptr > key_end) { - /* The last field in key was not a complete - field but a prefix of it. + /* The last field in key was not a complete key field + but a prefix of it. - Print a warning about this! HA_READ_PREFIX_LAST - does not currently work in InnoDB with partial-field - key value prefixes. Since MySQL currently uses a - padding trick to calculate LIKE 'abc%' type queries - there should never be partial-field prefixes - in searches. */ + Print a warning about this! HA_READ_PREFIX_LAST does + not currently work in InnoDB with partial-field key + value prefixes. Since MySQL currently uses a padding + trick to calculate LIKE 'abc%' type queries there + should never be partial-field prefixes in searches. */ ut_print_timestamp(stderr); fprintf(stderr, " InnoDB: Warning: using a partial-field key prefix in search\n"); - ut_ad(dfield_get_len(dfield) != UNIV_SQL_NULL); - - dfield_set_data(dfield, buf, - len - (ulint)(key_ptr - key_end)); + if (!is_null) { + dfield->len -= (ulint)(key_ptr - key_end); + } } - buf += len; - + n_fields++; + field++; dfield++; } - /* We set the length of tuple to n_fields: we assume that - the memory area allocated for it is big enough (usually - bigger than n_fields). */ + ut_a(buf <= original_buf + buf_len); + + /* We set the length of tuple to n_fields: we assume that the memory + area allocated for it is big enough (usually bigger than n_fields). */ dtuple_set_n_fields(tuple, n_fields); } /****************************************************************** Stores the row id to the prebuilt struct. */ -UNIV_INLINE +static void row_sel_store_row_id_to_prebuilt( /*=============================*/ @@ -1970,11 +2035,22 @@ row_sel_store_row_id_to_prebuilt( { byte* data; ulint len; + char err_buf[1000]; data = rec_get_nth_field(index_rec, dict_index_get_sys_col_pos(index, DATA_ROW_ID), &len); - ut_a(len == DATA_ROW_ID_LEN); + if (len != DATA_ROW_ID_LEN) { + rec_sprintf(err_buf, 900, index_rec); + + fprintf(stderr, +"InnoDB: Error: Row id field is wrong length %lu in table %s index %s\n" +"InnoDB: Field number %lu, record:\n%s\n", + len, index->table_name, index->name, + dict_index_get_sys_col_pos(index, DATA_ROW_ID), + err_buf); + ut_a(0); + } ut_memcpy(prebuilt->row_id, data, len); } @@ -3021,7 +3097,7 @@ rec_loop: if (prebuilt->select_lock_type != LOCK_NONE && set_also_gap_locks) { - /* Try to place a lock on the index record */ + /* Try to place a lock on the index record */ err = sel_set_rec_lock(rec, index, prebuilt->select_lock_type, diff --git a/innobase/row/row0umod.c b/innobase/row/row0umod.c index b84e55ca643..b22e494f891 100644 --- a/innobase/row/row0umod.c +++ b/innobase/row/row0umod.c @@ -428,7 +428,8 @@ row_undo_mod_del_unmark_sec( found = row_search_index_entry(index, entry, BTR_MODIFY_LEAF, &pcur, &mtr); if (!found) { - fprintf(stderr, "InnoDB: error in sec index entry del undo in\n" + fprintf(stderr, + "InnoDB: error in sec index entry del undo in\n" "InnoDB: index %s table %s\n", index->name, index->table->name); dtuple_sprintf(err_buf, 900, entry); @@ -570,7 +571,7 @@ row_undo_mod_upd_exist_sec( the row */ row_upd_index_replace_new_col_vals(entry, index, - node->update); + node->update, NULL); row_undo_mod_del_unmark_sec(node, thr, index, entry); } diff --git a/innobase/row/row0upd.c b/innobase/row/row0upd.c index 5fce1c1861b..db68479509d 100644 --- a/innobase/row/row0upd.c +++ b/innobase/row/row0upd.c @@ -72,8 +72,9 @@ searched delete is obviously to keep the x-latch for several steps of query graph execution. */ /*************************************************************** -Checks if an update vector changes some of the first fields of an index -record. */ +Checks if an update vector changes some of the first ordering fields of an +index record. This is only used in foreign key checks and we can assume +that index does not contain column prefixes. */ static ibool row_upd_changes_first_fields( @@ -234,7 +235,8 @@ row_upd_check_references_constraints( if (err != DB_SUCCESS) { if (got_s_lock) { - row_mysql_unfreeze_data_dictionary(trx); + row_mysql_unfreeze_data_dictionary( + trx); } mem_heap_free(heap); @@ -350,14 +352,15 @@ row_upd_index_entry_sys_field( } /*************************************************************** -Returns TRUE if row update changes size of some field in index -or if some field to be updated is stored externally in rec or update. */ +Returns TRUE if row update changes size of some field in index or if some +field to be updated is stored externally in rec or update. */ ibool -row_upd_changes_field_size( -/*=======================*/ +row_upd_changes_field_size_or_external( +/*===================================*/ /* out: TRUE if the update changes the size of - some field in index */ + some field in index or the field is external + in rec or update */ rec_t* rec, /* in: record in clustered index */ dict_index_t* index, /* in: clustered index */ upd_t* update) /* in: update vector */ @@ -820,72 +823,58 @@ void row_upd_index_replace_new_col_vals( /*===============================*/ dtuple_t* entry, /* in/out: index entry where replaced */ - dict_index_t* index, /* in: index; NOTE that may also be a + dict_index_t* index, /* in: index; NOTE that this may also be a non-clustered index */ - upd_t* update) /* in: update vector */ + upd_t* update, /* in: update vector */ + mem_heap_t* heap) /* in: memory heap to which we allocate and + copy the new values, set this as NULL if you + do not want allocation */ { + dict_field_t* field; upd_field_t* upd_field; dfield_t* dfield; dfield_t* new_val; - ulint field_no; - dict_index_t* clust_index; + ulint j; ulint i; ut_ad(index); - clust_index = dict_table_get_first_index(index->table); - dtuple_set_info_bits(entry, update->info_bits); - for (i = 0; i < upd_get_n_fields(update); i++) { + for (j = 0; j < dict_index_get_n_fields(index); j++) { - upd_field = upd_get_nth_field(update, i); + field = dict_index_get_nth_field(index, j); - field_no = dict_index_get_nth_col_pos(index, - dict_index_get_nth_col_no(clust_index, - upd_field->field_no)); - if (field_no != ULINT_UNDEFINED) { - dfield = dtuple_get_nth_field(entry, field_no); + for (i = 0; i < upd_get_n_fields(update); i++) { - new_val = &(upd_field->new_val); + upd_field = upd_get_nth_field(update, i); - dfield_set_data(dfield, new_val->data, new_val->len); + if (upd_field->field_no == field->col->clust_pos) { + + dfield = dtuple_get_nth_field(entry, j); + + new_val = &(upd_field->new_val); + + dfield_set_data(dfield, new_val->data, + new_val->len); + if (heap && new_val->len != UNIV_SQL_NULL) { + dfield->data = mem_heap_alloc(heap, + new_val->len); + ut_memcpy(dfield->data, new_val->data, + new_val->len); + } + + if (field->prefix_len > 0 + && new_val->len != UNIV_SQL_NULL + && new_val->len > field->prefix_len) { + + dfield->len = field->prefix_len; + } + } } } } -/*************************************************************** -Replaces the new column values stored in the update vector to the -clustered index entry given. */ - -void -row_upd_clust_index_replace_new_col_vals( -/*=====================================*/ - dtuple_t* entry, /* in/out: index entry where replaced */ - upd_t* update) /* in: update vector */ -{ - upd_field_t* upd_field; - dfield_t* dfield; - dfield_t* new_val; - ulint field_no; - ulint i; - - dtuple_set_info_bits(entry, update->info_bits); - - for (i = 0; i < upd_get_n_fields(update); i++) { - - upd_field = upd_get_nth_field(update, i); - - field_no = upd_field->field_no; - - dfield = dtuple_get_nth_field(entry, field_no); - - new_val = &(upd_field->new_val); - - dfield_set_data(dfield, new_val->data, new_val->len); - } -} - /*************************************************************** Checks if an update vector changes an ordering field of an index record. This function is fast if the update vector is short or the number of ordering @@ -931,9 +920,15 @@ row_upd_changes_ord_field_binary( upd_field = upd_get_nth_field(update, j); + /* Note that if the index field is a column prefix + then it may be that row does not contain an externally + stored part of the column value, and we cannot compare + the datas */ + if (col_pos == upd_field->field_no - && (row == NULL - || !dfield_datas_are_binary_equal( + && (row == NULL + || ind_field->prefix_len > 0 + || !dfield_datas_are_binary_equal( dtuple_get_nth_field(row, col_no), &(upd_field->new_val)))) { return(TRUE); @@ -978,8 +973,9 @@ row_upd_changes_some_index_ord_field_binary( } /*************************************************************** -Checks if an update vector changes some of the first fields of an index -record. */ +Checks if an update vector changes some of the first ordering fields of an +index record. This is only used in foreign key checks and we can assume +that index does not contain column prefixes. */ static ibool row_upd_changes_first_fields( @@ -1013,9 +1009,10 @@ row_upd_changes_first_fields( upd_field = upd_get_nth_field(update, j); if (col_pos == upd_field->field_no - && cmp_dfield_dfield( + && (ind_field->prefix_len > 0 + || 0 != cmp_dfield_dfield( dtuple_get_nth_field(entry, i), - &(upd_field->new_val))) { + &(upd_field->new_val)))) { return(TRUE); } } @@ -1204,7 +1201,7 @@ close_cur: } /* Build a new index entry */ - row_upd_index_replace_new_col_vals(entry, index, node->update); + row_upd_index_replace_new_col_vals(entry, index, node->update, NULL); /* Insert new index entry */ err = row_ins_index_entry(index, entry, NULL, 0, thr); @@ -1317,12 +1314,12 @@ row_upd_clust_rec_by_insert( entry = row_build_index_entry(node->row, index, heap); - row_upd_clust_index_replace_new_col_vals(entry, node->update); + row_upd_index_replace_new_col_vals(entry, index, node->update, NULL); row_upd_index_entry_sys_field(entry, index, DATA_TRX_ID, trx->id); /* If we return from a lock wait, for example, we may have - extern fields marked as not-owned in entry (marked if the + extern fields marked as not-owned in entry (marked in the if-branch above). We must unmark them. */ btr_cur_unmark_dtuple_extern_fields(entry, node->ext_vec, @@ -1702,9 +1699,9 @@ function_exit: /* Do some cleanup */ if (node->row != NULL) { - mem_heap_empty(node->heap); node->row = NULL; node->n_ext_vec = 0; + mem_heap_empty(node->heap); } node->state = UPD_NODE_UPDATE_CLUSTERED; diff --git a/innobase/row/row0vers.c b/innobase/row/row0vers.c index cd8b18e5e12..91aaba40812 100644 --- a/innobase/row/row0vers.c +++ b/innobase/row/row0vers.c @@ -27,6 +27,7 @@ Created 2/6/1997 Heikki Tuuri #include "row0upd.h" #include "rem0cmp.h" #include "read0read.h" +#include "lock0lock.h" /********************************************************************* Finds out if an active transaction has inserted or modified a secondary @@ -111,6 +112,14 @@ row_vers_impl_x_locked_off_kernel( return(NULL); } + if (!lock_check_trx_id_sanity(trx_id, clust_rec, clust_index, TRUE)) { + /* Corruption noticed: try to avoid a crash by returning */ + + mtr_commit(&mtr); + + return(NULL); + } + /* We look up if some earlier version of the clustered index record would require rec to be in a different state (delete marked or unmarked, or not existing). If there is such a version, then rec was @@ -177,7 +186,8 @@ row_vers_impl_x_locked_off_kernel( /* If we get here, we know that the trx_id transaction is still active and it has modified prev_version. Let us check - if prev_version would require rec to be in a different state. */ + if prev_version would require rec to be in a different + state. */ vers_del = rec_get_deleted_flag(prev_version); diff --git a/innobase/srv/srv0srv.c b/innobase/srv/srv0srv.c index 2a93ca966eb..d13d499dd17 100644 --- a/innobase/srv/srv0srv.c +++ b/innobase/srv/srv0srv.c @@ -286,6 +286,7 @@ ulint srv_test_n_mutexes = ULINT_MAX; i/o handler thread */ char* srv_io_thread_op_info[SRV_MAX_N_IO_THREADS]; +char* srv_io_thread_function[SRV_MAX_N_IO_THREADS]; time_t srv_last_monitor_time; @@ -2399,8 +2400,9 @@ srv_sprintf_innodb_monitor( srv_conc_n_threads, srv_conc_n_waiting_threads); #ifdef UNIV_LINUX buf += sprintf(buf, - "Main thread process no %lu, state: %s\n", + "Main thread process no. %lu, id %lu, state: %s\n", srv_main_thread_process_no, + srv_main_thread_id, srv_main_thread_op_info); #else buf += sprintf(buf, @@ -2464,8 +2466,8 @@ srv_lock_timeout_and_monitor_thread( ulint i; #ifdef UNIV_DEBUG_THREAD_CREATION - printf("Lock timeout thread starts\n"); - printf("Thread id %lu\n", os_thread_pf(os_thread_get_curr_id())); + printf("Lock timeout thread starts, id %lu\n", + os_thread_pf(os_thread_get_curr_id())); #endif UT_NOT_USED(arg); srv_last_monitor_time = time(NULL); @@ -2637,8 +2639,8 @@ srv_error_monitor_thread( UT_NOT_USED(arg); #ifdef UNIV_DEBUG_THREAD_CREATION - printf("Error monitor thread starts\n"); - printf("Thread id %lu\n", os_thread_pf(os_thread_get_curr_id())); + printf("Error monitor thread starts, id %lu\n", + os_thread_pf(os_thread_get_curr_id())); #endif loop: srv_error_monitor_active = TRUE; @@ -2760,8 +2762,8 @@ srv_master_thread( UT_NOT_USED(arg); #ifdef UNIV_DEBUG_THREAD_CREATION - printf("Master thread starts\n"); - printf("Thread id %lu\n", os_thread_pf(os_thread_get_curr_id())); + printf("Master thread starts, id %lu\n", + os_thread_pf(os_thread_get_curr_id())); #endif srv_main_thread_process_no = os_proc_get_number(); srv_main_thread_id = os_thread_pf(os_thread_get_curr_id()); diff --git a/innobase/srv/srv0start.c b/innobase/srv/srv0start.c index 1f278d82bc0..964e728b23c 100644 --- a/innobase/srv/srv0start.c +++ b/innobase/srv/srv0start.c @@ -415,8 +415,8 @@ io_handler_thread( segment = *((ulint*)arg); #ifdef UNIV_DEBUG_THREAD_CREATION - printf("Io handler thread %lu starts\n", segment); - printf("Thread id %lu\n", os_thread_pf(os_thread_get_curr_id())); + printf("Io handler thread %lu starts, id %lu\n", segment, + os_thread_pf(os_thread_get_curr_id())); #endif for (i = 0;; i++) { fil_aio_wait(segment); @@ -1492,7 +1492,9 @@ innobase_shutdown_for_mysql(void) } /* 1. Flush buffer pool to disk, write the current lsn to - the tablespace header(s), and copy all log data to archive */ + the tablespace header(s), and copy all log data to archive. + The step 1 is the real InnoDB shutdown. The remaining steps + just free data structures after the shutdown. */ logs_empty_and_mark_files_at_shutdown(); diff --git a/innobase/trx/trx0rec.c b/innobase/trx/trx0rec.c index 05e179e06a5..9453189d598 100644 --- a/innobase/trx/trx0rec.c +++ b/innobase/trx/trx0rec.c @@ -272,8 +272,8 @@ trx_undo_page_report_insert( mach_write_to_2(undo_page + TRX_UNDO_PAGE_HDR + TRX_UNDO_PAGE_FREE, ptr - undo_page); - /* Write the log entry to the REDO log of this change in the UNDO log */ - + /* Write the log entry to the REDO log of this change in the UNDO + log */ trx_undof_page_add_undo_rec_log(undo_page, first_free, ptr - undo_page, mtr); return(first_free); @@ -492,7 +492,8 @@ trx_undo_page_report_modify( /* Reserve 2 bytes for the pointer to the next undo log record */ ptr += 2; - /* Store first some general parameters to the undo log */ + /* Store first some general parameters to the undo log */ + if (update) { if (rec_get_deleted_flag(rec)) { type_cmpl = TRX_UNDO_UPD_DEL_REC; @@ -526,8 +527,7 @@ trx_undo_page_report_modify( /* Store the values of the system columns */ trx_id = dict_index_rec_get_sys_col(index, DATA_TRX_ID, rec); - roll_ptr = dict_index_rec_get_sys_col(index, DATA_ROLL_PTR, rec); - + roll_ptr = dict_index_rec_get_sys_col(index, DATA_ROLL_PTR, rec); len = mach_dulint_write_compressed(ptr, trx_id); ptr += len; @@ -632,7 +632,11 @@ trx_undo_page_report_modify( columns which occur as ordering fields in any index. This info is used in the purge of old versions where we use it to build and search the delete marked index records, to look if we can remove them from the - index tree. */ + index tree. Note that starting from 4.0.14 also externally stored + fields can be ordering in some index. But we always store at least + 384 first bytes locally to the clustered index record, which means + we can construct the column prefix fields in the index from the + stored data. */ if (!update || !(cmpl_info & UPD_NODE_NO_ORD_CHANGE)) { @@ -1408,11 +1412,11 @@ trx_undo_prev_version_build( return(DB_ERROR); } - if (row_upd_changes_field_size(rec, index, update)) { + if (row_upd_changes_field_size_or_external(rec, index, update)) { - entry = row_rec_to_index_entry(ROW_COPY_DATA, index, rec, heap); - - row_upd_clust_index_replace_new_col_vals(entry, update); + entry = row_rec_to_index_entry(ROW_COPY_DATA, index, rec, + heap); + row_upd_index_replace_new_col_vals(entry, index, update, heap); buf = mem_heap_alloc(heap, rec_get_converted_size(entry)); diff --git a/innobase/trx/trx0roll.c b/innobase/trx/trx0roll.c index a9f8c5ad22c..7d1b341221c 100644 --- a/innobase/trx/trx0roll.c +++ b/innobase/trx/trx0roll.c @@ -52,6 +52,11 @@ trx_general_rollback_for_mysql( que_thr_t* thr; roll_node_t* roll_node; + /* Tell Innobase server that there might be work for + utility threads: */ + + srv_active_wake_master_thread(); + trx_start_if_not_started(trx); heap = mem_heap_create(512); @@ -89,6 +94,11 @@ trx_general_rollback_for_mysql( ut_a(trx->error_state == DB_SUCCESS); + /* Tell Innobase server that there might be work for + utility threads: */ + + srv_active_wake_master_thread(); + return((int) trx->error_state); } @@ -110,20 +120,8 @@ trx_rollback_for_mysql( trx->op_info = (char *) "rollback"; - /* Tell Innobase server that there might be work for - utility threads: */ - - srv_active_wake_master_thread(); - err = trx_general_rollback_for_mysql(trx, FALSE, NULL); - trx_mark_sql_stat_end(trx); - - /* Tell Innobase server that there might be work for - utility threads: */ - - srv_active_wake_master_thread(); - trx->op_info = (char *) ""; return(err); @@ -147,25 +145,191 @@ trx_rollback_last_sql_stat_for_mysql( trx->op_info = (char *) "rollback of SQL statement"; - /* Tell Innobase server that there might be work for - utility threads: */ - - srv_active_wake_master_thread(); - err = trx_general_rollback_for_mysql(trx, TRUE, &(trx->last_sql_stat_start)); + /* The following call should not be needed, but we play safe: */ trx_mark_sql_stat_end(trx); - /* Tell Innobase server that there might be work for - utility threads: */ - - srv_active_wake_master_thread(); - trx->op_info = (char *) ""; return(err); } +/*********************************************************************** +Frees savepoint structs. */ + +void +trx_roll_savepoints_free( +/*=====================*/ + trx_t* trx, /* in: transaction handle */ + trx_named_savept_t* savep) /* in: free all savepoints > this one; + if this is NULL, free all savepoints + of trx */ +{ + trx_named_savept_t* next_savep; + + if (savep == NULL) { + savep = UT_LIST_GET_FIRST(trx->trx_savepoints); + } else { + savep = UT_LIST_GET_NEXT(trx_savepoints, savep); + } + + while (savep != NULL) { + next_savep = UT_LIST_GET_NEXT(trx_savepoints, savep); + + UT_LIST_REMOVE(trx_savepoints, trx->trx_savepoints, savep); + mem_free(savep->name); + mem_free(savep); + + savep = next_savep; + } +} + +/*********************************************************************** +Rolls back a transaction back to a named savepoint. Modifications after the +savepoint are undone but InnoDB does NOT release the corresponding locks +which are stored in memory. If a lock is 'implicit', that is, a new inserted +row holds a lock where the lock information is carried by the trx id stored in +the row, these locks are naturally released in the rollback. Savepoints which +were set after this savepoint are deleted. */ + +ulint +trx_rollback_to_savepoint_for_mysql( +/*================================*/ + /* out: if no savepoint + of the name found then + DB_NO_SAVEPOINT, + otherwise DB_SUCCESS */ + trx_t* trx, /* in: transaction handle */ + char* savepoint_name, /* in: savepoint name */ + ib_longlong* mysql_binlog_cache_pos) /* out: the MySQL binlog cache + position corresponding to this + savepoint; MySQL needs this + information to remove the + binlog entries of the queries + executed after the savepoint */ +{ + trx_named_savept_t* savep; + ulint err; + + savep = UT_LIST_GET_FIRST(trx->trx_savepoints); + + while (savep != NULL) { + if (0 == ut_strcmp(savep->name, savepoint_name)) { + /* Found */ + break; + } + savep = UT_LIST_GET_NEXT(trx_savepoints, savep); + } + + if (savep == NULL) { + + return(DB_NO_SAVEPOINT); + } + + if (trx->conc_state == TRX_NOT_STARTED) { + ut_print_timestamp(stderr); + fprintf(stderr, +" InnoDB: Error: transaction has a savepoint %s though it is not started\n", + savep->name); + return(DB_ERROR); + } + + /* We can now free all savepoints strictly later than this one */ + + trx_roll_savepoints_free(trx, savep); + + *mysql_binlog_cache_pos = savep->mysql_binlog_cache_pos; + + trx->op_info = (char *) "rollback to a savepoint"; + + err = trx_general_rollback_for_mysql(trx, TRUE, &(savep->savept)); + + /* Store the current undo_no of the transaction so that we know where + to roll back if we have to roll back the next SQL statement: */ + + trx_mark_sql_stat_end(trx); + + trx->op_info = (char *) ""; + + return(err); +} + +/*********************************************************************** +Creates a named savepoint. If the transaction is not yet started, starts it. +If there is already a savepoint of the same name, this call erases that old +savepoint and replaces it with a new. Savepoints are deleted in a transaction +commit or rollback. */ + +ulint +trx_savepoint_for_mysql( +/*====================*/ + /* out: always DB_SUCCESS */ + trx_t* trx, /* in: transaction handle */ + char* savepoint_name, /* in: savepoint name */ + ib_longlong binlog_cache_pos) /* in: MySQL binlog cache + position corresponding to this + connection at the time of the + savepoint */ +{ + trx_named_savept_t* savep; + + ut_a(trx); + ut_a(savepoint_name); + + trx_start_if_not_started(trx); + + savep = UT_LIST_GET_FIRST(trx->trx_savepoints); + + while (savep != NULL) { + if (0 == ut_strcmp(savep->name, savepoint_name)) { + /* Found */ + break; + } + savep = UT_LIST_GET_NEXT(trx_savepoints, savep); + } + + if (savep) { + /* There is a savepoint with the same name: free that */ + + UT_LIST_REMOVE(trx_savepoints, trx->trx_savepoints, savep); + + mem_free(savep->name); + mem_free(savep); + } + + /* Create a new savepoint and add it as the last in the list */ + + savep = mem_alloc(sizeof(trx_named_savept_t)); + + savep->name = mem_alloc(1 + ut_strlen(savepoint_name)); + ut_memcpy(savep->name, savepoint_name, 1 + ut_strlen(savepoint_name)); + + savep->savept = trx_savept_take(trx); + + savep->mysql_binlog_cache_pos = binlog_cache_pos; + + UT_LIST_ADD_LAST(trx_savepoints, trx->trx_savepoints, savep); + + return(DB_SUCCESS); +} + +/*********************************************************************** +Returns a transaction savepoint taken at this point in time. */ + +trx_savept_t +trx_savept_take( +/*============*/ + /* out: savepoint */ + trx_t* trx) /* in: transaction */ +{ + trx_savept_t savept; + + savept.least_undo_no = trx->undo_no; + + return(savept); +} + /*********************************************************************** Rollback or clean up transactions which have no user session. If the transaction already was committed, then we clean up a possible insert @@ -325,22 +489,6 @@ loop: goto loop; } - -/*********************************************************************** -Returns a transaction savepoint taken at this point in time. */ - -trx_savept_t -trx_savept_take( -/*============*/ - /* out: savepoint */ - trx_t* trx) /* in: transaction */ -{ - trx_savept_t savept; - - savept.least_undo_no = trx->undo_no; - - return(savept); -} /*********************************************************************** Creates an undo number array. */ diff --git a/innobase/trx/trx0sys.c b/innobase/trx/trx0sys.c index 51aad60d3e2..f1b03fff3bd 100644 --- a/innobase/trx/trx0sys.c +++ b/innobase/trx/trx0sys.c @@ -321,8 +321,8 @@ trx_sys_doublewrite_restore_corrupt_pages(void) for (i = 0; i < TRX_SYS_DOUBLEWRITE_BLOCK_SIZE * 2; i++) { - space_id = mach_read_from_4(page + FIL_PAGE_SPACE); page_no = mach_read_from_4(page + FIL_PAGE_OFFSET); + space_id = 0; if (!fil_check_adress_in_tablespace(space_id, page_no)) { fprintf(stderr, diff --git a/innobase/trx/trx0trx.c b/innobase/trx/trx0trx.c index d73d6327d76..9233e861784 100644 --- a/innobase/trx/trx0trx.c +++ b/innobase/trx/trx0trx.c @@ -135,6 +135,8 @@ trx_create( trx->lock_heap = mem_heap_create_in_buffer(256); UT_LIST_INIT(trx->trx_locks); + UT_LIST_INIT(trx->trx_savepoints); + trx->dict_operation_lock_mode = 0; trx->has_search_latch = FALSE; trx->search_latch_timeout = BTR_SEA_TIMEOUT; @@ -807,6 +809,9 @@ trx_commit_off_kernel( mutex_enter(&kernel_mutex); } + /* Free savepoints */ + trx_roll_savepoints_free(trx, NULL); + trx->conc_state = TRX_NOT_STARTED; trx->rseg = NULL; trx->undo_no = ut_dulint_zero; diff --git a/innobase/ut/ut0mem.c b/innobase/ut/ut0mem.c index 174ae4cc6bb..f5d207d8bba 100644 --- a/innobase/ut/ut0mem.c +++ b/innobase/ut/ut0mem.c @@ -166,7 +166,7 @@ ut_free( } /************************************************************************** -Frees all allocated memory not freed yet. */ +Frees in shutdown all allocated memory not freed yet. */ void ut_free_all_mem(void) @@ -174,7 +174,7 @@ ut_free_all_mem(void) { ut_mem_block_t* block; - os_fast_mutex_lock(&ut_list_mutex); + os_fast_mutex_free(&ut_list_mutex); while ((block = UT_LIST_GET_FIRST(ut_mem_block_list))) { @@ -187,11 +187,11 @@ ut_free_all_mem(void) free(block); } - os_fast_mutex_unlock(&ut_list_mutex); - - ut_a(ut_total_allocated_memory == 0); - - os_fast_mutex_free(&ut_list_mutex); + if (ut_total_allocated_memory != 0) { + fprintf(stderr, +"InnoDB: Warning: after shutdown total allocated memory is %lu\n", + ut_total_allocated_memory); + } } /************************************************************************** diff --git a/innobase/ut/ut0ut.c b/innobase/ut/ut0ut.c index 95037ec3570..06bfb5c45ba 100644 --- a/innobase/ut/ut0ut.c +++ b/innobase/ut/ut0ut.c @@ -53,6 +53,8 @@ ut_get_high32( ulint a) /* in: ulint */ { #if SIZEOF_LONG == 4 + UT_NOT_USED(a); + return 0; #else return(a >> 32); diff --git a/mysql-test/r/innodb.result b/mysql-test/r/innodb.result index e2dea324ff2..9d1c232e830 100644 --- a/mysql-test/r/innodb.result +++ b/mysql-test/r/innodb.result @@ -787,16 +787,6 @@ id id3 100 2 UNLOCK TABLES; DROP TABLE t1; -create table t1 (a char(20), unique (a(5))) type=innodb; -Incorrect sub part key. The used key part isn't a string, the used length is longer than the key part or the table handler doesn't support unique sub keys -create table t1 (a char(20), index (a(5))) type=innodb; -show create table t1; -Table Create Table -t1 CREATE TABLE `t1` ( - `a` char(20) default NULL, - KEY `a` (`a`) -) TYPE=InnoDB -drop table t1; create temporary table t1 (a int not null auto_increment, primary key(a)) type=innodb; insert into t1 values (NULL),(NULL),(NULL); delete from t1 where a=3; diff --git a/mysql-test/t/innodb.test b/mysql-test/t/innodb.test index dc3c76f1a91..cf203d87c8b 100644 --- a/mysql-test/t/innodb.test +++ b/mysql-test/t/innodb.test @@ -471,15 +471,6 @@ select id,id3 from t1; UNLOCK TABLES; DROP TABLE t1; -# -# Test prefix key -# ---error 1089 -create table t1 (a char(20), unique (a(5))) type=innodb; -create table t1 (a char(20), index (a(5))) type=innodb; -show create table t1; -drop table t1; - # # Test using temporary table and auto_increment # diff --git a/sql/ha_innodb.cc b/sql/ha_innodb.cc index fd030fff091..28e61f44111 100644 --- a/sql/ha_innodb.cc +++ b/sql/ha_innodb.cc @@ -129,12 +129,45 @@ static void innobase_print_error(const char* db_errpfx, char* buffer); /* General functions */ +/********************************************************************** +Save some CPU by testing the value of srv_thread_concurrency in inline +functions. */ +inline +void +innodb_srv_conc_enter_innodb( +/*=========================*/ + trx_t* trx) /* in: transaction handle */ +{ + if (srv_thread_concurrency >= 500) { + + return; + } + + srv_conc_enter_innodb(trx); +} + +/********************************************************************** +Save some CPU by testing the value of srv_thread_concurrency in inline +functions. */ +inline +void +innodb_srv_conc_exit_innodb( +/*========================*/ + trx_t* trx) /* in: transaction handle */ +{ + if (srv_thread_concurrency >= 500) { + + return; + } + + srv_conc_exit_innodb(trx); +} + /********************************************************************** Releases possible search latch and InnoDB thread FIFO ticket. These should be released at each SQL statement end, and also when mysqld passes the control to the client. It does no harm to release these also in the middle of an SQL statement. */ -static inline void innobase_release_stat_resources( @@ -183,7 +216,9 @@ innobase_active_small(void) } /************************************************************************ -Converts an InnoDB error code to a MySQL error code. */ +Converts an InnoDB error code to a MySQL error code and also tells to MySQL +about a possible transaction rollback inside InnoDB caused by a lock wait +timeout or a deadlock. */ static int convert_error_code_to_mysql( @@ -206,10 +241,10 @@ convert_error_code_to_mysql( } else if (error == (int) DB_ERROR) { - return(HA_ERR_NO_ACTIVE_RECORD); + return(-1); /* unspecified error */ } else if (error == (int) DB_DEADLOCK) { - /* Since we roll back the whole transaction, we must + /* Since we rolled back the whole transaction, we must tell it also to MySQL so that MySQL knows to empty the cached binlog for this transaction */ @@ -221,11 +256,10 @@ convert_error_code_to_mysql( } else if (error == (int) DB_LOCK_WAIT_TIMEOUT) { - /* Since we roll back the whole transaction, we must + /* Since we rolled back the whole transaction, we must tell it also to MySQL so that MySQL knows to empty the cached binlog for this transaction */ - if (thd) { ha_rollback(thd); } @@ -271,6 +305,9 @@ convert_error_code_to_mysql( } else if (error == (int) DB_CORRUPTION) { return(HA_ERR_CRASHED); + } else if (error == (int) DB_NO_SAVEPOINT) { + + return(HA_ERR_NO_SAVEPOINT); } else { return(-1); // Unknown error } @@ -941,18 +978,23 @@ innobase_commit( DBUG_ENTER("innobase_commit"); DBUG_PRINT("trans", ("ending transaction")); - /* The flag thd->transaction.all.innodb_active_trans is set to 1 - in ::external_lock and ::start_stmt, and it is only set to 0 in - a commit or a rollback. If it is 0 we know there cannot be resources - to be freed and we can return immediately. */ - - if (thd->transaction.all.innodb_active_trans == 0) { - - DBUG_RETURN(0); - } - trx = check_trx_exists(thd); + /* The flag thd->transaction.all.innodb_active_trans is set to 1 in + ::external_lock, ::start_stmt, and innobase_savepoint, and it is only + set to 0 in a commit or a rollback. If it is 0 we know there cannot be + resources to be freed and we could return immediately. For the time + being we play safe and do the cleanup though there should be nothing + to clean up. */ + + if (thd->transaction.all.innodb_active_trans == 0 + && trx->conc_state != TRX_NOT_STARTED) { + + fprintf(stderr, +"InnoDB: Error: thd->transaction.all.innodb_active_trans == 0\n" +"InnoDB: but trx->conc_state != TRX_NOT_STARTED\n"); + } + if (trx_handle != (void*)&innodb_dummy_stmt_trx_handle || (!(thd->options & (OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN)))) { @@ -964,9 +1006,9 @@ innobase_commit( /* If we had reserved the auto-inc lock for some table in this SQL statement we release it now */ - srv_conc_enter_innodb(trx); + innodb_srv_conc_enter_innodb(trx); row_unlock_table_autoinc_for_mysql(trx); - srv_conc_exit_innodb(trx); + innodb_srv_conc_exit_innodb(trx); } /* Store the current undo_no of the transaction so that we know where to roll back if we have to roll back the next @@ -1050,7 +1092,7 @@ innobase_commit_complete( } /********************************************************************* -Rolls back a transaction or the latest SQL statement in an InnoDB database. */ +Rolls back a transaction or the latest SQL statement. */ int innobase_rollback( @@ -1071,25 +1113,27 @@ innobase_rollback( trx = check_trx_exists(thd); if (trx->auto_inc_lock) { - - /* If we had reserved the auto-inc lock for - some table in this SQL statement, we release it now */ - - srv_conc_enter_innodb(trx); + /* If we had reserved the auto-inc lock for some table (if + we come here to roll back the latest SQL statement) we + release it now before a possibly lengthy rollback */ + + innodb_srv_conc_enter_innodb(trx); row_unlock_table_autoinc_for_mysql(trx); - srv_conc_exit_innodb(trx); + innodb_srv_conc_exit_innodb(trx); } - srv_conc_enter_innodb(trx); + innodb_srv_conc_enter_innodb(trx); + + if (trx_handle != (void*)&innodb_dummy_stmt_trx_handle + || (!(thd->options & (OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN)))) { - if (trx_handle != (void*)&innodb_dummy_stmt_trx_handle) { error = trx_rollback_for_mysql(trx); - thd->transaction.all.innodb_active_trans=0; + thd->transaction.all.innodb_active_trans = 0; } else { error = trx_rollback_last_sql_stat_for_mysql(trx); } - srv_conc_exit_innodb(trx); + innodb_srv_conc_exit_innodb(trx); /* Release a possible FIFO ticket and search latch */ innobase_release_stat_resources(trx); @@ -1097,6 +1141,83 @@ innobase_rollback( DBUG_RETURN(convert_error_code_to_mysql(error, NULL)); } +/********************************************************************* +Rolls back a transaction to a savepoint. */ + +int +innobase_rollback_to_savepoint( +/*===========================*/ + /* out: 0 if success, HA_ERR_NO_SAVEPOINT if + no savepoint with the given name */ + THD* thd, /* in: handle to the MySQL thread of the user + whose transaction should be rolled back */ + char* savepoint_name, /* in: savepoint name */ + my_off_t* binlog_cache_pos)/* out: position which corresponds to the + savepoint in the binlog cache of this + transaction, not defined if error */ +{ + ib_longlong mysql_binlog_cache_pos; + int error = 0; + trx_t* trx; + + DBUG_ENTER("innobase_rollback_to_savepoint"); + + trx = check_trx_exists(thd); + + innodb_srv_conc_enter_innodb(trx); + + error = trx_rollback_to_savepoint_for_mysql(trx, savepoint_name, + &mysql_binlog_cache_pos); + innodb_srv_conc_exit_innodb(trx); + + *binlog_cache_pos = (my_off_t)mysql_binlog_cache_pos; + + /* Release a possible FIFO ticket and search latch */ + innobase_release_stat_resources(trx); + + DBUG_RETURN(convert_error_code_to_mysql(error, NULL)); +} + +/********************************************************************* +Sets a transaction savepoint. */ + +int +innobase_savepoint( +/*===============*/ + /* out: always 0, that is, always succeeds */ + THD* thd, /* in: handle to the MySQL thread */ + char* savepoint_name, /* in: savepoint name */ + my_off_t binlog_cache_pos)/* in: offset up to which the current + transaction has cached log entries to its + binlog cache, not defined if no transaction + active, or we are in the autocommit state, or + binlogging is not switched on */ +{ + int error = 0; + trx_t* trx; + + DBUG_ENTER("innobase_savepoint"); + + if (!(thd->options & (OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN))) { + /* In the autocommit state there is no sense to set a + savepoint: we return immediate success */ + DBUG_RETURN(0); + } + + trx = check_trx_exists(thd); + + /* Setting a savepoint starts a transaction inside InnoDB since + it allocates resources for it (memory to store the savepoint name, + for example) */ + + thd->transaction.all.innodb_active_trans = 1; + + error = trx_savepoint_for_mysql(trx, savepoint_name, + (ib_longlong)binlog_cache_pos); + + DBUG_RETURN(convert_error_code_to_mysql(error, NULL)); +} + /********************************************************************* Frees a possible InnoDB trx object associated with the current THD. */ @@ -1220,7 +1341,6 @@ ha_innobase::open( { dict_table_t* ib_table; int error = 0; - uint buff_len; char norm_name[1000]; DBUG_ENTER("ha_innobase::open"); @@ -1245,11 +1365,11 @@ ha_innobase::open( fields when packed actually became 1 byte longer, when we also stored the string length as the first byte. */ - buff_len = table->reclength + table->max_key_length + upd_and_key_val_buff_len = table->reclength + table->max_key_length + MAX_REF_PARTS * 3; if (!(mysql_byte*) my_multi_malloc(MYF(MY_WME), - &upd_buff, buff_len, - &key_val_buff, buff_len, + &upd_buff, upd_and_key_val_buff_len, + &key_val_buff, upd_and_key_val_buff_len, NullS)) { free_share(share); DBUG_RETURN(1); @@ -1500,6 +1620,10 @@ innobase_mysql_cmp( case FIELD_TYPE_STRING: case FIELD_TYPE_VAR_STRING: + case FIELD_TYPE_TINY_BLOB: + case FIELD_TYPE_MEDIUM_BLOB: + case FIELD_TYPE_BLOB: + case FIELD_TYPE_LONG_BLOB: ret = my_sortncmp((const char*) a, a_length, (const char*) b, b_length); if (ret < 0) { @@ -1526,7 +1650,7 @@ get_innobase_type_from_mysql_type( /* out: DATA_BINARY, DATA_VARCHAR, ... */ Field* field) /* in: MySQL field */ { - /* The following asserts check that MySQL type code fits in + /* The following asserts check that the MySQL type code fits in 8 bits: this is used in ibuf and also when DATA_NOT_NULL is ORed to the type */ @@ -1537,6 +1661,8 @@ get_innobase_type_from_mysql_type( DBUG_ASSERT((ulint)FIELD_TYPE_DECIMAL < 256); switch (field->type()) { + /* NOTE that we only allow string types in DATA_MYSQL + and DATA_VARMYSQL */ case FIELD_TYPE_VAR_STRING: if (field->flags & BINARY_FLAG) { return(DATA_BINARY); @@ -1607,33 +1733,98 @@ ha_innobase::store_key_val_for_row( KEY_PART_INFO* key_part = key_info->key_part; KEY_PART_INFO* end = key_part + key_info->key_parts; char* buff_start = buff; + enum_field_types mysql_type; + Field* field; + ulint blob_len; + byte* blob_data; + ibool is_null; DBUG_ENTER("store_key_val_for_row"); + /* The format for storing a key field in MySQL is the following: + + 1. If the column can be NULL, then in the first byte we put 1 if the + field value is NULL, 0 otherwise. + + 2. If the column is of a BLOB type (it must be a column prefix field + in this case), then we put the length of the data in the field to the + next 2 bytes, in the little-endian format. If the field is SQL NULL, + then these 2 bytes are set to 0. Note that the length of data in the + field is <= column prefix length. + + 3. In a column prefix field, prefix_len next bytes are reserved for + data. In a normal field the max field length next bytes are reserved + for data. For a VARCHAR(n) the max field length is n. If the stored + value is the SQL NULL then these data bytes are set to 0. */ + + /* We have to zero-fill the 'ref' buffer so that MySQL is able to + use a simple memcmp to compare two key values to determine if they + are equal */ + + bzero(buff, ref_length); + for (; key_part != end; key_part++) { + is_null = FALSE; if (key_part->null_bit) { - /* Store 0 if the key part is a NULL part */ - if (record[key_part->null_offset] & key_part->null_bit) { - *buff++ = 1; - continue; - } - - *buff++ = 0; + *buff = 1; + is_null = TRUE; + } else { + *buff = 0; + } + buff++; } - memcpy(buff, record + key_part->offset, key_part->length); - buff += key_part->length; - } + field = key_part->field; + mysql_type = field->type(); - /* - We have to zero-fill the 'ref' buffer so that MySQL is able to - use a simple memcmp to compare two key values to determine if they - are equal - */ - bzero(buff, (ref_length- (uint) (buff - buff_start))); + if (mysql_type == FIELD_TYPE_TINY_BLOB + || mysql_type == FIELD_TYPE_MEDIUM_BLOB + || mysql_type == FIELD_TYPE_BLOB + || mysql_type == FIELD_TYPE_LONG_BLOB) { + + ut_a(key_part->key_part_flag & HA_PART_KEY); + + if (is_null) { + buff += key_part->length + 2; + + continue; + } + + blob_data = row_mysql_read_blob_ref(&blob_len, + (byte*) (record + + (ulint)get_field_offset(table, field)), + (ulint) field->pack_length()); + + ut_a(get_field_offset(table, field) + == key_part->offset); + if (blob_len > key_part->length) { + blob_len = key_part->length; + } + + /* MySQL reserves 2 bytes for the length and the + storage of the number is little-endian */ + + ut_a(blob_len < 256); + *buff = blob_len; + buff += 2; + + memcpy(buff, blob_data, blob_len); + + buff += key_part->length; + } else { + if (is_null) { + buff += key_part->length; + + continue; + } + memcpy(buff, record + key_part->offset, + key_part->length); + buff += key_part->length; + } + } DBUG_RETURN((uint)(buff - buff_start)); } @@ -1905,9 +2096,9 @@ ha_innobase::write_row( The lock is released at each SQL statement's end. */ - srv_conc_enter_innodb(prebuilt->trx); + innodb_srv_conc_enter_innodb(prebuilt->trx); error = row_lock_table_autoinc_for_mysql(prebuilt); - srv_conc_exit_innodb(prebuilt->trx); + innodb_srv_conc_exit_innodb(prebuilt->trx); if (error != DB_SUCCESS) { @@ -1918,14 +2109,15 @@ ha_innobase::write_row( dict_table_autoinc_update(prebuilt->table, auto_inc); } else { - srv_conc_enter_innodb(prebuilt->trx); + innodb_srv_conc_enter_innodb(prebuilt->trx); if (!prebuilt->trx->auto_inc_lock) { error = row_lock_table_autoinc_for_mysql( prebuilt); if (error != DB_SUCCESS) { - srv_conc_exit_innodb(prebuilt->trx); + innodb_srv_conc_exit_innodb( + prebuilt->trx); error = convert_error_code_to_mysql( error, user_thd); @@ -1939,7 +2131,7 @@ ha_innobase::write_row( auto_inc = dict_table_autoinc_get(prebuilt->table); incremented_auto_inc_counter = TRUE; - srv_conc_exit_innodb(prebuilt->trx); + innodb_srv_conc_exit_innodb(prebuilt->trx); /* We can give the new value for MySQL to place in the field */ @@ -1962,11 +2154,11 @@ ha_innobase::write_row( build_template(prebuilt, NULL, table, ROW_MYSQL_WHOLE_ROW); } - srv_conc_enter_innodb(prebuilt->trx); + innodb_srv_conc_enter_innodb(prebuilt->trx); error = row_insert_for_mysql((byte*) record, prebuilt); - srv_conc_exit_innodb(prebuilt->trx); + innodb_srv_conc_exit_innodb(prebuilt->trx); if (error != DB_SUCCESS) { /* If the insert did not succeed we restore the value of @@ -2037,7 +2229,6 @@ innobase_convert_and_store_changed_col( while (len > 0 && data[len - 1] == ' ') { len--; } - } else if (col_type == DATA_INT) { /* Store integer data in InnoDB in a big-endian format, sign bit negated, if signed */ @@ -2075,9 +2266,11 @@ calc_row_difference( struct st_table* table, /* in: table in MySQL data dictionary */ mysql_byte* upd_buff, /* in: buffer to use */ + ulint buff_len, /* in: buffer length */ row_prebuilt_t* prebuilt, /* in: InnoDB prebuilt struct */ THD* thd) /* in: user thread */ { + mysql_byte* original_upd_buff = upd_buff; Field* field; uint n_fields; ulint o_len; @@ -2159,12 +2352,13 @@ calc_row_difference( (prebuilt->table->cols + i)->clust_pos; n_changed++; } - ; } uvect->n_fields = n_changed; uvect->info_bits = 0; + ut_a(buf <= (byte*)original_upd_buff + buff_len); + return(0); } @@ -2213,17 +2407,19 @@ ha_innobase::update_row( (uses upd_buff of the handle) */ calc_row_difference(uvect, (mysql_byte*) old_row, new_row, table, - upd_buff, prebuilt, user_thd); + upd_buff, (ulint)upd_and_key_val_buff_len, + prebuilt, user_thd); + /* This is not a delete */ prebuilt->upd_node->is_delete = FALSE; assert(prebuilt->template_type == ROW_MYSQL_WHOLE_ROW); - srv_conc_enter_innodb(prebuilt->trx); + innodb_srv_conc_enter_innodb(prebuilt->trx); error = row_update_for_mysql((byte*) old_row, prebuilt); - srv_conc_exit_innodb(prebuilt->trx); + innodb_srv_conc_exit_innodb(prebuilt->trx); error = convert_error_code_to_mysql(error, user_thd); @@ -2267,11 +2463,11 @@ ha_innobase::delete_row( prebuilt->upd_node->is_delete = TRUE; - srv_conc_enter_innodb(prebuilt->trx); + innodb_srv_conc_enter_innodb(prebuilt->trx); error = row_update_for_mysql((byte*) record, prebuilt); - srv_conc_exit_innodb(prebuilt->trx); + innodb_srv_conc_exit_innodb(prebuilt->trx); error = convert_error_code_to_mysql(error, user_thd); @@ -2459,10 +2655,11 @@ ha_innobase::index_read( prebuilt->search_tuple */ row_sel_convert_mysql_key_to_innobase(prebuilt->search_tuple, - (byte*) key_val_buff, - index, - (byte*) key_ptr, - (ulint) key_len); + (byte*) key_val_buff, + (ulint)upd_and_key_val_buff_len, + index, + (byte*) key_ptr, + (ulint) key_len); } else { /* We position the cursor to the last or the first entry in the index */ @@ -2484,11 +2681,11 @@ ha_innobase::index_read( last_match_mode = match_mode; - srv_conc_enter_innodb(prebuilt->trx); + innodb_srv_conc_enter_innodb(prebuilt->trx); ret = row_search_for_mysql((byte*) buf, mode, prebuilt, match_mode, 0); - srv_conc_exit_innodb(prebuilt->trx); + innodb_srv_conc_exit_innodb(prebuilt->trx); if (ret == DB_SUCCESS) { error = 0; @@ -2632,11 +2829,11 @@ ha_innobase::general_fetch( ut_a(prebuilt->trx == (trx_t*) current_thd->transaction.all.innobase_tid); - srv_conc_enter_innodb(prebuilt->trx); + innodb_srv_conc_enter_innodb(prebuilt->trx); ret = row_search_for_mysql((byte*)buf, 0, prebuilt, match_mode, direction); - srv_conc_exit_innodb(prebuilt->trx); + innodb_srv_conc_exit_innodb(prebuilt->trx); if (ret == DB_SUCCESS) { error = 0; @@ -2939,7 +3136,6 @@ ha_innobase::position( } } - /********************************************************************* Creates a table definition to an InnoDB database. */ static @@ -2958,6 +3154,8 @@ create_table_def( ulint col_type; ulint nulls_allowed; ulint unsigned_type; + ulint binary_type; + ulint nonlatin1_type; ulint i; DBUG_ENTER("create_table_def"); @@ -2986,9 +3184,23 @@ create_table_def( unsigned_type = 0; } + if (strcmp(default_charset_info->name, "latin1") != 0) { + nonlatin1_type = DATA_NONLATIN1; + } else { + nonlatin1_type = 0; + } + + if (field->flags & BINARY_FLAG) { + binary_type = DATA_BINARY_TYPE; + nonlatin1_type = 0; + } else { + binary_type = 0; + } + dict_mem_table_add_col(table, (char*) field->field_name, col_type, (ulint)field->type() - | nulls_allowed | unsigned_type, + | nulls_allowed | unsigned_type + | nonlatin1_type | binary_type, field->pack_length(), 0); } @@ -3011,6 +3223,7 @@ create_index( const char* table_name, /* in: table name */ uint key_num) /* in: index number */ { + Field* field; dict_index_t* index; int error; ulint n_fields; @@ -3020,6 +3233,7 @@ create_index( ulint col_type; ulint prefix_len; ulint i; + ulint j; DBUG_ENTER("create_index"); @@ -3046,31 +3260,63 @@ create_index( for (i = 0; i < n_fields; i++) { key_part = key->key_part + i; - if (key_part->length != key_part->field->pack_length()) { + /* (The flag HA_PART_KEY denotes in MySQL a column prefix + field in an index: we only store a specified number of first + bytes of the column to the index field.) The flag does not + seem to be properly set by MySQL. Let us fall back on testing + the length of the key part versus the column. */ + + field = NULL; + for (j = 0; j < form->fields; j++) { + + field = form->field[j]; + + if (strlen(field->field_name) + == strlen(key_part->field->field_name) + && 0 == ut_cmp_in_lower_case( + (char*)field->field_name, + (char*)key_part->field->field_name, + strlen(field->field_name))) { + /* Found the corresponding column */ + + break; + } + } + + ut_a(j < form->fields); + + col_type = get_innobase_type_from_mysql_type(key_part->field); + + if (DATA_BLOB == col_type + || key_part->length < field->pack_length()) { + prefix_len = key_part->length; - col_type = get_innobase_type_from_mysql_type( - key_part->field); if (col_type == DATA_INT || col_type == DATA_FLOAT || col_type == DATA_DOUBLE || col_type == DATA_DECIMAL) { fprintf(stderr, "InnoDB: error: MySQL is trying to create a column prefix index field\n" -"InnoDB: on an inappropriate data type %lu. Table name %s, column name %s.\n", - col_type, table_name, - key_part->field->field_name); +"InnoDB: on an inappropriate data type. Table name %s, column name %s.\n", + table_name, key_part->field->field_name); prefix_len = 0; } } else { prefix_len = 0; - } + } + + if (prefix_len >= DICT_MAX_COL_PREFIX_LEN) { + DBUG_RETURN(-1); + } /* We assume all fields should be sorted in ascending order, hence the '0': */ + dict_mem_index_add_field(index, - (char*) key_part->field->field_name, 0); + (char*) key_part->field->field_name, + 0, prefix_len); } error = row_create_index_for_mysql(index, trx); @@ -3536,6 +3782,8 @@ ha_innobase::records_in_range( table->reclength + table->max_key_length + 100, MYF(MY_WME)); + ulint buff2_len = table->reclength + + table->max_key_length + 100; dtuple_t* range_start; dtuple_t* range_end; ib_longlong n_rows; @@ -3572,12 +3820,15 @@ ha_innobase::records_in_range( dict_index_copy_types(range_end, index, key->key_parts); row_sel_convert_mysql_key_to_innobase( - range_start, (byte*) key_val_buff, index, + range_start, (byte*) key_val_buff, + (ulint)upd_and_key_val_buff_len, + index, (byte*) start_key, (ulint) start_key_len); row_sel_convert_mysql_key_to_innobase( - range_end, (byte*) key_val_buff2, index, + range_end, (byte*) key_val_buff2, + buff2_len, index, (byte*) end_key, (ulint) end_key_len); @@ -3787,8 +4038,32 @@ ha_innobase::info( } for (i = 0; i < table->keys; i++) { + if (index == NULL) { + ut_print_timestamp(stderr); + fprintf(stderr, +" InnoDB: Error: table %s contains less indexes inside InnoDB\n" +"InnoDB: than are defined in the MySQL .frm file. Have you mixed up\n" +"InnoDB: .frm files from different installations? See section\n" +"InnoDB: 15.1 at http://www.innodb.com/ibman.html\n", + ib_table->name); + break; + } + for (j = 0; j < table->key_info[i].key_parts; j++) { + if (j + 1 > index->n_uniq) { + ut_print_timestamp(stderr); + fprintf(stderr, +" InnoDB: Error: index %s of %s has %lu columns unique inside InnoDB\n" +"InnoDB: but MySQL is asking statistics for %lu columns. Have you mixed up\n" +"InnoDB: .frm files from different installations? See section\n" +"InnoDB: 15.1 at http://www.innodb.com/ibman.html\n", + index->name, + ib_table->name, index->n_uniq, + j + 1); + break; + } + if (index->stat_n_diff_key_vals[j + 1] == 0) { rec_per_key = records; @@ -4046,10 +4321,11 @@ ha_innobase::reset(void) } /********************************************************************** -MySQL calls this function at the start of each SQL statement. Inside LOCK -TABLES the ::external_lock method does not work to mark SQL statement -borders. Note also a special case: if a temporary table is created inside -LOCK TABLES, MySQL has not called external_lock() at all on that table. */ +MySQL calls this function at the start of each SQL statement inside LOCK +TABLES. Inside LOCK TABLES the ::external_lock method does not work to +mark SQL statement borders. Note also a special case: if a temporary table +is created inside LOCK TABLES, MySQL has not called external_lock() at all +on that table. */ int ha_innobase::start_stmt( @@ -4448,9 +4724,9 @@ ha_innobase::innobase_read_and_init_auto_inc( return(0); } - srv_conc_enter_innodb(prebuilt->trx); + innodb_srv_conc_enter_innodb(prebuilt->trx); error = row_lock_table_autoinc_for_mysql(prebuilt); - srv_conc_exit_innodb(prebuilt->trx); + innodb_srv_conc_exit_innodb(prebuilt->trx); if (error != DB_SUCCESS) { error = convert_error_code_to_mysql(error, user_thd); diff --git a/sql/ha_innodb.h b/sql/ha_innodb.h index 1a9b1b16c64..bfa2687b87a 100644 --- a/sql/ha_innodb.h +++ b/sql/ha_innodb.h @@ -52,6 +52,9 @@ class ha_innobase: public handler byte* key_val_buff; /* buffer used in converting search key values from MySQL format to Innodb format */ + ulong upd_and_key_val_buff_len; + /* the length of each of the previous + two buffers */ ulong int_table_flags; uint primary_key; uint last_dup_key; @@ -83,13 +86,15 @@ class ha_innobase: public handler public: ha_innobase(TABLE *table): handler(table), int_table_flags(HA_REC_NOT_IN_SEQ | - HA_KEYPOS_TO_RNDPOS | HA_LASTKEY_ORDER | - HA_NULL_KEY | HA_CAN_SQL_HANDLER | + HA_KEYPOS_TO_RNDPOS | + HA_LASTKEY_ORDER | + HA_NULL_KEY | + HA_BLOB_KEY | + HA_CAN_SQL_HANDLER | HA_NOT_EXACT_COUNT | HA_NO_WRITE_DELAYED | HA_PRIMARY_KEY_IN_READ_INDEX | HA_DROP_BEFORE_CREATE | - HA_NO_PREFIX_CHAR_KEYS | HA_TABLE_SCAN_ON_INDEX), last_dup_key((uint) -1), start_of_scan(0) @@ -217,6 +222,14 @@ int innobase_report_binlog_offset_and_commit( int innobase_commit_complete( void* trx_handle); int innobase_rollback(THD *thd, void* trx_handle); +int innobase_rollback_to_savepoint( + THD* thd, + char* savepoint_name, + my_off_t* binlog_cache_pos); +int innobase_savepoint( + THD* thd, + char* savepoint_name, + my_off_t binlog_cache_pos); int innobase_close_connection(THD *thd); int innobase_drop_database(char *path); int innodb_show_status(THD* thd); diff --git a/sql/handler.cc b/sql/handler.cc index cae1777e958..4ea5bc0e9f5 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -379,7 +379,6 @@ int ha_commit_trans(THD *thd, THD_TRANS* trans) trans->innodb_active_trans=0; if (trans == &thd->transaction.all) operation_done= transaction_commited= 1; - } #endif #ifdef HAVE_QUERY_CACHE @@ -447,6 +446,70 @@ int ha_rollback_trans(THD *thd, THD_TRANS *trans) DBUG_RETURN(error); } + +/* +Rolls the current transaction back to a savepoint. +Return value: 0 if success, 1 if there was not a savepoint of the given +name. +*/ + +int ha_rollback_to_savepoint(THD *thd, char *savepoint_name) +{ + my_off_t binlog_cache_pos=0; + bool operation_done=0; + int error=0; + DBUG_ENTER("ha_rollback_to_savepoint"); +#ifdef USING_TRANSACTIONS + if (opt_using_transactions) + { +#ifdef HAVE_INNOBASE_DB + /* + Retrieve the trans_log binlog cache position corresponding to the + savepoint, and if the rollback is successful inside InnoDB reset the write + position in the binlog cache to what it was at the savepoint. + */ + if ((error=innobase_rollback_to_savepoint(thd, savepoint_name, + &binlog_cache_pos))) + { + my_error(ER_ERROR_DURING_ROLLBACK, MYF(0), error); + error=1; + } + else + reinit_io_cache(&thd->transaction.trans_log, WRITE_CACHE, + binlog_cache_pos, 0, 0); + operation_done=1; +#endif + if (operation_done) + statistic_increment(ha_rollback_count,&LOCK_status); + } +#endif /* USING_TRANSACTIONS */ + + DBUG_RETURN(error); +} + + +/* +Sets a transaction savepoint. +Return value: always 0, that is, succeeds always +*/ + +int ha_savepoint(THD *thd, char *savepoint_name) +{ + my_off_t binlog_cache_pos=0; + int error=0; + DBUG_ENTER("ha_savepoint"); +#ifdef USING_TRANSACTIONS + if (opt_using_transactions) + { + binlog_cache_pos=my_b_tell(&thd->transaction.trans_log); +#ifdef HAVE_INNOBASE_DB + innobase_savepoint(thd,savepoint_name, binlog_cache_pos); +#endif + } +#endif /* USING_TRANSACTIONS */ + DBUG_RETURN(error); +} + bool ha_flush_logs() { bool result=0; diff --git a/sql/handler.h b/sql/handler.h index fbad36bffdd..41f6fdc656a 100644 --- a/sql/handler.h +++ b/sql/handler.h @@ -376,6 +376,8 @@ int ha_commit_complete(THD *thd); int ha_release_temporary_latches(THD *thd); int ha_commit_trans(THD *thd, THD_TRANS *trans); int ha_rollback_trans(THD *thd, THD_TRANS *trans); +int ha_rollback_to_savepoint(THD *thd, char *savepoint_name); +int ha_savepoint(THD *thd, char *savepoint_name); int ha_autocommit_or_rollback(THD *thd, int error); void ha_set_spin_retries(uint retries); bool ha_flush_logs(void); diff --git a/sql/sql_lex.h b/sql/sql_lex.h index 7d931399782..0618f04a79b 100644 --- a/sql/sql_lex.h +++ b/sql/sql_lex.h @@ -53,8 +53,9 @@ enum enum_sql_command { SQLCOM_REPAIR, SQLCOM_REPLACE, SQLCOM_REPLACE_SELECT, SQLCOM_CREATE_FUNCTION, SQLCOM_DROP_FUNCTION, SQLCOM_REVOKE,SQLCOM_OPTIMIZE, SQLCOM_CHECK, - SQLCOM_FLUSH, SQLCOM_KILL, SQLCOM_ANALYZE, - SQLCOM_ROLLBACK, SQLCOM_COMMIT, SQLCOM_SAVEPOINT, + SQLCOM_FLUSH, SQLCOM_KILL, SQLCOM_ANALYZE, + SQLCOM_ROLLBACK, SQLCOM_ROLLBACK_TO_SAVEPOINT, + SQLCOM_COMMIT, SQLCOM_SAVEPOINT, SQLCOM_SLAVE_START, SQLCOM_SLAVE_STOP, SQLCOM_BEGIN, SQLCOM_LOAD_MASTER_TABLE, SQLCOM_CHANGE_MASTER, SQLCOM_RENAME_TABLE, SQLCOM_BACKUP_TABLE, SQLCOM_RESTORE_TABLE, diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index 3a8895ab120..bb06713ec4c 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -2536,8 +2536,22 @@ mysql_execute_command(void) res= -1; thd->options&= ~(ulong) (OPTION_BEGIN | OPTION_STATUS_NO_TRANS_UPDATE); break; + case SQLCOM_ROLLBACK_TO_SAVEPOINT: + if (!ha_rollback_to_savepoint(thd, lex->savepoint_name)) + { + if (thd->options & OPTION_STATUS_NO_TRANS_UPDATE) + send_warning(&thd->net,ER_WARNING_NOT_COMPLETE_ROLLBACK,0); + else + send_ok(&thd->net); + } + else + res= -1; + break; case SQLCOM_SAVEPOINT: - send_ok(&thd->net); + if (!ha_savepoint(thd, lex->savepoint_name)) + send_ok(&thd->net); + else + res= -1; break; default: /* Impossible */ send_ok(&thd->net); diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy index 2ef0992cdf7..4a41ad32dcc 100644 --- a/sql/sql_yacc.yy +++ b/sql/sql_yacc.yy @@ -3922,11 +3922,10 @@ rollback: ROLLBACK_SYM { Lex->sql_command = SQLCOM_ROLLBACK; - Lex->savepoint_name = NULL; } | ROLLBACK_SYM TO_SYM SAVEPOINT_SYM ident { - Lex->sql_command = SQLCOM_ROLLBACK; + Lex->sql_command = SQLCOM_ROLLBACK_TO_SAVEPOINT; Lex->savepoint_name = $4.str; }; savepoint: From 80edc81ed0dbab0a2464c4cbf49b23c35e9b4011 Mon Sep 17 00:00:00 2001 From: "guilhem@mysql.com" <> Date: Sun, 15 Jun 2003 12:01:51 +0200 Subject: [PATCH 35/40] Simplified a test. thd->enter_cond() and exit_cond(), so that the I/O thread accepts to stop when it's waiting for relay log space. Reset ignore_log_space_limit to 0 when the SQL thread terminates. --- mysql-test/r/rpl_relayspace.result | 10 ++++++-- mysql-test/t/rpl_relayspace-slave.opt | 2 +- mysql-test/t/rpl_relayspace.test | 33 +++++++++++++-------------- sql/slave.cc | 15 +++++++----- 4 files changed, 34 insertions(+), 26 deletions(-) diff --git a/mysql-test/r/rpl_relayspace.result b/mysql-test/r/rpl_relayspace.result index 5e552ef7400..721c6a882bd 100644 --- a/mysql-test/r/rpl_relayspace.result +++ b/mysql-test/r/rpl_relayspace.result @@ -6,8 +6,14 @@ drop table if exists t1,t2,t3,t4,t5,t6,t7,t8,t9; slave start; stop slave; create table t1 (a int); +drop table t1; +create table t1 (a int); +drop table t1; +reset slave; +start slave io_thread; +stop slave io_thread; reset slave; start slave; -select master_pos_wait('master-bin.001',5000,45)=-1; -master_pos_wait('master-bin.001',5000,45)=-1 +select master_pos_wait('master-bin.001',200,6)=-1; +master_pos_wait('master-bin.001',200,6)=-1 0 diff --git a/mysql-test/t/rpl_relayspace-slave.opt b/mysql-test/t/rpl_relayspace-slave.opt index 9365a2a0a26..05cb01731d2 100644 --- a/mysql-test/t/rpl_relayspace-slave.opt +++ b/mysql-test/t/rpl_relayspace-slave.opt @@ -1 +1 @@ - -O relay_log_space_limit=1024 \ No newline at end of file + -O relay_log_space_limit=10 \ No newline at end of file diff --git a/mysql-test/t/rpl_relayspace.test b/mysql-test/t/rpl_relayspace.test index 8d4f01339c7..bb82781b511 100644 --- a/mysql-test/t/rpl_relayspace.test +++ b/mysql-test/t/rpl_relayspace.test @@ -1,33 +1,32 @@ -# The slave is started with relay_log_space_limit=1024 bytes, -# to force the deadlock +# The slave is started with relay_log_space_limit=10 bytes, +# to force the deadlock after one event. source include/master-slave.inc; connection slave; stop slave; connection master; +# This will generate a master's binlog > 10 bytes create table t1 (a int); -let $1=200; -disable_query_log; -while ($1) -{ -# eval means expand $ expressions - eval insert into t1 values( $1 ); - dec $1; -} -# This will generate one 10kB master's binlog -enable_query_log; -save_master_pos; +drop table t1; +create table t1 (a int); +drop table t1; connection slave; reset slave; +start slave io_thread; +# Give the I/O thread time to block. +sleep 2; +# A bug caused the I/O thread to refuse stopping. +stop slave io_thread; +reset slave; start slave; # The I/O thread stops filling the relay log when -# it's 1kB. And the SQL thread cannot purge this relay log +# it's >10b. And the SQL thread cannot purge this relay log # as purge is done only when the SQL thread switches to another # relay log, which does not exist here. # So we should have a deadlock. # if it is not resolved automatically we'll detect -# it with master_pos_wait that waits for farther than 1kB; -# it will timeout after 45 seconds; +# it with master_pos_wait that waits for farther than 1Ob; +# it will timeout after 10 seconds; # also the slave will probably not cooperate to shutdown # (as 2 threads are locked) -select master_pos_wait('master-bin.001',5000,45)=-1; +select master_pos_wait('master-bin.001',200,6)=-1; diff --git a/sql/slave.cc b/sql/slave.cc index 74005c65672..73f1073ae09 100644 --- a/sql/slave.cc +++ b/sql/slave.cc @@ -1392,18 +1392,18 @@ static bool wait_for_relay_log_space(RELAY_LOG_INFO* rli) { bool slave_killed=0; MASTER_INFO* mi = rli->mi; - const char* save_proc_info; THD* thd = mi->io_thd; DBUG_ENTER("wait_for_relay_log_space"); pthread_mutex_lock(&rli->log_space_lock); - save_proc_info = thd->proc_info; - thd->proc_info = "Waiting for relay log space to free"; + const char* save_proc_info= thd->enter_cond(&rli->log_space_cond, + &rli->log_space_lock, + "Waiting for relay log space to free"); while (rli->log_space_limit < rli->log_space_total && !(slave_killed=io_slave_killed(thd,mi)) && !rli->ignore_log_space_limit) pthread_cond_wait(&rli->log_space_cond, &rli->log_space_lock); - thd->proc_info = save_proc_info; + thd->exit_cond(save_proc_info); pthread_mutex_unlock(&rli->log_space_lock); DBUG_RETURN(slave_killed); } @@ -2445,6 +2445,8 @@ reconnect done to recover from failed read"); for no reason, but this function will do a clean read, notice the clean value and exit immediately. */ + DBUG_PRINT("info", ("ignore_log_space_limit=%d", (int) + mi->rli.ignore_log_space_limit)); if (mi->rli.log_space_limit && mi->rli.log_space_limit < mi->rli.log_space_total && !mi->rli.ignore_log_space_limit) @@ -2626,6 +2628,7 @@ the slave SQL thread with \"SLAVE START\". We stopped at log \ pthread_mutex_unlock(&rli->data_lock); DBUG_PRINT("info",("Signaling possibly waiting master_pos_wait() functions")); pthread_cond_broadcast(&rli->data_cond); + rli->ignore_log_space_limit= 0; /* don't need any lock */ rli->save_temporary_tables = thd->temporary_tables; /* @@ -3268,8 +3271,8 @@ Log_event* next_event(RELAY_LOG_INFO* rli) log), and also when the SQL thread starts. We should also reset ignore_log_space_limit to 0 when the user does RESET SLAVE, but in fact, no need as RESET SLAVE requires that the slave - be stopped, and when the SQL thread is later restarted - ignore_log_space_limit will be reset to 0. + be stopped, and the SQL thread sets ignore_log_space_limit to 0 when + it stops. */ pthread_mutex_lock(&rli->log_space_lock); // prevent the I/O thread from blocking next times From 95ffe2d0cac112d495d49a3da65d10f0883ca874 Mon Sep 17 00:00:00 2001 From: "heikki@hundin.mysql.fi" <> Date: Sun, 15 Jun 2003 23:23:04 +0300 Subject: [PATCH 36/40] handler.h, ha_innodb.h, ha_innodb.cc: Cleanup; remove compiler warning on Windows --- sql/ha_innodb.cc | 26 +++++++++++++++----------- sql/ha_innodb.h | 3 ++- sql/handler.h | 3 ++- 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/sql/ha_innodb.cc b/sql/ha_innodb.cc index 28e61f44111..081b9a85c5c 100644 --- a/sql/ha_innodb.cc +++ b/sql/ha_innodb.cc @@ -43,7 +43,9 @@ InnoDB */ pthread_mutex_t innobase_mutex; /* Store MySQL definition of 'byte': in Linux it is char while InnoDB -uses unsigned char */ +uses unsigned char; the header univ.i which we include next defines +'byte' as a macro which expands to 'unsigned char' */ + typedef byte mysql_byte; #define INSIDE_HA_INNOBASE_CC @@ -1716,8 +1718,7 @@ get_innobase_type_from_mysql_type( } /*********************************************************************** -Stores a key value for a row to a buffer. This must currently only be used -to store a row reference to the 'ref' buffer of this table handle! */ +Stores a key value for a row to a buffer. */ uint ha_innobase::store_key_val_for_row( @@ -1725,8 +1726,8 @@ ha_innobase::store_key_val_for_row( /* out: key value length as stored in buff */ uint keynr, /* in: key number */ char* buff, /* in/out: buffer for the key value (in MySQL - format); currently this MUST be the 'ref' - buffer! */ + format) */ + uint buff_len,/* in: buffer length */ const mysql_byte* record)/* in: row in MySQL format */ { KEY* key_info = table->key_info + keynr; @@ -1757,11 +1758,11 @@ ha_innobase::store_key_val_for_row( for data. For a VARCHAR(n) the max field length is n. If the stored value is the SQL NULL then these data bytes are set to 0. */ - /* We have to zero-fill the 'ref' buffer so that MySQL is able to - use a simple memcmp to compare two key values to determine if they - are equal */ + /* We have to zero-fill the buffer so that MySQL is able to use a + simple memcmp to compare two key values to determine if they are + equal. MySQL does this to compare contents of two 'ref' values. */ - bzero(buff, ref_length); + bzero(buff, buff_len); for (; key_part != end; key_part++) { is_null = FALSE; @@ -1808,7 +1809,7 @@ ha_innobase::store_key_val_for_row( storage of the number is little-endian */ ut_a(blob_len < 256); - *buff = blob_len; + *((byte*)buff) = (byte)blob_len; buff += 2; memcpy(buff, blob_data, blob_len); @@ -1826,6 +1827,8 @@ ha_innobase::store_key_val_for_row( } } + ut_a(buff <= buff_start + buff_len); + DBUG_RETURN((uint)(buff - buff_start)); } @@ -3122,7 +3125,8 @@ ha_innobase::position( memcpy(ref, prebuilt->row_id, len); } else { - len = store_key_val_for_row(primary_key, (char*) ref, record); + len = store_key_val_for_row(primary_key, (char*)ref, + ref_length, record); } /* Since we do not store len to the buffer 'ref', we must assume diff --git a/sql/ha_innodb.h b/sql/ha_innodb.h index bfa2687b87a..a3fe56f6bcd 100644 --- a/sql/ha_innodb.h +++ b/sql/ha_innodb.h @@ -76,7 +76,8 @@ class ha_innobase: public handler longlong auto_inc_counter_for_this_stat; ulong max_row_length(const byte *buf); - uint store_key_val_for_row(uint keynr, char* buff, const byte* record); + uint store_key_val_for_row(uint keynr, char* buff, uint buff_len, + const byte* record); int update_thd(THD* thd); int change_active_index(uint keynr); int general_fetch(byte* buf, uint direction, uint match_mode); diff --git a/sql/handler.h b/sql/handler.h index 41f6fdc656a..56f63d1d521 100644 --- a/sql/handler.h +++ b/sql/handler.h @@ -189,7 +189,8 @@ class handler :public Sql_alloc public: byte *ref; /* Pointer to current row */ byte *dupp_ref; /* Pointer to dupp row */ - uint ref_length; /* Length of ref (1-8) */ + uint ref_length; /* Length of ref (1-8 or the clustered + key length) */ uint block_size; /* index block size */ ha_rows records; /* Records i datafilen */ ha_rows deleted; /* Deleted records */ From 29796e6240f3d0c8979d09f9a15a978317ca9f39 Mon Sep 17 00:00:00 2001 From: "heikki@hundin.mysql.fi" <> Date: Mon, 16 Jun 2003 00:39:46 +0300 Subject: [PATCH 37/40] row0mysql.c, dict0dict.ic: Cleanup ha_innodb.cc, data0type.h: Make sure non-latin1 users can downgrade from 4.0.14 to an earlier version if they have not created DATA_BLOB column prefix indexes --- innobase/include/data0type.h | 6 +++--- innobase/include/dict0dict.ic | 1 - innobase/row/row0mysql.c | 9 +++++++++ sql/ha_innodb.cc | 3 ++- 4 files changed, 14 insertions(+), 5 deletions(-) diff --git a/innobase/include/data0type.h b/innobase/include/data0type.h index 5e28f657f0c..4da686bf2e1 100644 --- a/innobase/include/data0type.h +++ b/innobase/include/data0type.h @@ -79,9 +79,9 @@ be less than 256 */ string, this is ORed to the precise type: this only holds for tables created with >= MySQL-4.0.14 */ -#define DATA_NONLATIN1 2048 /* if the data type is a character string - of a non-latin1 type, this is ORed to the - precise type: this only holds for tables +#define DATA_NONLATIN1 2048 /* if the data type is a DATA_BLOB (actually + TEXT) of a non-latin1 type, this is ORed to + the precise type: this only holds for tables created with >= MySQL-4.0.14 */ /*-------------------------------------------*/ diff --git a/innobase/include/dict0dict.ic b/innobase/include/dict0dict.ic index 71ea67117a7..c5982c162a7 100644 --- a/innobase/include/dict0dict.ic +++ b/innobase/include/dict0dict.ic @@ -203,7 +203,6 @@ dict_index_get_n_fields( { ut_ad(index); ut_ad(index->magic_n == DICT_INDEX_MAGIC_N); - ut_ad(index->cached); return(index->n_fields); } diff --git a/innobase/row/row0mysql.c b/innobase/row/row0mysql.c index 61ba9111b91..dc2a50c4f0c 100644 --- a/innobase/row/row0mysql.c +++ b/innobase/row/row0mysql.c @@ -507,6 +507,7 @@ row_get_prebuilt_insert_row( ins_node_t* node; dtuple_t* row; dict_table_t* table = prebuilt->table; + ulint i; ut_ad(prebuilt && table && prebuilt->trx); @@ -530,6 +531,14 @@ row_get_prebuilt_insert_row( dict_table_copy_types(row, table); + /* We init the value of every field to the SQL NULL to avoid + a debug assertion from failing */ + + for (i = 0; i < dtuple_get_n_fields(row); i++) { + + dtuple_get_nth_field(row, i)->len = UNIV_SQL_NULL; + } + ins_node_set_new_row(node, row); prebuilt->ins_graph = diff --git a/sql/ha_innodb.cc b/sql/ha_innodb.cc index 081b9a85c5c..795cffc0776 100644 --- a/sql/ha_innodb.cc +++ b/sql/ha_innodb.cc @@ -3188,7 +3188,8 @@ create_table_def( unsigned_type = 0; } - if (strcmp(default_charset_info->name, "latin1") != 0) { + if (col_type == DATA_BLOB + && strcmp(default_charset_info->name, "latin1") != 0) { nonlatin1_type = DATA_NONLATIN1; } else { nonlatin1_type = 0; From 3f33f17114cbd7673a03d68cdc7595cab19570b0 Mon Sep 17 00:00:00 2001 From: "guilhem@mysql.com" <> Date: Mon, 16 Jun 2003 15:49:54 +0200 Subject: [PATCH 38/40] Fix for nightly build test failure (test update). More messages. Testcase for bug 651. --- client/mysqltest.c | 3 ++- mysql-test/r/rpl_loaddata.result | 3 --- mysql-test/r/rpl_master_pos_wait.result | 8 ++++++-- mysql-test/t/rpl000001.test | 4 ++-- mysql-test/t/rpl_loaddata.test | 8 ++------ mysql-test/t/rpl_master_pos_wait.test | 10 ++++++++-- sql/slave.cc | 13 +++++++++++-- 7 files changed, 31 insertions(+), 18 deletions(-) diff --git a/client/mysqltest.c b/client/mysqltest.c index f6c999b18e4..f5afa0fa0df 100644 --- a/client/mysqltest.c +++ b/client/mysqltest.c @@ -996,7 +996,8 @@ int do_sync_with_master2(const char* p) if (!(row = mysql_fetch_row(res))) die("line %u: empty result in %s", start_lineno, query_buf); if (!row[0]) - die("Error on slave while syncing with master"); + die("line %u: could not sync with master ('%s' returned NULL)", + start_lineno, query_buf); mysql_free_result(res); last_result=0; if (rpl_parse) diff --git a/mysql-test/r/rpl_loaddata.result b/mysql-test/r/rpl_loaddata.result index 844a9d66cb3..b5154ca95cf 100644 --- a/mysql-test/r/rpl_loaddata.result +++ b/mysql-test/r/rpl_loaddata.result @@ -25,6 +25,3 @@ drop table t3; create table t1(a int, b int, unique(b)); insert into t1 values(1,10); load data infile '../../std_data/rpl_loaddata.dat' into table t1; -show status like 'slave_running'; -Variable_name Value -Slave_running OFF diff --git a/mysql-test/r/rpl_master_pos_wait.result b/mysql-test/r/rpl_master_pos_wait.result index 22c7aef621c..cb6ee31a54d 100644 --- a/mysql-test/r/rpl_master_pos_wait.result +++ b/mysql-test/r/rpl_master_pos_wait.result @@ -4,6 +4,10 @@ reset master; reset slave; drop table if exists t1,t2,t3,t4,t5,t6,t7,t8,t9; slave start; -select master_pos_wait('master-bin.999999',0,10); -master_pos_wait('master-bin.999999',0,10) +select master_pos_wait('master-bin.999999',0,2); +master_pos_wait('master-bin.999999',0,2) -1 + select master_pos_wait('master-bin.999999',0); +stop slave sql_thread; +master_pos_wait('master-bin.999999',0) +NULL diff --git a/mysql-test/t/rpl000001.test b/mysql-test/t/rpl000001.test index 4ffd7d1d78e..ebce3d0ac94 100644 --- a/mysql-test/t/rpl000001.test +++ b/mysql-test/t/rpl000001.test @@ -90,8 +90,8 @@ connection master; --error 1053; reap; connection slave; -sync_with_master; -#give the slave a chance to exit +# The SQL slave thread should now have stopped because the query was killed on +# the master (so it has a non-zero error code in the binlog). wait_for_slave_to_stop; # The following test can't be done because the result of Pos will differ diff --git a/mysql-test/t/rpl_loaddata.test b/mysql-test/t/rpl_loaddata.test index dc4eadda192..96a4eb3fb76 100644 --- a/mysql-test/t/rpl_loaddata.test +++ b/mysql-test/t/rpl_loaddata.test @@ -42,9 +42,5 @@ load data infile '../../std_data/rpl_loaddata.dat' into table t1; save_master_pos; connection slave; -# don't sync_with_master because the slave SQL thread should be stopped because -# of the error so MASTER_POS_WAIT() will not return; just sleep and hope the -# slave SQL thread will have had time to stop. - -sleep 1; -show status like 'slave_running'; +# The SQL slave thread should be stopped now. +wait_for_slave_to_stop; diff --git a/mysql-test/t/rpl_master_pos_wait.test b/mysql-test/t/rpl_master_pos_wait.test index a6aae222a89..24479636c91 100644 --- a/mysql-test/t/rpl_master_pos_wait.test +++ b/mysql-test/t/rpl_master_pos_wait.test @@ -5,5 +5,11 @@ save_master_pos; connection slave; sync_with_master; # Ask for a master log that has certainly not been reached yet -# timeout= 10 seconds -select master_pos_wait('master-bin.999999',0,10); +# timeout= 2 seconds +select master_pos_wait('master-bin.999999',0,2); +# Testcase for bug 651 (master_pos_wait() hangs if slave idle and STOP SLAVE). +send select master_pos_wait('master-bin.999999',0); +connection slave1; +stop slave sql_thread; +connection slave; +reap; diff --git a/sql/slave.cc b/sql/slave.cc index 73f1073ae09..dc9ce9715d8 100644 --- a/sql/slave.cc +++ b/sql/slave.cc @@ -2445,8 +2445,17 @@ reconnect done to recover from failed read"); for no reason, but this function will do a clean read, notice the clean value and exit immediately. */ - DBUG_PRINT("info", ("ignore_log_space_limit=%d", (int) - mi->rli.ignore_log_space_limit)); +#ifndef DBUG_OFF + { + char llbuf1[22], llbuf2[22]; + DBUG_PRINT("info", ("log_space_limit=%s log_space_total=%s \ +ignore_log_space_limit=%d", + llstr(mi->rli.log_space_limit,llbuf1), + llstr(mi->rli.log_space_total,llbuf2), + (int) mi->rli.ignore_log_space_limit)); + } +#endif + if (mi->rli.log_space_limit && mi->rli.log_space_limit < mi->rli.log_space_total && !mi->rli.ignore_log_space_limit) From 9b3dccd662590e9e525d936f80e4a936c2ad5308 Mon Sep 17 00:00:00 2001 From: "serg@serg.mylan" <> Date: Mon, 16 Jun 2003 19:07:48 +0200 Subject: [PATCH 39/40] reverting Monty's "fix" that turned a warning into an error. --- myisam/mi_open.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/myisam/mi_open.c b/myisam/mi_open.c index 99b97db3fbd..59fae36ac33 100644 --- a/myisam/mi_open.c +++ b/myisam/mi_open.c @@ -188,11 +188,8 @@ MI_INFO *mi_open(const char *name, int mode, uint open_flags) share->state_diff_length=len-MI_STATE_INFO_SIZE; if (share->state.header.fulltext_keys) - { - /* Not supported in this version */ - my_errno= HA_ERR_UNSUPPORTED; - goto err; - } + fprintf(stderr, "Warning: table file %s was created in MySQL 4.1+, use REPAIR TABLE ... USE_FRM to recreate it as a valid MySQL 4.0 table\n", name_buff); + mi_state_info_read(disk_cache, &share->state); len= mi_uint2korr(share->state.header.base_info_length); if (len != MI_BASE_INFO_SIZE) From b387b5add96fc092277ef79f57103ce15e3eb32f Mon Sep 17 00:00:00 2001 From: "serg@serg.mylan" <> Date: Mon, 16 Jun 2003 23:05:45 +0200 Subject: [PATCH 40/40] fulltext and left join bug fixed --- mysql-test/r/fulltext.result | 3 +++ mysql-test/r/fulltext_left_join.result | 11 +++++++++++ mysql-test/t/fulltext.test | 1 + mysql-test/t/fulltext_left_join.test | 12 ++++++++++++ sql/item_func.cc | 3 +++ sql/sql_select.cc | 3 +++ 6 files changed, 33 insertions(+) diff --git a/mysql-test/r/fulltext.result b/mysql-test/r/fulltext.result index eaaaf9c8880..6f15b2eb973 100644 --- a/mysql-test/r/fulltext.result +++ b/mysql-test/r/fulltext.result @@ -5,6 +5,9 @@ INSERT INTO t1 VALUES('MySQL has now support', 'for full-text search'), ('Only MyISAM tables','support collections'), ('Function MATCH ... AGAINST()','is used to do a search'), ('Full-text search in MySQL', 'implements vector space model'); +explain select * from t1 where MATCH(a,b) AGAINST ("collections"); +table type possible_keys key key_len ref rows Extra +t1 fulltext a a 0 1 Using where select * from t1 where MATCH(a,b) AGAINST ("collections"); a b Only MyISAM tables support collections diff --git a/mysql-test/r/fulltext_left_join.result b/mysql-test/r/fulltext_left_join.result index abc63358dbe..6875a517718 100644 --- a/mysql-test/r/fulltext_left_join.result +++ b/mysql-test/r/fulltext_left_join.result @@ -31,3 +31,14 @@ match(t1.texte,t1.sujet,t1.motsclefs) against('droit' IN BOOLEAN MODE) 1 0 drop table t1, t2; +create table t1 (venue_id int(11) default null, venue_text varchar(255) default null, dt datetime default null) type=myisam; +insert into t1 (venue_id, venue_text, dt) values (1, 'a1', '2003-05-23 19:30:00'),(null, 'a2', '2003-05-23 19:30:00'); +create table t2 (name varchar(255) not null default '', entity_id int(11) not null auto_increment, primary key (entity_id), fulltext key name (name)) type=myisam; +insert into t2 (name, entity_id) values ('aberdeen town hall', 1), ('glasgow royal concert hall', 2), ('queen\'s hall, edinburgh', 3); +select * from t1 left join t2 on venue_id = entity_id where match(name) against('aberdeen' in boolean mode) and dt = '2003-05-23 19:30:00'; +venue_id venue_text dt name entity_id +1 a1 2003-05-23 19:30:00 aberdeen town hall 1 +select * from t1 left join t2 on venue_id = entity_id where match(name) against('aberdeen') and dt = '2003-05-23 19:30:00'; +venue_id venue_text dt name entity_id +1 a1 2003-05-23 19:30:00 aberdeen town hall 1 +drop table t1,t2; diff --git a/mysql-test/t/fulltext.test b/mysql-test/t/fulltext.test index 128af680854..1b85f5903df 100644 --- a/mysql-test/t/fulltext.test +++ b/mysql-test/t/fulltext.test @@ -13,6 +13,7 @@ INSERT INTO t1 VALUES('MySQL has now support', 'for full-text search'), # nl search +explain select * from t1 where MATCH(a,b) AGAINST ("collections"); select * from t1 where MATCH(a,b) AGAINST ("collections"); select * from t1 where MATCH(a,b) AGAINST ("indexes"); select * from t1 where MATCH(a,b) AGAINST ("indexes collections"); diff --git a/mysql-test/t/fulltext_left_join.test b/mysql-test/t/fulltext_left_join.test index bcf7cbcc505..da4df13bc0c 100644 --- a/mysql-test/t/fulltext_left_join.test +++ b/mysql-test/t/fulltext_left_join.test @@ -28,3 +28,15 @@ select match(t1.texte,t1.sujet,t1.motsclefs) against('droit' IN BOOLEAN MODE) drop table t1, t2; +# +# Bug #484, reported by Stephen Brandon +# + +create table t1 (venue_id int(11) default null, venue_text varchar(255) default null, dt datetime default null) type=myisam; +insert into t1 (venue_id, venue_text, dt) values (1, 'a1', '2003-05-23 19:30:00'),(null, 'a2', '2003-05-23 19:30:00'); +create table t2 (name varchar(255) not null default '', entity_id int(11) not null auto_increment, primary key (entity_id), fulltext key name (name)) type=myisam; +insert into t2 (name, entity_id) values ('aberdeen town hall', 1), ('glasgow royal concert hall', 2), ('queen\'s hall, edinburgh', 3); +select * from t1 left join t2 on venue_id = entity_id where match(name) against('aberdeen' in boolean mode) and dt = '2003-05-23 19:30:00'; +select * from t1 left join t2 on venue_id = entity_id where match(name) against('aberdeen') and dt = '2003-05-23 19:30:00'; +drop table t1,t2; + diff --git a/sql/item_func.cc b/sql/item_func.cc index 532a7cedec0..e847b203006 100644 --- a/sql/item_func.cc +++ b/sql/item_func.cc @@ -2299,6 +2299,9 @@ double Item_func_match::val() if (ft_handler == NULL) DBUG_RETURN(-1.0); + if (table->null_row) /* NULL row from an outer join */ + return 0.0; + if (join_key) { if (table->file->ft_handler) diff --git a/sql/sql_select.cc b/sql/sql_select.cc index ff6fde1ca0c..0e8b191e4ef 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -1688,6 +1688,9 @@ add_ft_keys(DYNAMIC_ARRAY *keyuse_array, if (!cond_func || cond_func->key == NO_SUCH_KEY) return; + if (!(usable_tables & cond_func->table->map)) + return; + KEYUSE keyuse; keyuse.table= cond_func->table;