From 8c0a791a58ff6f4ae168d29359d62830c18cecc1 Mon Sep 17 00:00:00 2001 From: danielk1977 Date: Wed, 20 Aug 2008 14:49:23 +0000 Subject: [PATCH] Add the pcache module from the experimental branch. Also change things so that most of the built-in SQL functions are kept in single static hash-table, rather than creating and populating a separate hash table for each open database connection. (CVS 5566) FossilOrigin-Name: cb494e10d71852024647aaa254203579ad438ea9 --- Makefile.in | 21 +- main.mk | 18 +- manifest | 63 +- manifest.uuid | 2 +- src/btree.c | 26 +- src/callback.c | 92 ++- src/func.c | 229 +++--- src/main.c | 8 +- src/malloc.c | 4 +- src/pager.c | 1688 ++++++++---------------------------------- src/pager.h | 5 +- src/pcache.c | 1132 ++++++++++++++++++++++++++++ src/pcache.h | 175 +++++ src/sqliteInt.h | 7 +- src/test1.c | 5 +- src/test2.c | 25 +- src/test8.c | 5 +- src/vtab.c | 3 +- test/io.test | 3 +- test/ioerr2.test | 5 +- test/memsubsys1.test | 39 +- test/mutex1.test | 6 +- test/pager.test | 7 +- test/pager2.test | 13 +- test/pageropt.test | 36 +- test/shared3.test | 4 +- test/tkt2409.test | 35 +- tool/mkfunction.c | 181 +++++ tool/mksqlite3c.tcl | 4 +- 29 files changed, 2204 insertions(+), 1637 deletions(-) create mode 100644 src/pcache.c create mode 100644 src/pcache.h create mode 100644 tool/mkfunction.c diff --git a/Makefile.in b/Makefile.in index ce94298983..f59fab9ab1 100644 --- a/Makefile.in +++ b/Makefile.in @@ -157,12 +157,12 @@ NAWK = @AWK@ # OBJS0 = alter.lo analyze.lo attach.lo auth.lo bitvec.lo btmutex.lo \ btree.lo build.lo callback.lo complete.lo date.lo \ - delete.lo expr.lo fault.lo func.lo global.lo \ + delete.lo expr.lo fault.lo func2.lo global.lo \ hash.lo journal.lo insert.lo loadext.lo \ main.lo malloc.lo mem1.lo mem2.lo mem3.lo mem4.lo mem5.lo mem6.lo \ mutex.lo mutex_os2.lo mutex_unix.lo mutex_w32.lo \ opcodes.lo os.lo os_unix.lo os_win.lo os_os2.lo \ - pager.lo parse.lo pragma.lo prepare.lo printf.lo random.lo \ + pager.lo parse.lo pcache.lo pragma.lo prepare.lo printf.lo random.lo \ select.lo status.lo table.lo tokenize.lo trigger.lo update.lo \ util.lo vacuum.lo \ vdbe.lo vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbefifo.lo vdbemem.lo \ @@ -197,7 +197,6 @@ SRC = \ $(TOP)/src/delete.c \ $(TOP)/src/expr.c \ $(TOP)/src/fault.c \ - $(TOP)/src/func.c \ $(TOP)/src/global.c \ $(TOP)/src/hash.c \ $(TOP)/src/hash.h \ @@ -228,6 +227,8 @@ SRC = \ $(TOP)/src/pager.c \ $(TOP)/src/pager.h \ $(TOP)/src/parse.y \ + $(TOP)/src/pcache.c \ + $(TOP)/src/pcache.h \ $(TOP)/src/pragma.c \ $(TOP)/src/prepare.c \ $(TOP)/src/printf.c \ @@ -261,6 +262,7 @@ SRC = \ # Generated source code files # SRC += \ + func2.c \ keywordhash.h \ opcodes.c \ opcodes.h \ @@ -322,6 +324,7 @@ TESTSRC2 = \ $(TOP)/src/os_unix.c \ $(TOP)/src/os_win.c \ $(TOP)/src/pager.c \ + $(TOP)/src/pcache.c \ $(TOP)/src/pragma.c \ $(TOP)/src/prepare.c \ $(TOP)/src/printf.c \ @@ -515,8 +518,8 @@ expr.lo: $(TOP)/src/expr.c $(HDR) fault.lo: $(TOP)/src/fault.c $(HDR) $(LTCOMPILE) -c $(TOP)/src/fault.c -func.lo: $(TOP)/src/func.c $(HDR) - $(LTCOMPILE) -c $(TOP)/src/func.c +func2.lo: func2.c $(HDR) + $(LTCOMPILE) -c func2.c global.lo: $(TOP)/src/global.c $(HDR) $(LTCOMPILE) -c $(TOP)/src/global.c @@ -575,6 +578,9 @@ mutex_w32.lo: $(TOP)/src/mutex_w32.c $(HDR) pager.lo: $(TOP)/src/pager.c $(HDR) $(TOP)/src/pager.h $(LTCOMPILE) -c $(TOP)/src/pager.c +pcache.lo: $(TOP)/src/pcache.c $(HDR) $(TOP)/src/pcache.h + $(LTCOMPILE) -c $(TOP)/src/pcache.c + opcodes.lo: opcodes.c $(LTCOMPILE) -c opcodes.c @@ -607,6 +613,11 @@ parse.c: $(TOP)/src/parse.y lemon$(BEXE) $(TOP)/addopcodes.awk mv parse.h parse.h.temp $(NAWK) -f $(TOP)/addopcodes.awk parse.h.temp >parse.h +func2.c: $(TOP)/src/func.c $(HDR) + $(BCC) -o mkfunction$(BEXE) $(OPTS) $(TOP)/tool/mkfunction.c -I$(TOP)/src -I. + cat $(TOP)/src/func.c > func2.c + ./mkfunction$(BEXE) >> func2.c + pragma.lo: $(TOP)/src/pragma.c $(HDR) $(LTCOMPILE) -c $(TOP)/src/pragma.c diff --git a/main.mk b/main.mk index be78b8d6ea..1b23afc131 100644 --- a/main.mk +++ b/main.mk @@ -50,7 +50,7 @@ TCCX = $(TCC) $(OPTS) -I. -I$(TOP)/src -I$(TOP) -I$(TOP)/ext/rtree # LIBOBJ+= alter.o analyze.o attach.o auth.o bitvec.o btmutex.o btree.o build.o \ callback.o complete.o date.o delete.o \ - expr.o fault.o func.o global.o hash.o insert.o journal.o loadext.o \ + expr.o fault.o func2.o global.o hash.o insert.o journal.o loadext.o \ main.o malloc.o mem1.o mem2.o mem3.o mem4.o mem5.o mem6.o \ mutex.o mutex_os2.o mutex_unix.o mutex_w32.o \ opcodes.o os.o os_os2.o os_unix.o os_win.o \ @@ -58,7 +58,7 @@ LIBOBJ+= alter.o analyze.o attach.o auth.o bitvec.o btmutex.o btree.o build.o \ select.o status.o table.o $(TCLOBJ) tokenize.o trigger.o \ update.o util.o vacuum.o \ vdbe.o vdbeapi.o vdbeaux.o vdbeblob.o vdbefifo.o vdbemem.o \ - where.o utf.o legacy.o vtab.o rtree.o icu.o + where.o utf.o legacy.o vtab.o rtree.o icu.o pcache.o EXTOBJ = icu.o EXTOBJ += fts1.o \ @@ -98,7 +98,6 @@ SRC = \ $(TOP)/src/delete.c \ $(TOP)/src/expr.c \ $(TOP)/src/fault.c \ - $(TOP)/src/func.c \ $(TOP)/src/global.c \ $(TOP)/src/hash.c \ $(TOP)/src/hash.h \ @@ -155,6 +154,7 @@ SRC = \ $(TOP)/src/vdbeblob.c \ $(TOP)/src/vdbefifo.c \ $(TOP)/src/vdbemem.c \ + $(TOP)/src/pcache.c \ $(TOP)/src/vdbeInt.h \ $(TOP)/src/vtab.c \ $(TOP)/src/where.c @@ -204,6 +204,7 @@ SRC += \ opcodes.h \ parse.c \ parse.h \ + func2.c \ sqlite3.h @@ -241,10 +242,10 @@ TESTSRC = \ TESTSRC2 = \ $(TOP)/src/attach.c $(TOP)/src/btree.c $(TOP)/src/build.c $(TOP)/src/date.c \ - $(TOP)/src/expr.c $(TOP)/src/func.c $(TOP)/src/insert.c $(TOP)/src/os.c \ + $(TOP)/src/expr.c func2.c $(TOP)/src/insert.c $(TOP)/src/os.c \ $(TOP)/src/os_os2.c $(TOP)/src/os_unix.c $(TOP)/src/os_win.c \ $(TOP)/src/pager.c $(TOP)/src/pragma.c $(TOP)/src/prepare.c \ - $(TOP)/src/printf.c $(TOP)/src/random.c \ + $(TOP)/src/printf.c $(TOP)/src/random.c $(TOP)/src/pcache.c \ $(TOP)/src/select.c $(TOP)/src/tokenize.c \ $(TOP)/src/utf.c $(TOP)/src/util.c $(TOP)/src/vdbeapi.c $(TOP)/src/vdbeaux.c \ $(TOP)/src/vdbe.c $(TOP)/src/vdbemem.c $(TOP)/src/where.c parse.c @@ -262,6 +263,7 @@ HDR = \ $(TOP)/src/os.h \ $(TOP)/src/os_common.h \ $(TOP)/src/pager.h \ + $(TOP)/src/pcache.h \ parse.h \ sqlite3.h \ $(TOP)/src/sqlite3ext.h \ @@ -360,6 +362,10 @@ opcodes.c: opcodes.h $(TOP)/mkopcodec.awk opcodes.h: parse.h $(TOP)/src/vdbe.c $(TOP)/mkopcodeh.awk cat parse.h $(TOP)/src/vdbe.c |$(NAWK) -f $(TOP)/mkopcodeh.awk >opcodes.h +func2.c: $(TOP)/src/func.c $(HDR) + $(BCC) -o mkfunction $(OPTS) $(TOP)/tool/mkfunction.c -I$(TOP)/src -I. + cat $(TOP)/src/func.c > func2.c + ./mkfunction >> func2.c # Rules to build parse.c and parse.h - the outputs of lemon. # @@ -502,4 +508,4 @@ clean: rm -f *.da *.bb *.bbg gmon.out rm -rf tsrc target_source rm -f testloadext.dll libtestloadext.so - rm -f sqlite3.c fts?amal.c + rm -f sqlite3.c fts?amal.c tclsqlite3.c func2.c diff --git a/manifest b/manifest index e035eaedfa..bec1823904 100644 --- a/manifest +++ b/manifest @@ -1,7 +1,7 @@ -C Do\snot\sflatten\sthe\sright\sterm\sof\sa\sLEFT\sjoin.\s\sTicket\s#3300.\s(CVS\s5565) -D 2008-08-14T00:19:49 +C Add\sthe\spcache\smodule\sfrom\sthe\sexperimental\sbranch.\sAlso\schange\sthings\sso\sthat\smost\sof\sthe\sbuilt-in\sSQL\sfunctions\sare\skept\sin\ssingle\sstatic\shash-table,\srather\sthan\screating\sand\spopulating\sa\sseparate\shash\stable\sfor\seach\sopen\sdatabase\sconnection.\s(CVS\s5566) +D 2008-08-20T14:49:24 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 -F Makefile.in 2713ea64947be3b35f35d9a3158bb8299c90b019 +F Makefile.in e277c1f6dee97c18ef2f64db608da63eea4cc933 F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 F README b974cdc3f9f12b87e851b04e75996d720ebf81ac F VERSION 1d5b2c9192236ed2c6bad659a7c2d1662e39e7b9 @@ -76,7 +76,7 @@ F ext/rtree/rtree_util.tcl ee0a0311eb12175319d78bfb37302320496cee6e F ext/rtree/viewrtree.tcl 09526398dae87a5a87c5aac2b3854dbaf8376869 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 F ltmain.sh 09fe5815427dc7d0abb188bbcdf0e34896577210 -F main.mk eace65a99d12045ca496069dc707405a63197006 +F main.mk ec8a5d47c4cb447dc666472528a7e8da91384a57 F mkdll.sh 79d1ed6ae221c10589dd969f130f8a3cccfffbb7 F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f F mkextw.sh 4123480947681d9b434a5e7b1ee08135abe409ac @@ -96,17 +96,17 @@ F src/attach.c a85c14612e7e3410e0c3d2e0241832fa9688bd14 F src/auth.c c8b2ab5c8bad4bd90ed7c294694f48269162c627 F src/bitvec.c 95c86bd18d8fedf0533f5af196192546e10a7e7d F src/btmutex.c 709cad2cdca0afd013f0f612363810e53f59ec53 -F src/btree.c c38431aed9dcdf62916c9009d6f9971e588189fe +F src/btree.c c536edea3dd8a9f5f56101ca66cd5dfe699a5d7b F src/btree.h 6371c5e599fab391a150c96afbc10062b276d107 F src/btreeInt.h ab18c7b4980314e9e4b402e5dcde09f3c2545576 F src/build.c 931ed94fd3bbd67b6ac9d5ac6a45dc01e9f01726 -F src/callback.c c9f75a4c403f166af3761df47d78a806587d63af +F src/callback.c 1b1a5c580cdf7d83db24001bf8e5c09e2b08658f F src/complete.c cb14e06dbe79dee031031f0d9e686ff306afe07c F src/date.c 52a54811218a76da6235420f532ece841159a96d F src/delete.c 0d115c173863b3c688c3083ef7857c7f2e9f7a18 F src/expr.c 278d3950f55e04a2486e7b8dede3713a2ed6c0e1 F src/fault.c 3638519d1e0b82bccfafcb9f5ff491918b28f8e1 -F src/func.c 54efe220cc1ef3859a4b738011621b63a0d697c5 +F src/func.c d97ff7b72f3ddcd88970048e2894954a3f5b01d8 F src/global.c b9c96ee2317a6e1391763c7db1098a6473a91863 F src/hash.c eb64e48f3781100e5934f759fbe72a63a8fe78cb F src/hash.h 031cd9f915aff27e12262cb9eb570ac1b8326b53 @@ -115,8 +115,8 @@ F src/insert.c 89cd9af52a5ea6fb7d0cfc9c3b935d6406c360c4 F src/journal.c cffd2cd214e58c0e99c3ff632b3bee6c7cbb260e F src/legacy.c aac57bd984e666059011ea01ec4383892a253be3 F src/loadext.c eb1fe4f44d7c8ff53fc0c6a4388ab79fbd34cd64 -F src/main.c c5ed24ad2b27bc93aea580a17f370a19897165f6 -F src/malloc.c 22c68fc62f0c2df0f1deb8cd9a5ea968f995cac2 +F src/main.c c900d0f46ddb127866ba272a2124ce56dd8592dd +F src/malloc.c 56918cddc4da7a0590eef4a2497e97b781f5c25f F src/md5.c 008216bbb5d34c6fbab5357aa68575ad8a31516a F src/mem1.c 3a7fe31d8290baa3bb203af72f7dfd6323966bcd F src/mem2.c 7256327b96927020824e06ffb3443b99a28da047 @@ -135,9 +135,11 @@ F src/os_common.h 24525d8b7bce66c374dfc1810a6c9043f3359b60 F src/os_os2.c 676ed273b17bd260f905df81375c9f9950d85517 F src/os_unix.c fe0dbc35bcd3de49e46b132abfc0f45d6dd6a864 F src/os_win.c aefe9ee26430678a19a058a874e4e2bd91398142 -F src/pager.c b6a366f2343e7f127d7e70dbe76cd664336143cd -F src/pager.h 588c1ac195228b2da45c4e5f7ab6c2fd253d1751 +F src/pager.c 9f813a8f26680ff510335e2bbd01910b2a77277b +F src/pager.h fb9376af5ba8e1eb78ee3b4f15eb0f60658ffd65 F src/parse.y 84003422b2862f82bd187dfa2399557fd1f4ecbe +F src/pcache.c c1a9abb5e2aa3d1d52a2995c8e0a36535d4d1bc2 +F src/pcache.h 71ade7a84ed87d9d20507315260b1d91808d7c9a F src/pragma.c 6e207b4f69901089758c02c02e0bf86ed12a4d8f F src/prepare.c fceb567b359daaa6c6e2a4d04a01dec01ac0c907 F src/printf.c 2e984b2507291a7e16d89dc9bb60582904f6247d @@ -146,19 +148,19 @@ F src/select.c defdb8cdf7d2d8e1e0df117e50af6378fdaf1329 F src/shell.c d83b578a8ccdd3e0e7fef4388a0887ce9f810967 F src/sqlite.h.in 54e51c22e2294c5989156b0aec87aa44168ac1f0 F src/sqlite3ext.h 1e3887c9bd3ae66cb599e922824b04cd0d0f2c3e -F src/sqliteInt.h 7c68cacc760e8038ba6221c4d9ee3a7f0580e181 +F src/sqliteInt.h e49782bc45092732795297d83331b1855eb234b3 F src/sqliteLimit.h f435e728c6b620ef7312814d660a81f9356eb5c8 F src/status.c 8caa772cd9310bc297280f7cf0ede4d69ed5b801 F src/table.c 22744786199c9195720c15a7a42cb97b2e2728d8 F src/tclsqlite.c ec46084184f033ba396a9ee7b5514b695083d0f3 -F src/test1.c 0ae2203b03dec8ecf8ad731038df47ba27bfe68c -F src/test2.c 7a634c1e044be3ea5845e65155fdd1cab13936cb +F src/test1.c f92039530f6a6253ec8d3bfeaa54205a0036bbb6 +F src/test2.c 9601907ac0bab60f2f81695c3a6e9249621ae741 F src/test3.c e85b7ce5c28c3ce7fbdbf7f98e1467b19786c62b F src/test4.c 41056378671e7b00e6305fa9ac6fa27e6f96f406 F src/test5.c 162a1cea2105a2c460a3f39fa6919617b562a288 F src/test6.c 0a0304a69cfa4962a429d084c6d451ff9e4fb572 F src/test7.c 475b1fa7e3275408b40a3cbdc9508cbdc41ffa02 -F src/test8.c 7da151778887275c7f62b972363c78a5609f2bd8 +F src/test8.c 2f821eabefc73e4fd90ea24501d105e74b8e76d2 F src/test9.c 904ebe0ed1472d6bad17a81e2ecbfc20017dc237 F src/test_async.c da9f58f49faccd3a26ba89f58de125862351b6e2 F src/test_autoext.c f53b0cdf7bf5f08100009572a5d65cdb540bd0ad @@ -191,7 +193,7 @@ F src/vdbeaux.c 3e2e1f36c25eae32bdd605456c8be99f73e71eaf F src/vdbeblob.c f93110888ddc246215e9ba1f831d3d375bfd8355 F src/vdbefifo.c 20fda2a7c4c0bcee1b90eb7e545fefcdbf2e1de7 F src/vdbemem.c c37b2a266a49eaf0c0f5080157f9f1a908fdaac3 -F src/vtab.c 9c1bbb54d8b29a3412bff4eee32e9be309d85727 +F src/vtab.c edf8edb55dc93ec4bfe8b3f10213cf6c025edb9c F src/where.c a800184a2d023b15d6f2758b7a6c7ab011258fee F tclinstaller.tcl 4356d9d94d2b5ed5e68f9f0c80c4df3048dd7617 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -368,9 +370,9 @@ F test/insert4.test 6e382eaf7295a4463e6f29ea20fcd8e63d097eeb F test/insert5.test 1f93cbe9742110119133d7e8e3ccfe6d7c249766 F test/interrupt.test 42e7cf98646fd9cb4a3b131a93ed3c50b9e149f1 F test/intpkey.test 537669fd535f62632ca64828e435b9e54e8d677f -F test/io.test 23c52939ebf90f00d9cbd819ced6daa9f21445fa +F test/io.test 92cedb5eff70064f9fcf4ec51e86591b1f0ad130 F test/ioerr.test b42f249c9181b5864e53fdae38ef75475d71c66f -F test/ioerr2.test 5598405c48842c6c0187daad9eb49eff2c54f80d +F test/ioerr2.test 988bda4d8d6fa1593765a514bf7a24adb3240da9 F test/ioerr3.test d3cec5e1a11ad6d27527d0d38573fbff14c71bdd F test/ioerr4.test fc6eddfec2efc2f1ed217b9eae4c1c1d3516ce86 F test/ioerr5.test fe59ee3e3d45a121bcdb8f462b718076e66b4ca7 @@ -417,7 +419,7 @@ F test/malloc_common.tcl e082fe4791dad22b49d2ad3f7dcf1dcbee1a4cec F test/manydb.test 8de36b8d33aab5ef295b11d9e95310aeded31af8 F test/memdb.test a67bda4ff90a38f2b19f6c7f95aa7289e051d893 F test/memleak.test d2d2a1ff7105d32dc3fdf691458cf6cba58c7217 -F test/memsubsys1.test 3cfd3237dbbceaf65673be7265c58d8a34d63cbb +F test/memsubsys1.test bd578712272a3c327873b388396e351586d29e61 F test/memsubsys2.test 72a731225997ad5e8df89fdbeae9224616b6aecc F test/minmax.test 722d80816f7e096bf2c04f4111f1a6c1ba65453d F test/minmax2.test 33504c01a03bd99226144e4b03f7631a274d66e0 @@ -430,16 +432,16 @@ F test/misc5.test 6a5c1e3217a95b0db05ff9a0f1ecb5ce9043ffef F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test fd424ff93a83bb6a31463eb043c588777d8215a8 F test/misuse.test 30b3a458e5a70c31e74c291937b6c82204c59f33 -F test/mutex1.test 7f5e21fd11fe22de079e5dcd394ee4f6c257e68e +F test/mutex1.test 3f7c86418b85404f5c5d921f5d8023684bb3ac49 F test/mutex2.test 56f282f436596e9febdc6e0db2c507432b6724bb F test/nan.test 14c41572ff52dbc740b1c3303dd313a90dc6084c F test/notnull.test 44d600f916b770def8b095a9962dbe3be5a70d82 F test/null.test a8b09b8ed87852742343b33441a9240022108993 F test/openv2.test f5dd6b23e4dce828eb211649b600763c42a668df -F test/pager.test 60303481b22b240c18d6dd1b64edcecc2f4b5a97 -F test/pager2.test c025f91b75fe65e85febda64d9416428b8a5cab5 +F test/pager.test 1e1832795e9e07a359c959ffdddcf7275a88f54c +F test/pager2.test 070983b89a308adaba525a2f9c1ba0592c72fa3d F test/pager3.test 2323bf27fd5bd887b580247e5bce500ceee994b4 -F test/pageropt.test 6df72c441db0a037b7ec6990d16311c24fbea77b +F test/pageropt.test 3ee6578891baaca967f0bd349e4abfa736229e1a F test/pagesize.test e0a8b3fe80f8b8e808d94a00734c7a18c76c407e F test/permutations.test 4ad59e4489255b025aac0cc661789d35a83d87ec F test/pragma.test 2c675ed9a288094ed62bf55b35fbc749e25670fb @@ -471,7 +473,7 @@ F test/selectB.test 31e81ac9af7d224850e0706350f070ecb92fcbc7 F test/server1.test f5b790d4c0498179151ca8a7715a65a7802c859c F test/shared.test b9f3bbd3ba727c5f1f8c815b7d0199262aacf214 F test/shared2.test 0ee9de8964d70e451936a48c41cb161d9134ccf4 -F test/shared3.test 987316be601e2349e6a340a6d5f8ed981e507931 +F test/shared3.test 9c880afc081d797da514ef64bccf36f3fce2f09c F test/shared4.test d0fadacb50bb6981b2fb9dc6d1da30fa1edddf83 F test/shared_err.test 776ab7196ecda8b07a075e115b0725806991e151 F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3 @@ -527,7 +529,7 @@ F test/tkt2285.test cca17be61cf600b397188e77e7143844d2b977e9 F test/tkt2332.test fc955609b958ca86dfa102832243370a0cc84070 F test/tkt2339.test 73bd17818924cd2ac442e5fd9916b58565739450 F test/tkt2391.test ab7a11be7402da8b51a5be603425367aa0684567 -F test/tkt2409.test 20318bf6acd9b834b4420548f277b8e3a7420cd1 +F test/tkt2409.test 695269f90bbd30285fb1dd1499e8cc0d827a647d F test/tkt2450.test 77ed94863f2049c1420288ddfea2d41e5e0971d6 F test/tkt2640.test 28134f5d1e05658ef182520cf0b680fa3de5211b F test/tkt2643.test 3f3ebb743da00d4fed4fcf6daed92a0e18e57813 @@ -601,9 +603,10 @@ F tool/lempar.c 4d115ee7c0c8a749d5b22abed731abb4e6546a5f F tool/memleak.awk 4e7690a51bf3ed757e611273d43fe3f65b510133 F tool/memleak2.awk 9cc20c8e8f3c675efac71ea0721ee6874a1566e8 F tool/memleak3.tcl 7707006ee908cffff210c98158788d85bb3fcdbf +F tool/mkfunction.c a785a1970027bc7a833aa8afb43d6786a33c11ae F tool/mkkeywordhash.c ef93810fc41fb3d3dbacf9a33a29be88ea99ffa9 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e x -F tool/mksqlite3c.tcl bc5a951735e78eb37cd47f539b2400318c436872 +F tool/mksqlite3c.tcl 5b4b6f974d9d761477dfb24e2a0c578b27419ef5 F tool/mksqlite3internalh.tcl 7b43894e21bcb1bb39e11547ce7e38a063357e87 F tool/omittest.tcl 5a25ea687df5da8dd9b94bf1683f5cf2c210e51d F tool/opcodeDoc.awk b3a2a3d5d3075b8bd90b7afe24283efdd586659c @@ -618,7 +621,7 @@ F tool/speedtest16.c c8a9c793df96db7e4933f0852abb7a03d48f2e81 F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 1dbced29de5f59ba2ebf877edcadf171540374d1 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e -P a519cdb2f46fffe16e666f161479a22463616cb3 -R 4f817c5e785c5409dbaded5f21e587f1 -U drh -Z 3d91f3841f779b95c0adaebee822ab63 +P 8947c72f93d0b79c8061a3bfd5ab595edfb155a5 +R 10a01c477903bc3de835b14e423b9901 +U danielk1977 +Z 0dd70edbac3b8ebacf6c1d5a239b80f0 diff --git a/manifest.uuid b/manifest.uuid index 1ead3a4268..53912c9ad6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8947c72f93d0b79c8061a3bfd5ab595edfb155a5 \ No newline at end of file +cb494e10d71852024647aaa254203579ad438ea9 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index f00422da43..a641188707 100644 --- a/src/btree.c +++ b/src/btree.c @@ -9,7 +9,7 @@ ** May you share freely, never taking more than you give. ** ************************************************************************* -** $Id: btree.c,v 1.497 2008/08/13 19:11:48 drh Exp $ +** $Id: btree.c,v 1.498 2008/08/20 14:49:24 danielk1977 Exp $ ** ** This file implements a external (disk-based) database using BTrees. ** See the header comment on "btreeInt.h" for additional information. @@ -1132,18 +1132,19 @@ static void releasePage(MemPage *pPage){ ** reaches zero. We need to unref the pParent pointer when that ** happens. */ -static void pageDestructor(DbPage *pData, int pageSize){ +static void pageDestructor(DbPage *pData){ MemPage *pPage; - assert( (pageSize & 7)==0 ); pPage = (MemPage *)sqlite3PagerGetExtra(pData); - assert( pPage->isInit==0 || sqlite3_mutex_held(pPage->pBt->mutex) ); - if( pPage->pParent ){ - MemPage *pParent = pPage->pParent; - assert( pParent->pBt==pPage->pBt ); - pPage->pParent = 0; - releasePage(pParent); + if( pPage ){ + assert( pPage->isInit==0 || sqlite3_mutex_held(pPage->pBt->mutex) ); + if( pPage->pParent ){ + MemPage *pParent = pPage->pParent; + assert( pParent->pBt==pPage->pBt ); + pPage->pParent = 0; + releasePage(pParent); + } + pPage->isInit = 0; } - pPage->isInit = 0; } /* @@ -1287,7 +1288,7 @@ int sqlite3BtreeOpen( } pBt->busyHdr.xFunc = sqlite3BtreeInvokeBusyHandler; pBt->busyHdr.pArg = pBt; - rc = sqlite3PagerOpen(pVfs, &pBt->pPager, zFilename, + rc = sqlite3PagerOpen(pVfs, &pBt->pPager, zFilename, pageDestructor, EXTRA_SIZE, flags, vfsFlags); if( rc==SQLITE_OK ){ rc = sqlite3PagerReadFileheader(pBt->pPager,sizeof(zDbHeader),zDbHeader); @@ -1298,7 +1299,7 @@ int sqlite3BtreeOpen( sqlite3PagerSetBusyhandler(pBt->pPager, &pBt->busyHdr); p->pBt = pBt; - sqlite3PagerSetDestructor(pBt->pPager, pageDestructor); + /* sqlite3PagerSetDestructor(pBt->pPager, pageDestructor); */ sqlite3PagerSetReiniter(pBt->pPager, pageReinit); pBt->pCursor = 0; pBt->pPage1 = 0; @@ -3488,6 +3489,7 @@ void sqlite3BtreeMoveToParent(BtCursor *pCur){ assert( !sqlite3BtreeIsRootPage(pPage) ); pParent = pPage->pParent; assert( pParent!=0 ); + assert( pPage->pDbPage->nRef>0 ); idxParent = pPage->idxParent; sqlite3PagerRef(pParent->pDbPage); releasePage(pPage); diff --git a/src/callback.c b/src/callback.c index a77f994100..90d9e594d8 100644 --- a/src/callback.c +++ b/src/callback.c @@ -13,7 +13,7 @@ ** This file contains functions used to access the internal hash tables ** of user defined functions and collation sequences. ** -** $Id: callback.c,v 1.26 2008/07/28 19:34:53 drh Exp $ +** $Id: callback.c,v 1.27 2008/08/20 14:49:24 danielk1977 Exp $ */ #include "sqliteInt.h" @@ -222,6 +222,44 @@ CollSeq *sqlite3FindCollSeq( return pColl; } +/* During the search for the best function definition, this procedure +** is called to test how well the function passed as the first argument +** matches the request for a function with nArg arguments in a system +** that uses encoding enc. The value returned indicates how well the +** request is matched. A higher value indicates a better match. +** +** The returned value is always between 1 and 6, as follows: +** +** 1: A variable arguments function that prefers UTF-8 when a UTF-16 +** encoding is requested, or vice versa. +** 2: A variable arguments function that uses UTF-16BE when UTF-16LE is +** requested, or vice versa. +** 3: A variable arguments function using the same text encoding. +** 4: A function with the exact number of arguments requested that +** prefers UTF-8 when a UTF-16 encoding is requested, or vice versa. +** 5: A function with the exact number of arguments requested that +** prefers UTF-16LE when UTF-16BE is requested, or vice versa. +** 6: An exact match. +** +*/ +static int matchQuality(FuncDef *p, int nArg, u8 enc){ + int match = 0; + if( p->nArg==-1 || p->nArg==nArg || nArg==-1 ){ + match = 1; + if( p->nArg==nArg || nArg==-1 ){ + match = 4; + } + if( enc==p->iPrefEnc ){ + match += 2; + } + else if( (enc==SQLITE_UTF16LE && p->iPrefEnc==SQLITE_UTF16BE) || + (enc==SQLITE_UTF16BE && p->iPrefEnc==SQLITE_UTF16LE) ){ + match += 1; + } + } + return match; +} + /* ** Locate a user function given a name, a number of arguments and a flag ** indicating whether the function prefers UTF-16 over UTF-8. Return a @@ -261,39 +299,26 @@ FuncDef *sqlite3FindFunction( pFirst = (FuncDef*)sqlite3HashFind(&db->aFunc, zName, nName); for(p=pFirst; p; p=p->pNext){ - /* During the search for the best function definition, bestmatch is set - ** as follows to indicate the quality of the match with the definition - ** pointed to by pBest: - ** - ** 0: pBest is NULL. No match has been found. - ** 1: A variable arguments function that prefers UTF-8 when a UTF-16 - ** encoding is requested, or vice versa. - ** 2: A variable arguments function that uses UTF-16BE when UTF-16LE is - ** requested, or vice versa. - ** 3: A variable arguments function using the same text encoding. - ** 4: A function with the exact number of arguments requested that - ** prefers UTF-8 when a UTF-16 encoding is requested, or vice versa. - ** 5: A function with the exact number of arguments requested that - ** prefers UTF-16LE when UTF-16BE is requested, or vice versa. - ** 6: An exact match. - ** - ** A larger value of 'matchqual' indicates a more desirable match. - */ - if( p->nArg==-1 || p->nArg==nArg || nArg==-1 ){ - int match = 1; /* Quality of this match */ - if( p->nArg==nArg || nArg==-1 ){ - match = 4; - } - if( enc==p->iPrefEnc ){ - match += 2; - } - else if( (enc==SQLITE_UTF16LE && p->iPrefEnc==SQLITE_UTF16BE) || - (enc==SQLITE_UTF16BE && p->iPrefEnc==SQLITE_UTF16LE) ){ - match += 1; - } + int match = matchQuality(p, nArg, enc); + if( match>bestmatch ){ + pBest = p; + bestmatch = match; + } + } + /* If the createFlag parameter is false and no match was found amongst + ** the custom functions stored in sqlite3.aFunc, try to find a built-in + ** function to use. + */ + if( !createFlag && !pBest ){ + FuncDef *aFunc; + int nFunc; + int i; + nFunc = sqlite3GetBuiltinFunction(zName, nName, &aFunc); + for(i=0; ibestmatch ){ - pBest = p; + pBest = &aFunc[i]; bestmatch = match; } } @@ -304,7 +329,8 @@ FuncDef *sqlite3FindFunction( ** new entry to the hash table and return it. */ if( createFlag && bestmatch<6 && - (pBest = sqlite3DbMallocZero(db, sizeof(*pBest)+nName))!=0 ){ + (pBest = sqlite3DbMallocZero(db, sizeof(*pBest)+nName+1))!=0 ){ + pBest->zName = (char *)&pBest[1]; pBest->nArg = nArg; pBest->pNext = pFirst; pBest->iPrefEnc = enc; diff --git a/src/func.c b/src/func.c index d4489f9d4c..975f7dc811 100644 --- a/src/func.c +++ b/src/func.c @@ -16,15 +16,17 @@ ** sqliteRegisterBuildinFunctions() found at the bottom of the file. ** All other code has file scope. ** -** $Id: func.c,v 1.196 2008/07/28 19:34:53 drh Exp $ +** $Id: func.c,v 1.197 2008/08/20 14:49:24 danielk1977 Exp $ */ + +#ifndef CREATE_BUILTIN_HASHTABLE + #include "sqliteInt.h" #include #include #include #include "vdbeInt.h" - /* ** Return the collating function associated with a function. */ @@ -1214,107 +1216,12 @@ static void groupConcatFinalize(sqlite3_context *context){ ** external linkage. */ void sqlite3RegisterBuiltinFunctions(sqlite3 *db){ - static const struct { - char *zName; - signed char nArg; - u8 argType; /* 1: 0, 2: 1, 3: 2,... N: N-1. */ - u8 eTextRep; /* 1: UTF-16. 0: UTF-8 */ - u8 needCollSeq; - void (*xFunc)(sqlite3_context*,int,sqlite3_value **); - } aFuncs[] = { - { "min", -1, 0, SQLITE_UTF8, 1, minmaxFunc }, - { "min", 0, 0, SQLITE_UTF8, 1, 0 }, - { "max", -1, 1, SQLITE_UTF8, 1, minmaxFunc }, - { "max", 0, 1, SQLITE_UTF8, 1, 0 }, - { "typeof", 1, 0, SQLITE_UTF8, 0, typeofFunc }, - { "length", 1, 0, SQLITE_UTF8, 0, lengthFunc }, - { "substr", 2, 0, SQLITE_UTF8, 0, substrFunc }, - { "substr", 3, 0, SQLITE_UTF8, 0, substrFunc }, - { "abs", 1, 0, SQLITE_UTF8, 0, absFunc }, - { "round", 1, 0, SQLITE_UTF8, 0, roundFunc }, - { "round", 2, 0, SQLITE_UTF8, 0, roundFunc }, - { "upper", 1, 0, SQLITE_UTF8, 0, upperFunc }, - { "lower", 1, 0, SQLITE_UTF8, 0, lowerFunc }, - { "coalesce", -1, 0, SQLITE_UTF8, 0, ifnullFunc }, - { "coalesce", 0, 0, SQLITE_UTF8, 0, 0 }, - { "coalesce", 1, 0, SQLITE_UTF8, 0, 0 }, - { "hex", 1, 0, SQLITE_UTF8, 0, hexFunc }, - { "ifnull", 2, 0, SQLITE_UTF8, 1, ifnullFunc }, - { "random", -1, 0, SQLITE_UTF8, 0, randomFunc }, - { "randomblob", 1, 0, SQLITE_UTF8, 0, randomBlob }, - { "nullif", 2, 0, SQLITE_UTF8, 1, nullifFunc }, - { "sqlite_version", 0, 0, SQLITE_UTF8, 0, versionFunc}, - { "quote", 1, 0, SQLITE_UTF8, 0, quoteFunc }, - { "last_insert_rowid", 0, 0, SQLITE_UTF8, 0, last_insert_rowid }, - { "changes", 0, 0, SQLITE_UTF8, 0, changes }, - { "total_changes", 0, 0, SQLITE_UTF8, 0, total_changes }, - { "replace", 3, 0, SQLITE_UTF8, 0, replaceFunc }, - { "ltrim", 1, 1, SQLITE_UTF8, 0, trimFunc }, - { "ltrim", 2, 1, SQLITE_UTF8, 0, trimFunc }, - { "rtrim", 1, 2, SQLITE_UTF8, 0, trimFunc }, - { "rtrim", 2, 2, SQLITE_UTF8, 0, trimFunc }, - { "trim", 1, 3, SQLITE_UTF8, 0, trimFunc }, - { "trim", 2, 3, SQLITE_UTF8, 0, trimFunc }, - { "zeroblob", 1, 0, SQLITE_UTF8, 0, zeroblobFunc }, -#ifdef SQLITE_SOUNDEX - { "soundex", 1, 0, SQLITE_UTF8, 0, soundexFunc}, -#endif -#ifndef SQLITE_OMIT_LOAD_EXTENSION - { "load_extension", 1, 0, SQLITE_UTF8, 0, loadExt }, - { "load_extension", 2, 0, SQLITE_UTF8, 0, loadExt }, -#endif - }; - static const struct { - char *zName; - signed char nArg; - u8 argType; - u8 needCollSeq; - void (*xStep)(sqlite3_context*,int,sqlite3_value**); - void (*xFinalize)(sqlite3_context*); - } aAggs[] = { - { "min", 1, 0, 1, minmaxStep, minMaxFinalize }, - { "max", 1, 1, 1, minmaxStep, minMaxFinalize }, - { "sum", 1, 0, 0, sumStep, sumFinalize }, - { "total", 1, 0, 0, sumStep, totalFinalize }, - { "avg", 1, 0, 0, sumStep, avgFinalize }, - { "count", 0, 0, 0, countStep, countFinalize }, - { "count", 1, 0, 0, countStep, countFinalize }, - { "group_concat", -1, 0, 0, groupConcatStep, groupConcatFinalize }, - }; - int i; - - for(i=0; ineedCollSeq = 1; - } - } - } #ifndef SQLITE_OMIT_ALTERTABLE sqlite3AlterFunctions(db); #endif #ifndef SQLITE_OMIT_PARSER sqlite3AttachFunctions(db); #endif - for(i=0; ineedCollSeq = 1; - } - } - } sqlite3RegisterDateTimeFunctions(db); if( !db->mallocFailed ){ int rc = sqlite3_overload_function(db, "MATCH", 2); @@ -1326,11 +1233,6 @@ void sqlite3RegisterBuiltinFunctions(sqlite3 *db){ #ifdef SQLITE_SSE (void)sqlite3SseFunctions(db); #endif -#ifdef SQLITE_CASE_SENSITIVE_LIKE - sqlite3RegisterLikeFunctions(db, 1); -#else - sqlite3RegisterLikeFunctions(db, 0); -#endif } /* @@ -1397,3 +1299,126 @@ int sqlite3IsLikeFunction(sqlite3 *db, Expr *pExpr, int *pIsNocase, char *aWc){ *pIsNocase = (pDef->flags & SQLITE_FUNC_CASE)==0; return 1; } + +/* +** The following three macros, FUNCTION(), LIKEFUNC() and AGGREGATE() are +** used to create the literal values used for the FuncDef structures in +** the global aBuiltinFunc[] array (see below). +** +** FUNCTION(zName, nArg, iArg, bNC, xFunc) +** Used to create a scalar function definition of a function zName +** implemented by C function xFunc that accepts nArg arguments. The +** value passed as iArg is cast to a (void*) and made available +** as the user-data (sqlite3_user_data()) for the function. If +** argument bNC is true, then the FuncDef.needCollate flag is set. +** +** AGGREGATE(zName, nArg, iArg, bNC, xStep, xFinal) +** Used to create an aggregate function definition implemented by +** the C functions xStep and xFinal. The first four parameters +** are interpreted in the same way as the first 4 parameters to +** FUNCTION(). +** +** LIKEFUNC(zName, nArg, pArg, flags) +** Used to create a scalar function definition of a function zName +** that accepts nArg arguments and is implemented by a call to C +** function likeFunc. Argument pArg is cast to a (void *) and made +** available as the function user-data (sqlite3_user_data()). The +** FuncDef.flags variable is set to the value passed as the flags +** parameter. +** +** See below for examples. +*/ +#define FUNCTION(zName, nArg, iArg, bNC, xFunc) \ + {nArg, SQLITE_UTF8, bNC, 0, SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName} + +#define LIKEFUNC(zName, nArg, arg, flags) \ + {nArg, SQLITE_UTF8, 0, flags, (void *)arg, 0, likeFunc, 0, 0, #zName} + +#define AGGREGATE(zName, nArg, arg, nc, xStep, xFinal) \ + {nArg, SQLITE_UTF8, nc, 0, SQLITE_INT_TO_PTR(arg), 0, 0, xStep,xFinal, #zName} + +#endif + +/* +** This array of FuncDef structures contains most of the "built-in" functions +** and aggregates. Function users (the other routines in the SQL compiler) +** call the following function to access the FuncDef structures stored in +** this array: +** +** int sqlite3GetBuiltinFunction(const char *z, int n, FuncDef **paFunc); +** +** The caller passes the name of the required function and its length +** in parameters z and n, respectively. The value returned is the number +** of FuncDef structures found in the aBuiltinFunc[] array that match +** the requested name. If no match is found, 0 is returned. *paFunc is +** set to point at a static array containing that subset of aBuiltinFunc[] +** before returning. +** +** The implementation of sqlite3GetBuiltinFunction() is generated by +** the program found in tool/mkfunction.c, which is compiled and executed +** as part of the build process. The routine generated by this program +** assumes that if there are two or more entries in the aBuiltinFunc[] +** array with the same name (i.e. two versions of the "max" function), +** then they must be stored in adjacent slots. +*/ +static FuncDef aBuiltinFunc[] = { + FUNCTION(ltrim, 1, 1, 0, trimFunc ), + FUNCTION(ltrim, 2, 1, 0, trimFunc ), + FUNCTION(rtrim, 1, 2, 0, trimFunc ), + FUNCTION(rtrim, 2, 2, 0, trimFunc ), + FUNCTION(trim, 1, 3, 0, trimFunc ), + FUNCTION(trim, 2, 3, 0, trimFunc ), + FUNCTION(min, -1, 0, 1, minmaxFunc ), + FUNCTION(min, 0, 0, 1, 0 ), + AGGREGATE(min, 1, 0, 1, minmaxStep, minMaxFinalize ), + FUNCTION(max, -1, 1, 1, minmaxFunc ), + FUNCTION(max, 0, 1, 1, 0 ), + AGGREGATE(max, 1, 1, 1, minmaxStep, minMaxFinalize ), + FUNCTION(typeof, 1, 0, 0, typeofFunc ), + FUNCTION(length, 1, 0, 0, lengthFunc ), + FUNCTION(substr, 2, 0, 0, substrFunc ), + FUNCTION(substr, 3, 0, 0, substrFunc ), + FUNCTION(abs, 1, 0, 0, absFunc ), + FUNCTION(round, 1, 0, 0, roundFunc ), + FUNCTION(round, 2, 0, 0, roundFunc ), + FUNCTION(upper, 1, 0, 0, upperFunc ), + FUNCTION(lower, 1, 0, 0, lowerFunc ), + FUNCTION(coalesce, 1, 0, 0, 0 ), + FUNCTION(coalesce, -1, 0, 0, ifnullFunc ), + FUNCTION(coalesce, 0, 0, 0, 0 ), + FUNCTION(hex, 1, 0, 0, hexFunc ), + FUNCTION(ifnull, 2, 0, 1, ifnullFunc ), + FUNCTION(random, -1, 0, 0, randomFunc ), + FUNCTION(randomblob, 1, 0, 0, randomBlob ), + FUNCTION(nullif, 2, 0, 1, nullifFunc ), + FUNCTION(sqlite_version, 0, 0, 0, versionFunc ), + FUNCTION(quote, 1, 0, 0, quoteFunc ), + FUNCTION(last_insert_rowid, 0, 0, 0, last_insert_rowid), + FUNCTION(changes, 0, 0, 0, changes ), + FUNCTION(total_changes, 0, 0, 0, total_changes ), + FUNCTION(replace, 3, 0, 0, replaceFunc ), + FUNCTION(zeroblob, 1, 0, 0, zeroblobFunc ), +#ifdef SQLITE_SOUNDEX + FUNCTION(soundex, 1, 0, 0, soundexFunc ), +#endif +#ifndef SQLITE_OMIT_LOAD_EXTENSION + FUNCTION(load_extension, 1, 0, 0, loadExt ), + FUNCTION(load_extension, 2, 0, 0, loadExt ), +#endif + AGGREGATE(sum, 1, 0, 0, sumStep, sumFinalize ), + AGGREGATE(total, 1, 0, 0, sumStep, totalFinalize ), + AGGREGATE(avg, 1, 0, 0, sumStep, avgFinalize ), + AGGREGATE(count, 0, 0, 0, countStep, countFinalize ), + AGGREGATE(count, 1, 0, 0, countStep, countFinalize ), + AGGREGATE(group_concat, -1, 0, 0, groupConcatStep, groupConcatFinalize), + + LIKEFUNC(glob, 2, &globInfo, SQLITE_FUNC_LIKE|SQLITE_FUNC_CASE), +#ifdef SQLITE_CASE_SENSITIVE_LIKE + LIKEFUNC(like, 2, &likeInfoAlt, SQLITE_FUNC_LIKE|SQLITE_FUNC_CASE), + LIKEFUNC(like, 3, &likeInfoAlt, SQLITE_FUNC_LIKE|SQLITE_FUNC_CASE), +#else + LIKEFUNC(like, 2, &likeInfoNorm, SQLITE_FUNC_LIKE), + LIKEFUNC(like, 3, &likeInfoNorm, SQLITE_FUNC_LIKE), +#endif +}; + diff --git a/src/main.c b/src/main.c index af8aadc1d0..1d0e2f3a0e 100644 --- a/src/main.c +++ b/src/main.c @@ -14,7 +14,7 @@ ** other files are for internal use by SQLite and should not be ** accessed by users of the library. ** -** $Id: main.c,v 1.488 2008/08/12 15:21:12 drh Exp $ +** $Id: main.c,v 1.489 2008/08/20 14:49:24 danielk1977 Exp $ */ #include "sqliteInt.h" #include @@ -148,6 +148,11 @@ int sqlite3_initialize(void){ if( sqlite3Config.isInit==0 && inProgress==0 ){ inProgress = 1; rc = sqlite3_os_init(); + if( rc==SQLITE_OK ){ + rc = sqlite3PcacheInitialize(); + sqlite3PCacheBufferSetup(sqlite3Config.pPage, sqlite3Config.szPage, + sqlite3Config.nPage); + } inProgress = 0; sqlite3Config.isInit = (rc==SQLITE_OK ? 1 : 0); } @@ -193,6 +198,7 @@ int sqlite3_initialize(void){ */ int sqlite3_shutdown(void){ sqlite3Config.isMallocInit = 0; + sqlite3PcacheShutdown(); if( sqlite3Config.isInit ){ sqlite3_os_end(); } diff --git a/src/malloc.c b/src/malloc.c index fa9c88f3c9..663e44f1ff 100644 --- a/src/malloc.c +++ b/src/malloc.c @@ -12,7 +12,7 @@ ** ** Memory allocation functions used throughout sqlite. ** -** $Id: malloc.c,v 1.34 2008/08/05 17:53:23 drh Exp $ +** $Id: malloc.c,v 1.35 2008/08/20 14:49:24 danielk1977 Exp $ */ #include "sqliteInt.h" #include @@ -382,6 +382,7 @@ void sqlite3ScratchFree(void *p){ ** and that memory is of the right size and is not completely ** consumed. Otherwise, failover to sqlite3Malloc(). */ +#if 0 void *sqlite3PageMalloc(int n){ void *p; assert( n>0 ); @@ -462,6 +463,7 @@ void sqlite3PageFree(void *p){ } } } +#endif /* ** TRUE if p is a lookaside memory allocation from db diff --git a/src/pager.c b/src/pager.c index bf33514adf..6683f1743d 100644 --- a/src/pager.c +++ b/src/pager.c @@ -18,12 +18,10 @@ ** file simultaneously, or one process from reading the database while ** another is writing. ** -** @(#) $Id: pager.c,v 1.469 2008/08/02 03:50:39 drh Exp $ +** @(#) $Id: pager.c,v 1.470 2008/08/20 14:49:24 danielk1977 Exp $ */ #ifndef SQLITE_OMIT_DISKIO #include "sqliteInt.h" -#include -#include /* ** Macros for troubleshooting. Normally turned off @@ -131,173 +129,6 @@ */ #define FORCE_ALIGNMENT(X) (((X)+7)&~7) -typedef struct PgHdr PgHdr; - -/* -** Each pager stores all currently unreferenced pages in a list sorted -** in least-recently-used (LRU) order (i.e. the first item on the list has -** not been referenced in a long time, the last item has been recently -** used). An instance of this structure is included as part of each -** pager structure for this purpose (variable Pager.lru). -** -** Additionally, if memory-management is enabled, all unreferenced pages -** are stored in a global LRU list (global variable sqlite3LruPageList). -** -** In both cases, the PagerLruList.pFirstSynced variable points to -** the first page in the corresponding list that does not require an -** fsync() operation before its memory can be reclaimed. If no such -** page exists, PagerLruList.pFirstSynced is set to NULL. -*/ -typedef struct PagerLruList PagerLruList; -struct PagerLruList { - PgHdr *pFirst; /* First page in LRU list */ - PgHdr *pLast; /* Last page in LRU list (the most recently used) */ - PgHdr *pFirstSynced; /* First page in list with PgHdr.needSync==0 */ -}; - -/* -** The following structure contains the next and previous pointers used -** to link a PgHdr structure into a PagerLruList linked list. -*/ -typedef struct PagerLruLink PagerLruLink; -struct PagerLruLink { - PgHdr *pNext; - PgHdr *pPrev; -}; - -/* -** Each in-memory image of a page begins with the following header. -** This header is only visible to this pager module. The client -** code that calls pager sees only the data that follows the header. -** -** Client code should call sqlite3PagerWrite() on a page prior to making -** any modifications to that page. The first time sqlite3PagerWrite() -** is called, the original page contents are written into the rollback -** journal and PgHdr.inJournal and PgHdr.needSync are set. Later, once -** the journal page has made it onto the disk surface, PgHdr.needSync -** is cleared. The modified page cannot be written back into the original -** database file until the journal pages has been synced to disk and the -** PgHdr.needSync has been cleared. -** -** The PgHdr.dirty flag is set when sqlite3PagerWrite() is called and -** is cleared again when the page content is written back to the original -** database file. -** -** Details of important structure elements: -** -** needSync -** -** If this is true, this means that it is not safe to write the page -** content to the database because the original content needed -** for rollback has not by synced to the main rollback journal. -** The original content may have been written to the rollback journal -** but it has not yet been synced. So we cannot write to the database -** file because power failure might cause the page in the journal file -** to never reach the disk. It is as if the write to the journal file -** does not occur until the journal file is synced. -** -** This flag is false if the page content exactly matches what -** currently exists in the database file. The needSync flag is also -** false if the original content has been written to the main rollback -** journal and synced. If the page represents a new page that has -** been added onto the end of the database during the current -** transaction, the needSync flag is true until the original database -** size in the journal header has been synced to disk. -** -** inJournal -** -** This is true if the original page has been written into the main -** rollback journal. This is always false for new pages added to -** the end of the database file during the current transaction. -** And this flag says nothing about whether or not the journal -** has been synced to disk. For pages that are in the original -** database file, the following expression should always be true: -** -** inJournal = sqlite3BitvecTest(pPager->pInJournal, pgno) -** -** The pPager->pInJournal object is only valid for the original -** pages of the database, not new pages that are added to the end -** of the database, so obviously the above expression cannot be -** valid for new pages. For new pages inJournal is always 0. -** -** dirty -** -** When true, this means that the content of the page has been -** modified and needs to be written back to the database file. -** If false, it means that either the content of the page is -** unchanged or else the content is unimportant and we do not -** care whether or not it is preserved. -** -** alwaysRollback -** -** This means that the sqlite3PagerDontRollback() API should be -** ignored for this page. The DontRollback() API attempts to say -** that the content of the page on disk is unimportant (it is an -** unused page on the freelist) so that it is unnecessary to -** rollback changes to this page because the content of the page -** can change without changing the meaning of the database. This -** flag overrides any DontRollback() attempt. This flag is set -** when a page that originally contained valid data is added to -** the freelist. Later in the same transaction, this page might -** be pulled from the freelist and reused for something different -** and at that point the DontRollback() API will be called because -** pages taken from the freelist do not need to be protected by -** the rollback journal. But this flag says that the page was -** not originally part of the freelist so that it still needs to -** be rolled back in spite of any subsequent DontRollback() calls. -** -** needRead -** -** This flag means (when true) that the content of the page has -** not yet been loaded from disk. The in-memory content is just -** garbage. (Actually, we zero the content, but you should not -** make any assumptions about the content nevertheless.) If the -** content is needed in the future, it should be read from the -** original database file. -*/ -struct PgHdr { - Pager *pPager; /* The pager to which this page belongs */ - Pgno pgno; /* The page number for this page */ - PgHdr *pNextHash, *pPrevHash; /* Hash collision chain for PgHdr.pgno */ - PagerLruLink free; /* Next and previous free pages */ - PgHdr *pNextAll; /* A list of all pages */ - u8 inJournal; /* TRUE if has been written to journal */ - u8 dirty; /* TRUE if we need to write back changes */ - u8 needSync; /* Sync journal before writing this page */ - u8 alwaysRollback; /* Disable DontRollback() for this page */ - u8 needRead; /* Read content if PagerWrite() is called */ - short int nRef; /* Number of users of this page */ - PgHdr *pDirty, *pPrevDirty; /* Dirty pages */ -#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT - PgHdr *pPrevAll; /* A list of all pages */ - PagerLruLink gfree; /* Global list of nRef==0 pages */ -#endif -#ifdef SQLITE_CHECK_PAGES - u32 pageHash; -#endif - void *pData; /* Page data */ - /* Pager.nExtra bytes of local data appended to this header */ -}; - -/* -** For an in-memory only database, some extra information is recorded about -** each page so that changes can be rolled back. (Journal files are not -** used for in-memory databases.) The following information is added to -** the end of every EXTRA block for in-memory databases. -** -** This information could have been added directly to the PgHdr structure. -** But then it would take up an extra 8 bytes of storage on every PgHdr -** even for disk-based databases. Splitting it out saves 8 bytes. This -** is only a savings of 0.8% but those percentages add up. -*/ -typedef struct PgHistory PgHistory; -struct PgHistory { - u8 *pOrig; /* Original page text. Restore to this on a full rollback */ - u8 *pStmt; /* Text as it was at the beginning of the current statement */ - PgHdr *pNextStmt, *pPrevStmt; /* List of pages in the statement journal */ - u8 inStmt; /* TRUE if in the statement subjournal */ -}; - /* ** A macro used for invoking the codec if there is one */ @@ -309,15 +140,6 @@ struct PgHistory { # define CODEC2(P,D,N,X) ((char*)D) #endif -/* -** Convert a pointer to a PgHdr into a pointer to its data -** and back again. -*/ -#define PGHDR_TO_DATA(P) ((P)->pData) -#define PGHDR_TO_EXTRA(G,P) ((void*)&((G)[1])) -#define PGHDR_TO_HIST(P,PGR) \ - ((PgHistory*)&((char*)(&(P)[1]))[(PGR)->nExtra]) - /* ** A open page cache is an instance of the following structure. ** @@ -365,7 +187,6 @@ struct Pager { int nExtra; /* Add this many bytes to each in-memory page */ int pageSize; /* Number of bytes in a page */ int nPage; /* Total number of in-memory pages */ - int nRef; /* Number of in-memory pages with PgHdr.nRef>0 */ int mxPage; /* Maximum number of pages to hold in cache */ Pgno mxPgno; /* Maximum allowed size of the database */ Bitvec *pInJournal; /* One bit for each page in the database file */ @@ -376,10 +197,6 @@ struct Pager { sqlite3_file *fd, *jfd; /* File descriptors for database and journal */ sqlite3_file *stfd; /* File descriptor for the statement subjournal*/ BusyHandler *pBusyHandler; /* Pointer to sqlite.busyHandler */ - PagerLruList lru; /* LRU list of free pages */ - PgHdr *pAll; /* List of all pages */ - PgHdr *pStmt; /* List of pages in the statement subjournal */ - PgHdr *pDirty; /* List of all dirty pages */ i64 journalOff; /* Current byte offset in the journal file */ i64 journalHdr; /* Byte offset to previous journal header */ i64 stmtHdrOff; /* First journal header written this statement */ @@ -395,18 +212,11 @@ struct Pager { #ifdef SQLITE_HAS_CODEC void *(*xCodec)(void*,void*,Pgno,int); /* Routine for en/decoding data */ void *pCodecArg; /* First argument to xCodec() */ -#endif - int nHash; /* Size of the pager hash table */ - PgHdr **aHash; /* Hash table to map page number to PgHdr */ -#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT - Pager *pNext; /* Doubly linked list of pagers on which */ - Pager *pPrev; /* sqlite3_release_memory() will work */ - volatile int iInUseMM; /* Non-zero if unavailable to MM */ - volatile int iInUseDB; /* Non-zero if in sqlite3_release_memory() */ #endif char *pTmpSpace; /* Pager.pageSize bytes of space for tmp use */ char dbFileVers[16]; /* Changes whenever database file changes */ i64 journalSizeLimit; /* Size limit for persistent journal files */ + PCache *pPCache; /* Pointer to page cache object */ }; /* @@ -418,22 +228,11 @@ struct Pager { int sqlite3_pager_readdb_count = 0; /* Number of full pages read from DB */ int sqlite3_pager_writedb_count = 0; /* Number of full pages written to DB */ int sqlite3_pager_writej_count = 0; /* Number of pages written to journal */ -int sqlite3_pager_pgfree_count = 0; /* Number of cache pages freed */ # define PAGER_INCR(v) v++ #else # define PAGER_INCR(v) #endif -/* -** The following variable points to the head of a double-linked list -** of all pagers that are eligible for page stealing by the -** sqlite3_release_memory() interface. Access to this list is -** protected by the SQLITE_MUTEX_STATIC_MEM2 mutex. -*/ -#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT -static Pager *sqlite3PagerList = 0; -static PagerLruList sqlite3LruPageList = {0, 0, 0}; -#endif /* @@ -504,162 +303,16 @@ static const unsigned char aJournalMagic[] = { #define PAGER_MAX_PGNO 2147483647 /* -** The pagerEnter() and pagerLeave() routines acquire and release -** a mutex on each pager. The mutex is recursive. -** -** This is a special-purpose mutex. It only provides mutual exclusion -** between the Btree and the Memory Management sqlite3_release_memory() -** function. It does not prevent, for example, two Btrees from accessing -** the same pager at the same time. Other general-purpose mutexes in -** the btree layer handle that chore. +** The following two macros act as a type of recursive mutex. Their +** only purpose is to provide mutual exclusion between the "normal" +** users of a pager object (the btree.c module) and the user of the +** pagerStress() function (the pcache.c module). While the mutex +** obtained using pagerEnter() is held, the pcache module guarantees +** that the pagerStress() callback will not be invoked from a thread +** other than the holder of the mutex. */ -#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT - static void pagerEnter(Pager *p){ - p->iInUseDB++; - if( p->iInUseMM && p->iInUseDB==1 ){ -#ifndef SQLITE_MUTEX_NOOP - sqlite3_mutex *mutex; - mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MEM2); -#endif - p->iInUseDB = 0; - sqlite3_mutex_enter(mutex); - p->iInUseDB = 1; - sqlite3_mutex_leave(mutex); - } - assert( p->iInUseMM==0 ); - } - static void pagerLeave(Pager *p){ - p->iInUseDB--; - assert( p->iInUseDB>=0 ); - } -#else -# define pagerEnter(X) -# define pagerLeave(X) -#endif - -/* -** Add page pPg to the end of the linked list managed by structure -** pList (pPg becomes the last entry in the list - the most recently -** used). Argument pLink should point to either pPg->free or pPg->gfree, -** depending on whether pPg is being added to the pager-specific or -** global LRU list. -*/ -static void listAdd(PagerLruList *pList, PagerLruLink *pLink, PgHdr *pPg){ - pLink->pNext = 0; - pLink->pPrev = pList->pLast; - -#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT - assert(pLink==&pPg->free || pLink==&pPg->gfree); - assert(pLink==&pPg->gfree || pList!=&sqlite3LruPageList); -#endif - - if( pList->pLast ){ - int iOff = (char *)pLink - (char *)pPg; - PagerLruLink *pLastLink = (PagerLruLink *)(&((u8 *)pList->pLast)[iOff]); - pLastLink->pNext = pPg; - }else{ - assert(!pList->pFirst); - pList->pFirst = pPg; - } - - pList->pLast = pPg; - if( !pList->pFirstSynced && pPg->needSync==0 ){ - pList->pFirstSynced = pPg; - } -} - -/* -** Remove pPg from the list managed by the structure pointed to by pList. -** -** Argument pLink should point to either pPg->free or pPg->gfree, depending -** on whether pPg is being added to the pager-specific or global LRU list. -*/ -static void listRemove(PagerLruList *pList, PagerLruLink *pLink, PgHdr *pPg){ - int iOff = (char *)pLink - (char *)pPg; - -#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT - assert(pLink==&pPg->free || pLink==&pPg->gfree); - assert(pLink==&pPg->gfree || pList!=&sqlite3LruPageList); -#endif - - if( pPg==pList->pFirst ){ - pList->pFirst = pLink->pNext; - } - if( pPg==pList->pLast ){ - pList->pLast = pLink->pPrev; - } - if( pLink->pPrev ){ - PagerLruLink *pPrevLink = (PagerLruLink *)(&((u8 *)pLink->pPrev)[iOff]); - pPrevLink->pNext = pLink->pNext; - } - if( pLink->pNext ){ - PagerLruLink *pNextLink = (PagerLruLink *)(&((u8 *)pLink->pNext)[iOff]); - pNextLink->pPrev = pLink->pPrev; - } - if( pPg==pList->pFirstSynced ){ - PgHdr *p = pLink->pNext; - while( p && p->needSync ){ - PagerLruLink *pL = (PagerLruLink *)(&((u8 *)p)[iOff]); - p = pL->pNext; - } - pList->pFirstSynced = p; - } - - pLink->pNext = pLink->pPrev = 0; -} - -/* -** Add page pPg to the list of free pages for the pager. If -** memory-management is enabled, also add the page to the global -** list of free pages. -*/ -static void lruListAdd(PgHdr *pPg){ - listAdd(&pPg->pPager->lru, &pPg->free, pPg); -#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT - if( !pPg->pPager->memDb ){ - sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_LRU)); - listAdd(&sqlite3LruPageList, &pPg->gfree, pPg); - sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_LRU)); - } -#endif -} - -/* -** Remove page pPg from the list of free pages for the associated pager. -** If memory-management is enabled, also remove pPg from the global list -** of free pages. -*/ -static void lruListRemove(PgHdr *pPg){ - listRemove(&pPg->pPager->lru, &pPg->free, pPg); -#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT - if( !pPg->pPager->memDb ){ - sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_LRU)); - listRemove(&sqlite3LruPageList, &pPg->gfree, pPg); - sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_LRU)); - } -#endif -} - -/* -** This function is called just after the needSync flag has been cleared -** from all pages managed by pPager (usually because the journal file -** has just been synced). It updates the pPager->lru.pFirstSynced variable -** and, if memory-management is enabled, the sqlite3LruPageList.pFirstSynced -** variable also. -*/ -static void lruListSetFirstSynced(Pager *pPager){ - pPager->lru.pFirstSynced = pPager->lru.pFirst; -#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT - if( !pPager->memDb ){ - PgHdr *p; - sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_LRU)); - for(p=sqlite3LruPageList.pFirst; p && p->needSync; p=p->gfree.pNext); - assert(p==pPager->lru.pFirstSynced || p==sqlite3LruPageList.pFirstSynced); - sqlite3LruPageList.pFirstSynced = p; - sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_LRU)); - } -#endif -} +#define pagerEnter(p) (sqlite3PcacheLock(p->pPCache)) +#define pagerLeave(p) (sqlite3PcacheUnlock(p->pPCache)) /* ** Return true if page *pPg has already been written to the statement @@ -669,53 +322,12 @@ static void lruListSetFirstSynced(Pager *pPager){ static int pageInStatement(PgHdr *pPg){ Pager *pPager = pPg->pPager; if( MEMDB ){ - return PGHDR_TO_HIST(pPg, pPager)->inStmt; + return pPg->apSave[1]!=0; }else{ return sqlite3BitvecTest(pPager->pInStmt, pPg->pgno); } } -/* -** Change the size of the pager hash table to N. N must be a power -** of two. -*/ -static void pager_resize_hash_table(Pager *pPager, int N){ - PgHdr **aHash, *pPg; - assert( N>0 && (N&(N-1))==0 ); -#ifdef SQLITE_MALLOC_SOFT_LIMIT - if( N*sizeof(aHash[0])>SQLITE_MALLOC_SOFT_LIMIT ){ - N = SQLITE_MALLOC_SOFT_LIMIT/sizeof(aHash[0]); - } - if( N==pPager->nHash ) return; -#endif - pagerLeave(pPager); - if( pPager->aHash!=0 ) sqlite3BeginBenignMalloc(); - aHash = sqlite3MallocZero( sizeof(aHash[0])*N ); - if( pPager->aHash!=0 ) sqlite3EndBenignMalloc(); - pagerEnter(pPager); - if( aHash==0 ){ - /* Failure to rehash is not an error. It is only a performance hit. */ - return; - } - sqlite3_free(pPager->aHash); - pPager->nHash = N; - pPager->aHash = aHash; - for(pPg=pPager->pAll; pPg; pPg=pPg->pNextAll){ - int h; - if( pPg->pgno==0 ){ - assert( pPg->pNextHash==0 && pPg->pPrevHash==0 ); - continue; - } - h = pPg->pgno & (N-1); - pPg->pNextHash = aHash[h]; - if( aHash[h] ){ - aHash[h]->pPrevHash = pPg; - } - aHash[h] = pPg; - pPg->pPrevHash = 0; - } -} - /* ** Read a 32-bit integer from the given file descriptor. Store the integer ** that is read in *pRes. Return SQLITE_OK if everything worked, or an @@ -825,7 +437,9 @@ static int pager_error(Pager *pPager, int rc){ rc2==SQLITE_CORRUPT ){ pPager->errCode = rc; - if( pPager->state==PAGER_UNLOCK && pPager->nRef==0 ){ + if( pPager->state==PAGER_UNLOCK + && sqlite3PcacheRefCount(pPager->pPCache)==0 + ){ /* If the pager is already unlocked, call pager_unlock() now to ** clear the error state and ensure that the pager-cache is ** completely empty. @@ -854,8 +468,10 @@ static u32 pager_datahash(int nByte, unsigned char *pData){ return hash; } static u32 pager_pagehash(PgHdr *pPage){ - return pager_datahash(pPage->pPager->pageSize, - (unsigned char *)PGHDR_TO_DATA(pPage)); + return pager_datahash(pPage->pPager->pageSize, (unsigned char *)pPage->pData); +} +static u32 pager_set_pagehash(PgHdr *pPage){ + pPage->pageHash = pager_pagehash(pPage); } /* @@ -866,8 +482,8 @@ static u32 pager_pagehash(PgHdr *pPage){ #define CHECK_PAGE(x) checkPage(x) static void checkPage(PgHdr *pPg){ Pager *pPager = pPg->pPager; - assert( !pPg->pageHash || pPager->errCode || MEMDB || pPg->dirty || - pPg->pageHash==pager_pagehash(pPg) ); + assert( !pPg->pageHash || pPager->errCode || MEMDB + || (pPg->flags&PGHDR_DIRTY) || pPg->pageHash==pager_pagehash(pPg) ); } #else @@ -1244,41 +860,13 @@ static int writeMasterJournal(Pager *pPager, const char *zMaster){ return rc; } -/* -** Add or remove a page from the list of all pages that are in the -** statement journal. -** -** The Pager keeps a separate list of pages that are currently in -** the statement journal. This helps the sqlite3PagerStmtCommit() -** routine run MUCH faster for the common case where there are many -** pages in memory but only a few are in the statement journal. -*/ -static void page_add_to_stmt_list(PgHdr *pPg){ - Pager *pPager = pPg->pPager; - PgHistory *pHist = PGHDR_TO_HIST(pPg, pPager); - assert( MEMDB ); - if( !pHist->inStmt ){ - assert( pHist->pPrevStmt==0 && pHist->pNextStmt==0 ); - if( pPager->pStmt ){ - PGHDR_TO_HIST(pPager->pStmt, pPager)->pPrevStmt = pPg; - } - pHist->pNextStmt = pPager->pStmt; - pPager->pStmt = pPg; - pHist->inStmt = 1; - } -} - /* ** Find a page in the hash table given its page number. Return ** a pointer to the page or NULL if not found. */ static PgHdr *pager_lookup(Pager *pPager, Pgno pgno){ PgHdr *p; - if( pPager->aHash==0 ) return 0; - p = pPager->aHash[pgno & (pPager->nHash-1)]; - while( p && p->pgno!=pgno ){ - p = p->pNextHash; - } + sqlite3PcacheFetch(pPager->pPCache, pgno, 0, &p); return p; } @@ -1289,27 +877,8 @@ static PgHdr *pager_lookup(Pager *pPager, Pgno pgno){ ** to access those pages will likely result in a coredump. */ static void pager_reset(Pager *pPager){ - PgHdr *pPg, *pNext; if( pPager->errCode ) return; - for(pPg=pPager->pAll; pPg; pPg=pNext){ - IOTRACE(("PGFREE %p %d\n", pPager, pPg->pgno)); - PAGER_INCR(sqlite3_pager_pgfree_count); - pNext = pPg->pNextAll; - lruListRemove(pPg); - sqlite3PageFree(pPg->pData); - sqlite3_free(pPg); - } - assert(pPager->lru.pFirst==0); - assert(pPager->lru.pFirstSynced==0); - assert(pPager->lru.pLast==0); - pPager->pStmt = 0; - pPager->pAll = 0; - pPager->pDirty = 0; - pPager->nHash = 0; - sqlite3_free(pPager->aHash); - pPager->nPage = 0; - pPager->aHash = 0; - pPager->nRef = 0; + sqlite3PcacheClear(pPager->pPCache); } /* @@ -1374,17 +943,12 @@ static void pager_unlock(Pager *pPager){ ** do not attempt the rollback. */ static void pagerUnlockAndRollback(Pager *p){ - /* assert( p->state>=PAGER_RESERVED || p->journalOpen==0 ); */ if( p->errCode==SQLITE_OK && p->state>=PAGER_RESERVED ){ sqlite3BeginBenignMalloc(); sqlite3PagerRollback(p); sqlite3EndBenignMalloc(); } pager_unlock(p); -#if 0 - assert( p->errCode || !p->journalOpen || (p->exclusiveMode&&!p->journalOff) ); - assert( p->errCode || !p->stmtOpen || p->exclusiveMode ); -#endif } /* @@ -1405,7 +969,6 @@ static void pagerUnlockAndRollback(Pager *p){ ** a file is an expensive operation. */ static int pager_end_transaction(Pager *pPager, int hasMaster){ - PgHdr *pPg; int rc = SQLITE_OK; int rc2 = SQLITE_OK; assert( !MEMDB ); @@ -1434,16 +997,13 @@ static int pager_end_transaction(Pager *pPager, int hasMaster){ } sqlite3BitvecDestroy(pPager->pInJournal); pPager->pInJournal = 0; - for(pPg=pPager->pAll; pPg; pPg=pPg->pNextAll){ - pPg->inJournal = 0; - pPg->dirty = 0; - pPg->needSync = 0; - pPg->alwaysRollback = 0; + sqlite3PcacheCleanAll(pPager->pPCache); #ifdef SQLITE_CHECK_PAGES - pPg->pageHash = pager_pagehash(pPg); + sqlite3PcacheIterate(pPager->pPCache, pager_set_pagehash); #endif - } - pPager->pDirty = 0; + sqlite3PcacheSetFlags(pPager->pPCache, + ~(PGHDR_IN_JOURNAL | PGHDR_NEED_SYNC | PGHDR_ALWAYS_ROLLBACK), 0 + ); pPager->dirtyCache = 0; pPager->nRec = 0; }else{ @@ -1459,7 +1019,7 @@ static int pager_end_transaction(Pager *pPager, int hasMaster){ pPager->origDbSize = 0; pPager->setMaster = 0; pPager->needSync = 0; - lruListSetFirstSynced(pPager); + /* lruListSetFirstSynced(pPager); */ pPager->dbSize = -1; pPager->dbModified = 0; @@ -1588,13 +1148,12 @@ static int pager_playback_one_page( pPg = pager_lookup(pPager, pgno); PAGERTRACE4("PLAYBACK %d page %d hash(%08x)\n", PAGERID(pPager), pgno, pager_datahash(pPager->pageSize, aData)); - if( pPager->state>=PAGER_EXCLUSIVE && (pPg==0 || pPg->needSync==0) - && pPager->fd->pMethods ){ + if( (pPager->state>=PAGER_EXCLUSIVE) + && (pPg==0 || 0==(pPg->flags&PGHDR_NEED_SYNC)) + && (pPager->fd->pMethods) + ){ i64 offset = (pgno-1)*(i64)pPager->pageSize; rc = sqlite3OsWrite(pPager->fd, aData, pPager->pageSize, offset); - if( pPg ){ - makeClean(pPg); - } } if( pPg ){ /* No page should ever be explicitly rolled back that is in use, except @@ -1605,11 +1164,12 @@ static int pager_playback_one_page( */ void *pData; /* assert( pPg->nRef==0 || pPg->pgno==1 ); */ - pData = PGHDR_TO_DATA(pPg); + pData = pPg->pData; memcpy(pData, aData, pPager->pageSize); if( pPager->xReiniter ){ pPager->xReiniter(pPg, pPager->pageSize); } + makeClean(pPg); #ifdef SQLITE_CHECK_PAGES pPg->pageHash = pager_pagehash(pPg); #endif @@ -1621,6 +1181,7 @@ static int pager_playback_one_page( /* Decode the page just read from disk */ CODEC1(pPager, pData, pPg->pgno, 3); + sqlite3PcacheRelease(pPg); } return rc; } @@ -2074,11 +1635,7 @@ end_stmt_playback: ** Change the maximum number of in-memory pages that are allowed. */ void sqlite3PagerSetCachesize(Pager *pPager, int mxPage){ - if( mxPage>10 ){ - pPager->mxPage = mxPage; - }else{ - pPager->mxPage = 10; - } + sqlite3PcacheSetCachesize(pPager->pPCache, mxPage); } /* @@ -2150,6 +1707,8 @@ static int sqlite3PagerOpentemp( return rc; } +static int pagerStress(void *); + /* ** Create a new page cache and put a pointer to the page cache in *ppPager. ** The file to be cached need not exist. The file is not locked until @@ -2168,6 +1727,7 @@ int sqlite3PagerOpen( sqlite3_vfs *pVfs, /* The virtual file system to use */ Pager **ppPager, /* Return the Pager structure here */ const char *zFilename, /* Name of the database file to open */ + void (*xDesc)(DbPage*), /* Page destructor function */ int nExtra, /* Extra bytes append to each in-memory page */ int flags, /* flags controlling this file */ int vfsFlags /* flags passed through to sqlite3_vfs.xOpen() */ @@ -2182,6 +1742,7 @@ int sqlite3PagerOpen( int useJournal = (flags & PAGER_OMIT_JOURNAL)==0; int noReadlock = (flags & PAGER_NO_READLOCK)!=0; int journalFileSize = sqlite3JournalSize(pVfs); + int pcacheSize = sqlite3PcacheSize(); int szPageDflt = SQLITE_DEFAULT_PAGE_SIZE; char *zPathname = 0; int nPathname = 0; @@ -2218,6 +1779,7 @@ int sqlite3PagerOpen( /* Allocate memory for the pager structure */ pPager = sqlite3MallocZero( sizeof(*pPager) + /* Pager structure */ + pcacheSize + /* PCache object */ journalFileSize + /* The journal file structure */ pVfs->szOsFile * 3 + /* The main db and two journal files */ 3*nPathname + 40 /* zFilename, zDirectory, zJournal */ @@ -2226,7 +1788,8 @@ int sqlite3PagerOpen( sqlite3_free(zPathname); return SQLITE_NOMEM; } - pPtr = (u8 *)&pPager[1]; + pPager->pPCache = (PCache *)&pPager[1]; + pPtr = ((u8 *)&pPager[1]) + pcacheSize; pPager->vfsFlags = vfsFlags; pPager->fd = (sqlite3_file*)&pPtr[pVfs->szOsFile*0]; pPager->stfd = (sqlite3_file*)&pPtr[pVfs->szOsFile*1]; @@ -2304,6 +1867,8 @@ int sqlite3PagerOpen( sqlite3_free(pPager); return ((rc==SQLITE_OK)?SQLITE_NOMEM:rc); } + nExtra = FORCE_ALIGNMENT(nExtra); + sqlite3PcacheOpen(szPageDflt, nExtra, !memDb, xDesc, !memDb?pagerStress:0, (void *)pPager, pPager->pPCache); PAGERTRACE3("OPEN %d %s\n", FILEHANDLEID(pPager->fd), pPager->zFilename); IOTRACE(("OPEN %p %s\n", pPager, pPager->zFilename)) @@ -2351,7 +1916,7 @@ int sqlite3PagerOpen( /* pPager->pFirst = 0; */ /* pPager->pFirstSynced = 0; */ /* pPager->pLast = 0; */ - pPager->nExtra = FORCE_ALIGNMENT(nExtra); + pPager->nExtra = nExtra; pPager->journalSizeLimit = SQLITE_DEFAULT_JOURNAL_SIZE_LIMIT; assert(pPager->fd->pMethods||memDb||tempFile); if( !memDb ){ @@ -2360,24 +1925,6 @@ int sqlite3PagerOpen( /* pPager->pBusyHandler = 0; */ /* memset(pPager->aHash, 0, sizeof(pPager->aHash)); */ *ppPager = pPager; -#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT - pPager->iInUseMM = 0; - pPager->iInUseDB = 0; - if( !memDb ){ -#ifndef SQLITE_MUTEX_NOOP - sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MEM2); -#endif - sqlite3_mutex_enter(mutex); - pPager->pNext = sqlite3PagerList; - if( sqlite3PagerList ){ - assert( sqlite3PagerList->pPrev==0 ); - sqlite3PagerList->pPrev = pPager; - } - pPager->pPrev = 0; - sqlite3PagerList = pPager; - sqlite3_mutex_leave(mutex); - } -#endif return SQLITE_OK; } @@ -2420,23 +1967,24 @@ int sqlite3PagerSetPagesize(Pager *pPager, u16 *pPageSize){ int rc = SQLITE_OK; u16 pageSize = *pPageSize; assert( pageSize==0 || (pageSize>=512 && pageSize<=SQLITE_MAX_PAGE_SIZE) ); + pagerEnter(pPager); if( pageSize && pageSize!=pPager->pageSize - && !pPager->memDb && pPager->nRef==0 + && !pPager->memDb && sqlite3PcacheRefCount(pPager->pPCache)==0 ){ char *pNew = (char *)sqlite3PageMalloc(pageSize); if( !pNew ){ rc = SQLITE_NOMEM; }else{ - pagerEnter(pPager); pager_reset(pPager); pPager->pageSize = pageSize; setSectorSize(pPager); sqlite3PageFree(pPager->pTmpSpace); pPager->pTmpSpace = pNew; - pagerLeave(pPager); + sqlite3PcacheSetPageSize(pPager->pPCache, pageSize); } } *pPageSize = pPager->pageSize; + pagerLeave(pPager); return rc; } @@ -2505,6 +2053,7 @@ int sqlite3PagerReadFileheader(Pager *pPager, int N, unsigned char *pDest){ int rc = SQLITE_OK; memset(pDest, 0, N); assert(MEMDB||pPager->fd->pMethods||pPager->tempFile); + pagerEnter(pPager); if( pPager->fd->pMethods ){ IOTRACE(("DBHDR %p 0 %d\n", pPager, N)) rc = sqlite3OsRead(pPager->fd, pDest, N, 0); @@ -2512,6 +2061,7 @@ int sqlite3PagerReadFileheader(Pager *pPager, int N, unsigned char *pDest){ rc = SQLITE_OK; } } + pagerLeave(pPager); return rc; } @@ -2528,8 +2078,11 @@ int sqlite3PagerPagecount(Pager *pPager, int *pnPage){ i64 n = 0; int rc; assert( pPager!=0 ); + pagerEnter(pPager); if( pPager->errCode ){ - return pPager->errCode; + rc = pPager->errCode; + pagerLeave(pPager); + return rc; } if( pPager->dbSize>=0 ){ n = pPager->dbSize; @@ -2537,9 +2090,8 @@ int sqlite3PagerPagecount(Pager *pPager, int *pnPage){ assert(pPager->fd->pMethods||pPager->tempFile); if( (pPager->fd->pMethods) && (rc = sqlite3OsFileSize(pPager->fd, &n))!=SQLITE_OK ){ - pPager->nRef++; pager_error(pPager, rc); - pPager->nRef--; + pagerLeave(pPager); return rc; } if( n>0 && npageSize ){ @@ -2560,71 +2112,15 @@ int sqlite3PagerPagecount(Pager *pPager, int *pnPage){ if( pnPage ){ *pnPage = n; } + pagerLeave(pPager); return SQLITE_OK; } - -#ifndef SQLITE_OMIT_MEMORYDB -/* -** Clear a PgHistory block -*/ -static void clearHistory(PgHistory *pHist){ - sqlite3PageFree(pHist->pOrig); - sqlite3PageFree(pHist->pStmt); - pHist->pOrig = 0; - pHist->pStmt = 0; -} -#else -#define clearHistory(x) -#endif - /* ** Forward declaration */ static int syncJournal(Pager*); -/* -** Unlink pPg from its hash chain. Also set the page number to 0 to indicate -** that the page is not part of any hash chain. This is required because the -** sqlite3PagerMovepage() routine can leave a page in the -** pNextFree/pPrevFree list that is not a part of any hash-chain. -*/ -static void unlinkHashChain(Pager *pPager, PgHdr *pPg){ - if( pPg->pgno==0 ){ - assert( pPg->pNextHash==0 && pPg->pPrevHash==0 ); - return; - } - if( pPg->pNextHash ){ - pPg->pNextHash->pPrevHash = pPg->pPrevHash; - } - if( pPg->pPrevHash ){ - assert( pPager->aHash[pPg->pgno & (pPager->nHash-1)]!=pPg ); - pPg->pPrevHash->pNextHash = pPg->pNextHash; - }else{ - int h = pPg->pgno & (pPager->nHash-1); - pPager->aHash[h] = pPg->pNextHash; - } - if( MEMDB ){ - clearHistory(PGHDR_TO_HIST(pPg, pPager)); - } - pPg->pgno = 0; - pPg->pNextHash = pPg->pPrevHash = 0; -} - -/* -** Unlink a page from the free list (the list of all pages where nRef==0) -** and from its hash collision chain. -*/ -static void unlinkPage(PgHdr *pPg){ - Pager *pPager = pPg->pPager; - - /* Unlink from free page list */ - lruListRemove(pPg); - - /* Unlink from the pgno hash table */ - unlinkHashChain(pPager, pPg); -} - /* ** This routine is used to truncate the cache when a database ** is truncated. Drop from the cache all pages whose pgno is @@ -2638,33 +2134,7 @@ static void unlinkPage(PgHdr *pPg){ ** to zero it and hope that we error out sanely. */ static void pager_truncate_cache(Pager *pPager){ - PgHdr *pPg; - PgHdr **ppPg; - int dbSize = pPager->dbSize; - - ppPg = &pPager->pAll; - while( (pPg = *ppPg)!=0 ){ - if( pPg->pgno<=dbSize ){ - ppPg = &pPg->pNextAll; - }else if( pPg->nRef>0 ){ - memset(PGHDR_TO_DATA(pPg), 0, pPager->pageSize); - ppPg = &pPg->pNextAll; - }else{ - *ppPg = pPg->pNextAll; -#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT - if( *ppPg ){ - (*ppPg)->pPrevAll = pPg->pPrevAll; - } -#endif - IOTRACE(("PGFREE %p %d\n", pPager, pPg->pgno)); - PAGER_INCR(sqlite3_pager_pgfree_count); - unlinkPage(pPg); - makeClean(pPg); - sqlite3PageFree(pPg->pData); - sqlite3_free(pPg); - pPager->nPage--; - } - } + sqlite3PcacheTruncate(pPager->pPCache, pPager->dbSize); } /* @@ -2705,37 +2175,31 @@ static int pager_wait_on_lock(Pager *pPager, int locktype){ ** Truncate the file to the number of pages specified. */ int sqlite3PagerTruncate(Pager *pPager, Pgno nPage){ - int rc; + int rc = SQLITE_OK; assert( pPager->state>=PAGER_SHARED || MEMDB ); + + pagerEnter(pPager); + sqlite3PagerPagecount(pPager, 0); if( pPager->errCode ){ rc = pPager->errCode; - return rc; - } - if( nPage>=(unsigned)pPager->dbSize ){ - return SQLITE_OK; - } - if( MEMDB ){ - pPager->dbSize = nPage; - pager_truncate_cache(pPager); - return SQLITE_OK; - } - pagerEnter(pPager); - rc = syncJournal(pPager); - pagerLeave(pPager); - if( rc!=SQLITE_OK ){ - return rc; + }else if( nPage<(unsigned)pPager->dbSize ){ + if( MEMDB ){ + pPager->dbSize = nPage; + pager_truncate_cache(pPager); + }else{ + rc = syncJournal(pPager); + if( rc==SQLITE_OK ){ + /* Get an exclusive lock on the database before truncating. */ + rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); + } + if( rc==SQLITE_OK ){ + rc = pager_truncate(pPager, nPage); + } + } } - /* Get an exclusive lock on the database before truncating. */ - pagerEnter(pPager); - rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); pagerLeave(pPager); - if( rc!=SQLITE_OK ){ - return rc; - } - - rc = pager_truncate(pPager, nPage); return rc; } @@ -2754,23 +2218,7 @@ int sqlite3PagerTruncate(Pager *pPager, Pgno nPage){ ** to the caller. */ int sqlite3PagerClose(Pager *pPager){ -#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT - if( !MEMDB ){ -#ifndef SQLITE_MUTEX_NOOP - sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MEM2); -#endif - sqlite3_mutex_enter(mutex); - if( pPager->pPrev ){ - pPager->pPrev->pNext = pPager->pNext; - }else{ - sqlite3PagerList = pPager->pNext; - } - if( pPager->pNext ){ - pPager->pNext->pPrev = pPager->pPrev; - } - sqlite3_mutex_leave(mutex); - } -#endif + pagerEnter(pPager); disable_simulated_io_errors(); sqlite3BeginBenignMalloc(); @@ -2796,8 +2244,8 @@ int sqlite3PagerClose(Pager *pPager){ ** } */ - sqlite3_free(pPager->aHash); sqlite3PageFree(pPager->pTmpSpace); + sqlite3PcacheClose(pPager->pPCache); sqlite3_free(pPager); return SQLITE_OK; } @@ -2811,42 +2259,13 @@ Pgno sqlite3PagerPagenumber(DbPage *p){ } #endif -/* -** The page_ref() function increments the reference count for a page. -** If the page is currently on the freelist (the reference count is zero) then -** remove it from the freelist. -** -** For non-test systems, page_ref() is a macro that calls _page_ref() -** online of the reference count is zero. For test systems, page_ref() -** is a real function so that we can set breakpoints and trace it. -*/ -static void _page_ref(PgHdr *pPg){ - if( pPg->nRef==0 ){ - /* The page is currently on the freelist. Remove it. */ - lruListRemove(pPg); - pPg->pPager->nRef++; - } - pPg->nRef++; -} -#ifdef SQLITE_DEBUG - static void page_ref(PgHdr *pPg){ - if( pPg->nRef==0 ){ - _page_ref(pPg); - }else{ - pPg->nRef++; - } - } -#else -# define page_ref(P) ((P)->nRef==0?_page_ref(P):(void)(P)->nRef++) -#endif - /* ** Increment the reference count for a page. The input pointer is ** a reference to the page data. */ int sqlite3PagerRef(DbPage *pPg){ pagerEnter(pPg->pPager); - page_ref(pPg); + sqlite3PcacheRef(pPg); pagerLeave(pPg->pPager); return SQLITE_OK; } @@ -2877,7 +2296,6 @@ int sqlite3PagerRef(DbPage *pPg){ ** memory. */ static int syncJournal(Pager *pPager){ - PgHdr *pPg; int rc = SQLITE_OK; /* Sync the journal before modifying the main database @@ -2927,10 +2345,8 @@ static int syncJournal(Pager *pPager){ /* Erase the needSync flag from every page. */ - for(pPg=pPager->pAll; pPg; pPg=pPg->pNextAll){ - pPg->needSync = 0; - } - lruListSetFirstSynced(pPager); + sqlite3PcacheSetFlags(pPager->pPCache, ~PGHDR_NEED_SYNC, 0); + /* lruListSetFirstSynced(pPager); */ } #ifndef NDEBUG @@ -2939,90 +2355,14 @@ static int syncJournal(Pager *pPager){ ** invariant is true. */ else{ - for(pPg=pPager->pAll; pPg; pPg=pPg->pNextAll){ - assert( pPg->needSync==0 ); - } - assert( pPager->lru.pFirstSynced==pPager->lru.pFirst ); + sqlite3PcacheAssertFlags(pPager->pPCache, 0, PGHDR_NEED_SYNC); + /* assert( pPager->lru.pFirstSynced==pPager->lru.pFirst ); */ } #endif return rc; } -/* -** Merge two lists of pages connected by pDirty and in pgno order. -** Do not both fixing the pPrevDirty pointers. -*/ -static PgHdr *merge_pagelist(PgHdr *pA, PgHdr *pB){ - PgHdr result, *pTail; - pTail = &result; - while( pA && pB ){ - if( pA->pgnopgno ){ - pTail->pDirty = pA; - pTail = pA; - pA = pA->pDirty; - }else{ - pTail->pDirty = pB; - pTail = pB; - pB = pB->pDirty; - } - } - if( pA ){ - pTail->pDirty = pA; - }else if( pB ){ - pTail->pDirty = pB; - }else{ - pTail->pDirty = 0; - } - return result.pDirty; -} - -/* -** Sort the list of pages in accending order by pgno. Pages are -** connected by pDirty pointers. The pPrevDirty pointers are -** corrupted by this sort. -*/ -#define N_SORT_BUCKET_ALLOC 25 -#define N_SORT_BUCKET 25 -#ifdef SQLITE_TEST - int sqlite3_pager_n_sort_bucket = 0; - #undef N_SORT_BUCKET - #define N_SORT_BUCKET \ - (sqlite3_pager_n_sort_bucket?sqlite3_pager_n_sort_bucket:N_SORT_BUCKET_ALLOC) -#endif -static PgHdr *sort_pagelist(PgHdr *pIn){ - PgHdr *a[N_SORT_BUCKET_ALLOC], *p; - int i; - memset(a, 0, sizeof(a)); - while( pIn ){ - p = pIn; - pIn = p->pDirty; - p->pDirty = 0; - for(i=0; ipDirty){ - assert( p->dirty ); - p->dirty = 0; - } while( pList ){ /* If the file has not yet been opened, open it now. */ @@ -3078,7 +2412,7 @@ static int pager_write_pagelist(PgHdr *pList){ */ if( pList->pgno<=pPager->dbSize ){ i64 offset = (pList->pgno-1)*(i64)pPager->pageSize; - char *pData = CODEC2(pPager, PGHDR_TO_DATA(pList), pList->pgno, 6); + char *pData = CODEC2(pPager, pList->pData, pList->pgno, 6); PAGERTRACE4("STORE %d page %d hash(%08x)\n", PAGERID(pPager), pList->pgno, pager_pagehash(pList)); IOTRACE(("PGOUT %p %d\n", pPager, pList->pgno)); @@ -3098,33 +2432,48 @@ static int pager_write_pagelist(PgHdr *pList){ #ifdef SQLITE_CHECK_PAGES pList->pageHash = pager_pagehash(pList); #endif + makeClean(pList); pList = pList->pDirty; } + + /* sqlite3PcacheCleanAll(pPager->pPCache); */ return SQLITE_OK; } /* -** Collect every dirty page into a dirty list and -** return a pointer to the head of that list. All pages are -** collected even if they are still in use. +** This function is called by the pcache layer when it has reached some +** soft memory limit. The argument is a pointer to a purgeable Pager +** object. This function attempts to make a single dirty page that has no +** outstanding references (if one exists) clean so that it can be recycled +** by the pcache layer. */ -static PgHdr *pager_get_all_dirty_pages(Pager *pPager){ +static int pagerStress(void *p){ + Pager *pPager = (Pager *)p; + PgHdr *pPg = sqlite3PcacheDirtyPage(pPager->pPCache); + int rc = SQLITE_OK; -#ifndef NDEBUG - /* Verify the sanity of the dirty list when we are running - ** in debugging mode. This is expensive, so do not - ** do this on a normal build. */ - int n1 = 0; - int n2 = 0; - PgHdr *p; - for(p=pPager->pAll; p; p=p->pNextAll){ if( p->dirty ) n1++; } - for(p=pPager->pDirty; p; p=p->pDirty){ n2++; } - assert( n1==n2 ); -#endif - - return pPager->pDirty; + if( pPg && pPager->errCode==SQLITE_OK ){ + assert( pPg->flags&PGHDR_DIRTY ); + if( pPg->flags&PGHDR_NEED_SYNC ){ + rc = syncJournal(pPager); + if( rc==SQLITE_OK && pPager->fullSync + && !(sqlite3OsDeviceCharacteristics(pPager->fd)&SQLITE_IOCAP_SAFE_APPEND) + ){ + pPager->nRec = 0; + rc = writeJournalHdr(pPager); + } + } + if( rc==SQLITE_OK ){ + rc = pager_write_pagelist(pPg); + } + if( rc!=SQLITE_OK ){ + pager_error(pPager, rc); + } + } + return rc; } + /* ** Return 1 if there is a hot journal on the given pager. ** A hot journal is one that needs to be played back. @@ -3173,98 +2522,6 @@ static int hasHotJournal(Pager *pPager, int *pExists){ return rc; } -/* -** Try to find a page in the cache that can be recycled. -** -** This routine may return SQLITE_IOERR, SQLITE_FULL or SQLITE_OK. It -** does not set the pPager->errCode variable. -*/ -static int pager_recycle(Pager *pPager, PgHdr **ppPg){ - PgHdr *pPg; - *ppPg = 0; - - /* It is illegal to call this function unless the pager object - ** pointed to by pPager has at least one free page (page with nRef==0). - */ - assert(!MEMDB); - assert(pPager->lru.pFirst); - - /* Find a page to recycle. Try to locate a page that does not - ** require us to do an fsync() on the journal. - */ - pPg = pPager->lru.pFirstSynced; - - /* If we could not find a page that does not require an fsync() - ** on the journal file then fsync the journal file. This is a - ** very slow operation, so we work hard to avoid it. But sometimes - ** it can't be helped. - */ - if( pPg==0 && pPager->lru.pFirst ){ - if( !pPager->errCode ){ - int iDc = sqlite3OsDeviceCharacteristics(pPager->fd); - int rc = syncJournal(pPager); - if( rc!=0 ){ - return rc; - } - if( pPager->fullSync && 0==(iDc&SQLITE_IOCAP_SAFE_APPEND) ){ - /* If in full-sync mode, write a new journal header into the - ** journal file. This is done to avoid ever modifying a journal - ** header that is involved in the rollback of pages that have - ** already been written to the database (in case the header is - ** trashed when the nRec field is updated). - */ - pPager->nRec = 0; - assert( pPager->journalOff > 0 ); - assert( pPager->doNotSync==0 ); - rc = writeJournalHdr(pPager); - if( rc!=0 ){ - return rc; - } - } - } - pPg = pPager->lru.pFirst; - } - - assert( pPg->nRef==0 ); - - /* Write the page to the database file if it is dirty. - */ - if( pPg->dirty && !pPager->errCode ){ - int rc; - assert( pPg->needSync==0 ); - makeClean(pPg); - pPg->dirty = 1; - pPg->pDirty = 0; - rc = pager_write_pagelist( pPg ); - pPg->dirty = 0; - if( rc!=SQLITE_OK ){ - return rc; - } - } - assert( pPg->dirty==0 || pPager->errCode ); - - /* If the page we are recycling is marked as alwaysRollback, then - ** set the global alwaysRollback flag, thus disabling the - ** sqlite3PagerDontRollback() optimization for the rest of this transaction. - ** It is necessary to do this because the page marked alwaysRollback - ** might be reloaded at a later time but at that point we won't remember - ** that is was marked alwaysRollback. This means that all pages must - ** be marked as alwaysRollback from here on out. - */ - if( pPg->alwaysRollback ){ - IOTRACE(("ALWAYS_ROLLBACK %p\n", pPager)) - pPager->alwaysRollback = 1; - } - - /* Unlink the old page from the free list and the hash table - */ - unlinkPage(pPg); - assert( pPg->pgno==0 ); - - *ppPg = pPg; - return SQLITE_OK; -} - #ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT /* ** This function is called to free superfluous dynamically allocated memory @@ -3276,125 +2533,7 @@ static int pager_recycle(Pager *pPager, PgHdr **ppPg){ ** of bytes of memory released. */ int sqlite3PagerReleaseMemory(int nReq){ - int nReleased = 0; /* Bytes of memory released so far */ - Pager *pPager; /* For looping over pagers */ - BusyHandler *savedBusy; /* Saved copy of the busy handler */ - int rc = SQLITE_OK; - - /* Acquire the memory-management mutex - */ -#ifndef SQLITE_MUTEX_NOOP - sqlite3_mutex *mutex; /* The MEM2 mutex */ - mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MEM2); -#endif - sqlite3_mutex_enter(mutex); - - /* Signal all database connections that memory management wants - ** to have access to the pagers. - */ - for(pPager=sqlite3PagerList; pPager; pPager=pPager->pNext){ - pPager->iInUseMM = 1; - } - - while( rc==SQLITE_OK && (nReq<0 || nReleasedneedSync || pPg->pPager->iInUseDB) ){ - pPg = pPg->gfree.pNext; - } - if( !pPg ){ - pPg = sqlite3LruPageList.pFirst; - while( pPg && pPg->pPager->iInUseDB ){ - pPg = pPg->gfree.pNext; - } - } - sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_LRU)); - - /* If pPg==0, then the block above has failed to find a page to - ** recycle. In this case return early - no further memory will - ** be released. - */ - if( !pPg ) break; - - pPager = pPg->pPager; - assert(!pPg->needSync || pPg==pPager->lru.pFirst); - assert(pPg->needSync || pPg==pPager->lru.pFirstSynced); - - savedBusy = pPager->pBusyHandler; - pPager->pBusyHandler = 0; - rc = pager_recycle(pPager, &pRecycled); - pPager->pBusyHandler = savedBusy; - assert(pRecycled==pPg || rc!=SQLITE_OK); - if( rc==SQLITE_OK ){ - /* We've found a page to free. At this point the page has been - ** removed from the page hash-table, free-list and synced-list - ** (pFirstSynced). It is still in the all pages (pAll) list. - ** Remove it from this list before freeing. - ** - ** Todo: Check the Pager.pStmt list to make sure this is Ok. It - ** probably is though. - */ - PgHdr *pTmp; - assert( pPg ); - if( pPg==pPager->pAll ){ - assert(pPg->pPrevAll==0); - assert(pPg->pNextAll==0 || pPg->pNextAll->pPrevAll==pPg); - pPager->pAll = pPg->pNextAll; - if( pPager->pAll ){ - pPager->pAll->pPrevAll = 0; - } - }else{ - assert(pPg->pPrevAll); - assert(pPg->pPrevAll->pNextAll==pPg); - pTmp = pPg->pPrevAll; - pTmp->pNextAll = pPg->pNextAll; - if( pTmp->pNextAll ){ - pTmp->pNextAll->pPrevAll = pTmp; - } - } - nReleased += ( - sizeof(*pPg) + pPager->pageSize - + sizeof(u32) + pPager->nExtra - + MEMDB*sizeof(PgHistory) - ); - IOTRACE(("PGFREE %p %d *\n", pPager, pPg->pgno)); - PAGER_INCR(sqlite3_pager_pgfree_count); - sqlite3PageFree(pPg->pData); - sqlite3_free(pPg); - pPager->nPage--; - }else{ - /* An error occured whilst writing to the database file or - ** journal in pager_recycle(). The error is not returned to the - ** caller of this function. Instead, set the Pager.errCode variable. - ** The error will be returned to the user (or users, in the case - ** of a shared pager cache) of the pager for which the error occured. - */ - assert( - (rc&0xff)==SQLITE_IOERR || - rc==SQLITE_FULL || - rc==SQLITE_BUSY - ); - assert( pPager->state>=PAGER_RESERVED ); - pager_error(pPager, rc); - } - } - - /* Clear the memory management flags and release the mutex - */ - for(pPager=sqlite3PagerList; pPager; pPager=pPager->pNext){ - pPager->iInUseMM = 0; - } - sqlite3_mutex_leave(mutex); - - /* Return the number of bytes released - */ - return nReleased; + return 0; } #endif /* SQLITE_ENABLE_MEMORY_MANAGEMENT */ @@ -3410,12 +2549,12 @@ static int readDbPage(Pager *pPager, PgHdr *pPg, Pgno pgno){ return SQLITE_IOERR_SHORT_READ; } offset = (pgno-1)*(i64)pPager->pageSize; - rc = sqlite3OsRead(pPager->fd, PGHDR_TO_DATA(pPg), pPager->pageSize, offset); + rc = sqlite3OsRead(pPager->fd, pPg->pData, pPager->pageSize, offset); PAGER_INCR(sqlite3_pager_readdb_count); PAGER_INCR(pPager->nRead); IOTRACE(("PGIN %p %d\n", pPager, pgno)); if( pgno==1 ){ - memcpy(&pPager->dbFileVers, &((u8*)PGHDR_TO_DATA(pPg))[24], + memcpy(&pPager->dbFileVers, &((u8*)pPg->pData)[24], sizeof(pPager->dbFileVers)); } CODEC1(pPager, PGHDR_TO_DATA(pPg), pPg->pgno, 3); @@ -3443,7 +2582,9 @@ static int pagerSharedLock(Pager *pPager){ ** the error. Discard the contents of the pager-cache and treat any ** open journal file as a hot-journal. */ - if( !MEMDB && pPager->exclusiveMode && pPager->nRef==0 && pPager->errCode ){ + if( !MEMDB && pPager->exclusiveMode + && sqlite3PcacheRefCount(pPager->pPCache)==0 && pPager->errCode + ){ if( pPager->journalOpen ){ isErrorReset = 1; } @@ -3463,7 +2604,7 @@ static int pagerSharedLock(Pager *pPager){ sqlite3_vfs *pVfs = pPager->pVfs; if( !MEMDB ){ int isHotJournal; - assert( pPager->nRef==0 ); + assert( sqlite3PcacheRefCount(pPager->pPCache)==0 ); if( !pPager->noReadlock ){ rc = pager_wait_on_lock(pPager, SHARED_LOCK); if( rc!=SQLITE_OK ){ @@ -3557,7 +2698,7 @@ static int pagerSharedLock(Pager *pPager){ ); } - if( pPager->pAll ){ + if( sqlite3PcachePagecount(pPager->pPCache)>0 ){ /* The shared-lock has just been acquired on the database file ** and there are already pages in the cache (from a previous ** read or write transaction). Check to see if the database @@ -3611,104 +2752,6 @@ static int pagerSharedLock(Pager *pPager){ return rc; } -/* -** Allocate a PgHdr object. Either create a new one or reuse -** an existing one that is not otherwise in use. -** -** A new PgHdr structure is created if any of the following are -** true: -** -** (1) We have not exceeded our maximum allocated cache size -** as set by the "PRAGMA cache_size" command. -** -** (2) There are no unused PgHdr objects available at this time. -** -** (3) This is an in-memory database. -** -** (4) There are no PgHdr objects that do not require a journal -** file sync and a sync of the journal file is currently -** prohibited. -** -** Otherwise, reuse an existing PgHdr. In other words, reuse an -** existing PgHdr if all of the following are true: -** -** (1) We have reached or exceeded the maximum cache size -** allowed by "PRAGMA cache_size". -** -** (2) There is a PgHdr available with PgHdr->nRef==0 -** -** (3) We are not in an in-memory database -** -** (4) Either there is an available PgHdr that does not need -** to be synced to disk or else disk syncing is currently -** allowed. -*/ -static int pagerAllocatePage(Pager *pPager, PgHdr **ppPg){ - int rc = SQLITE_OK; - PgHdr *pPg; - int nByteHdr; - - /* Create a new PgHdr if any of the four conditions defined - ** above are met: */ - if( pPager->nPagemxPage - || pPager->lru.pFirst==0 - || MEMDB - || (pPager->lru.pFirstSynced==0 && pPager->doNotSync) - ){ - void *pData; - if( pPager->nPage>=pPager->nHash ){ - pager_resize_hash_table(pPager, - pPager->nHash<256 ? 256 : pPager->nHash*2); - if( pPager->nHash==0 ){ - rc = SQLITE_NOMEM; - goto pager_allocate_out; - } - } - pagerLeave(pPager); - nByteHdr = sizeof(*pPg) + sizeof(u32) + pPager->nExtra - + MEMDB*sizeof(PgHistory); - pPg = sqlite3Malloc( nByteHdr ); - if( pPg ){ - pData = sqlite3PageMalloc( pPager->pageSize ); - if( pData==0 ){ - sqlite3_free(pPg); - pPg = 0; - } - } - pagerEnter(pPager); - if( pPg==0 ){ - rc = SQLITE_NOMEM; - goto pager_allocate_out; - } - memset(pPg, 0, nByteHdr); - pPg->pData = pData; - pPg->pPager = pPager; - pPg->pNextAll = pPager->pAll; -#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT - if( pPg->pNextAll ){ - pPg->pNextAll->pPrevAll = pPg; - } -#endif - pPager->pAll = pPg; - pPager->nPage++; - }else{ - /* Recycle an existing page with a zero ref-count. */ - rc = pager_recycle(pPager, &pPg); - if( rc==SQLITE_BUSY ){ - rc = SQLITE_IOERR_BLOCKED; - } - if( rc!=SQLITE_OK ){ - goto pager_allocate_out; - } - assert( pPager->state>=SHARED_LOCK ); - assert(pPg); - } - *ppPg = pPg; - -pager_allocate_out: - return rc; -} - /* ** Make sure we have the content for a page. If the page was ** previously acquired with noContent==1, then the content was @@ -3717,10 +2760,10 @@ pager_allocate_out: ** have it. Read it in if we do not have it already. */ static int pager_get_content(PgHdr *pPg){ - if( pPg->needRead ){ + if( pPg->flags&PGHDR_NEED_READ ){ int rc = readDbPage(pPg->pPager, pPg, pPg->pgno); if( rc==SQLITE_OK ){ - pPg->needRead = 0; + pPg->flags &= ~PGHDR_NEED_READ; }else{ return rc; } @@ -3728,6 +2771,30 @@ static int pager_get_content(PgHdr *pPg){ return SQLITE_OK; } +/* +** If the reference count has reached zero, and the pager is not in the +** middle of a write transaction or opened in exclusive mode, unlock it. +*/ +static void pagerUnlockIfUnused(Pager *pPager){ + if( (sqlite3PcacheRefCount(pPager->pPCache)==0) + && (!pPager->exclusiveMode || pPager->journalOff>0) + ){ + pagerUnlockAndRollback(pPager); + } +} + +/* +** Drop a page from the cache using sqlite3PcacheDrop(). +** +** If this means there are now no pages with references to them, a rollback +** occurs and the lock on the database is removed. +*/ +static void pagerDropPage(DbPage *pPg){ + Pager *pPager = pPg->pPager; + sqlite3PcacheDrop(pPg); + pagerUnlockIfUnused(pPager); +} + /* ** Acquire a page. ** @@ -3766,10 +2833,13 @@ static int pagerAcquire( DbPage **ppPage, /* Write a pointer to the page here */ int noContent /* Do not bother reading content from disk if true */ ){ - PgHdr *pPg; + PgHdr *pPg = 0; int rc; - assert( pPager->state==PAGER_UNLOCK || pPager->nRef>0 || pgno==1 ); + assert( pPager->state==PAGER_UNLOCK + || sqlite3PcacheRefCount(pPager->pPCache)>0 + || pgno==1 + ); /* The maximum page number is 2^31. Return SQLITE_CORRUPT if a page ** number greater than this, or zero, is requested. @@ -3793,84 +2863,71 @@ static int pagerAcquire( } assert( pPager->state!=PAGER_UNLOCK ); - pPg = pager_lookup(pPager, pgno); - if( pPg==0 ){ - /* The requested page is not in the page cache. */ + rc = sqlite3PcacheFetch(pPager->pPCache, pgno, 1, &pPg); + if( rc!=SQLITE_OK ){ + return rc; + } + if( pPager->errCode && pPg->nRef==1 ){ + sqlite3PcacheDrop(pPg); + return pPager->errCode; + } + if( pPg->pPager==0 ){ + /* The pager cache has created a new page. Its content needs to + ** be initialized. + */ int nMax; - int h; PAGER_INCR(pPager->nMiss); - rc = pagerAllocatePage(pPager, &pPg); - if( rc!=SQLITE_OK ){ - return rc; + /* pPager->nRef++; */ + pPg->pPager = pPager; + if( sqlite3BitvecTest(pPager->pInJournal, pgno) ){ + pPg->flags |= PGHDR_IN_JOURNAL; } + memset(pPg->pExtra, 0, pPager->nExtra); - pPg->pgno = pgno; - assert( !MEMDB || pgno>pPager->stmtSize ); - pPg->inJournal = sqlite3BitvecTest(pPager->pInJournal, pgno); - pPg->needSync = 0; - - makeClean(pPg); - pPg->nRef = 1; - - pPager->nRef++; - if( pPager->nExtra>0 ){ - memset(PGHDR_TO_EXTRA(pPg, pPager), 0, pPager->nExtra); - } rc = sqlite3PagerPagecount(pPager, &nMax); if( rc!=SQLITE_OK ){ sqlite3PagerUnref(pPg); return rc; } - /* Populate the page with data, either by reading from the database - ** file, or by setting the entire page to zero. - */ if( nMax<(int)pgno || MEMDB || (noContent && !pPager->alwaysRollback) ){ if( pgno>pPager->mxPgno ){ sqlite3PagerUnref(pPg); return SQLITE_FULL; } - memset(PGHDR_TO_DATA(pPg), 0, pPager->pageSize); - pPg->needRead = noContent && !pPager->alwaysRollback; + memset(pPg->pData, 0, pPager->pageSize); + if( noContent && !pPager->alwaysRollback ){ + pPg->flags |= PGHDR_NEED_READ; + } IOTRACE(("ZERO %p %d\n", pPager, pgno)); }else{ rc = readDbPage(pPager, pPg, pgno); if( rc!=SQLITE_OK && rc!=SQLITE_IOERR_SHORT_READ ){ - pPg->pgno = 0; - sqlite3PagerUnref(pPg); + /* sqlite3PagerUnref(pPg); */ + pagerDropPage(pPg); return rc; } - pPg->needRead = 0; } - - /* Link the page into the page hash table */ - h = pgno & (pPager->nHash-1); - assert( pgno!=0 ); - pPg->pNextHash = pPager->aHash[h]; - pPager->aHash[h] = pPg; - if( pPg->pNextHash ){ - assert( pPg->pNextHash->pPrevHash==0 ); - pPg->pNextHash->pPrevHash = pPg; - } - #ifdef SQLITE_CHECK_PAGES pPg->pageHash = pager_pagehash(pPg); #endif }else{ /* The requested page is in the page cache. */ - assert(pPager->nRef>0 || pgno==1); + assert(sqlite3PcacheRefCount(pPager->pPCache)>0 || pgno==1); PAGER_INCR(pPager->nHit); if( !noContent ){ rc = pager_get_content(pPg); if( rc ){ + sqlite3PagerUnref(pPg); return rc; } } - page_ref(pPg); } + *ppPage = pPg; return SQLITE_OK; } + int sqlite3PagerAcquire( Pager *pPager, /* The pager open on the database file */ Pgno pgno, /* Page number to fetch */ @@ -3898,19 +2955,17 @@ int sqlite3PagerAcquire( */ DbPage *sqlite3PagerLookup(Pager *pPager, Pgno pgno){ PgHdr *pPg = 0; - assert( pPager!=0 ); assert( pgno!=0 ); pagerEnter(pPager); - if( pPager->state==PAGER_UNLOCK ){ - assert( !pPager->pAll || pPager->exclusiveMode ); - }else if( pPager->errCode && pPager->errCode!=SQLITE_FULL ){ - /* Do nothing */ - }else if( (pPg = pager_lookup(pPager, pgno))!=0 ){ - page_ref(pPg); + if( (pPager->state!=PAGER_UNLOCK) + && (pPager->errCode==SQLITE_OK || pPager->errCode==SQLITE_FULL) + ){ + sqlite3PcacheFetch(pPager->pPCache, pgno, 0, &pPg); } pagerLeave(pPager); + return pPg; } @@ -3923,39 +2978,13 @@ DbPage *sqlite3PagerLookup(Pager *pPager, Pgno pgno){ ** removed. */ int sqlite3PagerUnref(DbPage *pPg){ - Pager *pPager; - - if( pPg==0 ) return SQLITE_OK; - pPager = pPg->pPager; - - /* Decrement the reference count for this page - */ - assert( pPg->nRef>0 ); - pagerEnter(pPg->pPager); - pPg->nRef--; - - CHECK_PAGE(pPg); - - /* When the number of references to a page reach 0, call the - ** destructor and add the page to the freelist. - */ - if( pPg->nRef==0 ){ - - lruListAdd(pPg); - if( pPager->xDestructor ){ - pPager->xDestructor(pPg, pPager->pageSize); - } - - /* When all pages reach the freelist, drop the read lock from - ** the database file. - */ - pPager->nRef--; - assert( pPager->nRef>=0 ); - if( pPager->nRef==0 && (!pPager->exclusiveMode || pPager->journalOff>0) ){ - pagerUnlockAndRollback(pPager); - } + if( pPg ){ + Pager *pPager = pPg->pPager; + pagerEnter(pPager); + sqlite3PcacheRelease(pPg); + pagerUnlockIfUnused(pPager); + pagerLeave(pPager); } - pagerLeave(pPager); return SQLITE_OK; } @@ -4073,6 +3102,7 @@ int sqlite3PagerBegin(DbPage *pPg, int exFlag){ assert( pPager->state!=PAGER_UNLOCK ); if( pPager->state==PAGER_SHARED ){ assert( pPager->pInJournal==0 ); + sqlite3PcacheAssertFlags(pPager->pPCache, 0, PGHDR_IN_JOURNAL); if( MEMDB ){ pPager->state = PAGER_EXCLUSIVE; pPager->origDbSize = pPager->dbSize; @@ -4106,9 +3136,7 @@ int sqlite3PagerBegin(DbPage *pPg, int exFlag){ assert( pPager->origDbSize==0 ); assert( pPager->pInJournal==0 ); sqlite3PagerPagecount(pPager, 0); - pagerLeave(pPager); pPager->pInJournal = sqlite3BitvecCreate( pPager->dbSize ); - pagerEnter(pPager); if( !pPager->pInJournal ){ rc = SQLITE_NOMEM; }else{ @@ -4126,16 +3154,7 @@ int sqlite3PagerBegin(DbPage *pPg, int exFlag){ ** page list. */ static void makeDirty(PgHdr *pPg){ - if( pPg->dirty==0 ){ - Pager *pPager = pPg->pPager; - pPg->dirty = 1; - pPg->pDirty = pPager->pDirty; - if( pPager->pDirty ){ - pPager->pDirty->pPrevDirty = pPg; - } - pPg->pPrevDirty = 0; - pPager->pDirty = pPg; - } + sqlite3PcacheMakeDirty(pPg); } /* @@ -4143,20 +3162,7 @@ static void makeDirty(PgHdr *pPg){ ** dirty page list. */ static void makeClean(PgHdr *pPg){ - if( pPg->dirty ){ - pPg->dirty = 0; - if( pPg->pDirty ){ - assert( pPg->pDirty->pPrevDirty==pPg ); - pPg->pDirty->pPrevDirty = pPg->pPrevDirty; - } - if( pPg->pPrevDirty ){ - assert( pPg->pPrevDirty->pDirty==pPg ); - pPg->pPrevDirty->pDirty = pPg->pDirty; - }else{ - assert( pPg->pPager->pDirty==pPg ); - pPg->pPager->pDirty = pPg->pDirty; - } - } + sqlite3PcacheMakeClean(pPg); } @@ -4178,7 +3184,7 @@ static void makeClean(PgHdr *pPg){ ** reset. */ static int pager_write(PgHdr *pPg){ - void *pData = PGHDR_TO_DATA(pPg); + void *pData = pPg->pData; Pager *pPager = pPg->pPager; int rc = SQLITE_OK; @@ -4212,7 +3218,9 @@ static int pager_write(PgHdr *pPg){ ** to the journal then we can return right away. */ makeDirty(pPg); - if( pPg->inJournal && (pageInStatement(pPg) || pPager->stmtInUse==0) ){ + if( (pPg->flags&PGHDR_IN_JOURNAL) + && (pageInStatement(pPg) || pPager->stmtInUse==0) + ){ pPager->dirtyCache = 1; pPager->dbModified = 1; }else{ @@ -4242,17 +3250,14 @@ static int pager_write(PgHdr *pPg){ ** EXCLUSIVE lock on the main database file. Write the current page to ** the transaction journal if it is not there already. */ - if( !pPg->inJournal && (pPager->journalOpen || MEMDB) ){ + if( !(pPg->flags&PGHDR_IN_JOURNAL) && (pPager->journalOpen || MEMDB) ){ if( (int)pPg->pgno <= pPager->origDbSize ){ if( MEMDB ){ - PgHistory *pHist = PGHDR_TO_HIST(pPg, pPager); PAGERTRACE3("JOURNAL %d page %d\n", PAGERID(pPager), pPg->pgno); - assert( pHist->pOrig==0 ); - pHist->pOrig = sqlite3PageMalloc( pPager->pageSize ); - if( !pHist->pOrig ){ - return SQLITE_NOMEM; + rc = sqlite3PcachePreserve(pPg, 0); + if( rc!=SQLITE_OK ){ + return rc; } - memcpy(pHist->pOrig, PGHDR_TO_DATA(pPg), pPager->pageSize); }else{ u32 cksum; char *pData2; @@ -4277,7 +3282,8 @@ static int pager_write(PgHdr *pPg){ pPager->journalOff, pPager->pageSize)); PAGER_INCR(sqlite3_pager_writej_count); PAGERTRACE5("JOURNAL %d page %d needSync=%d hash(%08x)\n", - PAGERID(pPager), pPg->pgno, pPg->needSync, pager_pagehash(pPg)); + PAGERID(pPager), pPg->pgno, + ((pPg->flags&PGHDR_NEED_SYNC)?1:0), pager_pagehash(pPg)); /* An error has occured writing to the journal file. The ** transaction will be rolled back by the layer above. @@ -4289,20 +3295,25 @@ static int pager_write(PgHdr *pPg){ pPager->nRec++; assert( pPager->pInJournal!=0 ); sqlite3BitvecSet(pPager->pInJournal, pPg->pgno); - pPg->needSync = !pPager->noSync; + if( !pPager->noSync ){ + pPg->flags |= PGHDR_NEED_SYNC; + } if( pPager->stmtInUse ){ sqlite3BitvecSet(pPager->pInStmt, pPg->pgno); } } }else{ - pPg->needSync = !pPager->journalStarted && !pPager->noSync; + if( !pPager->journalStarted && !pPager->noSync ){ + pPg->flags |= PGHDR_NEED_SYNC; + } PAGERTRACE4("APPEND %d page %d needSync=%d\n", - PAGERID(pPager), pPg->pgno, pPg->needSync); + PAGERID(pPager), pPg->pgno, + ((pPg->flags&PGHDR_NEED_SYNC)?1:0)); } - if( pPg->needSync ){ + if( pPg->flags&PGHDR_NEED_SYNC ){ pPager->needSync = 1; } - pPg->inJournal = 1; + pPg->flags |= PGHDR_IN_JOURNAL; } /* If the statement journal is open and the page is not in it, @@ -4314,16 +3325,15 @@ static int pager_write(PgHdr *pPg){ && !pageInStatement(pPg) && (int)pPg->pgno<=pPager->stmtSize ){ - assert( pPg->inJournal || (int)pPg->pgno>pPager->origDbSize ); + assert( + (pPg->flags&PGHDR_IN_JOURNAL) || (int)pPg->pgno>pPager->origDbSize ); if( MEMDB ){ - PgHistory *pHist = PGHDR_TO_HIST(pPg, pPager); - assert( pHist->pStmt==0 ); - pHist->pStmt = sqlite3PageMalloc( pPager->pageSize ); - if( pHist->pStmt ){ - memcpy(pHist->pStmt, PGHDR_TO_DATA(pPg), pPager->pageSize); + rc = sqlite3PcachePreserve(pPg, 1); + if( rc!=SQLITE_OK ){ + return rc; } PAGERTRACE3("STMT-JOURNAL %d page %d\n", PAGERID(pPager), pPg->pgno); - page_add_to_stmt_list(pPg); + /* page_add_to_stmt_list(pPg); */ }else{ i64 offset = pPager->stmtNRec*(4+pPager->pageSize); char *pData2 = CODEC2(pPager, pData, pPg->pgno, 7); @@ -4411,16 +3421,17 @@ int sqlite3PagerWrite(DbPage *pDbPage){ rc = sqlite3PagerGet(pPager, pg, &pPage); if( rc==SQLITE_OK ){ rc = pager_write(pPage); - if( pPage->needSync ){ + if( pPage->flags&PGHDR_NEED_SYNC ){ needSync = 1; } sqlite3PagerUnref(pPage); } } }else if( (pPage = pager_lookup(pPager, pg))!=0 ){ - if( pPage->needSync ){ + if( pPage->flags&PGHDR_NEED_SYNC ){ needSync = 1; } + sqlite3PagerUnref(pPage); } } @@ -4433,7 +3444,8 @@ int sqlite3PagerWrite(DbPage *pDbPage){ if( needSync ){ for(ii=0; iineedSync = 1; + if( pPage ) pPage->flags |= PGHDR_NEED_SYNC; + sqlite3PagerUnref(pPage); } assert(pPager->needSync); } @@ -4454,7 +3466,7 @@ int sqlite3PagerWrite(DbPage *pDbPage){ */ #ifndef NDEBUG int sqlite3PagerIswriteable(DbPage *pPg){ - return pPg->dirty; + return pPg->flags&PGHDR_DIRTY; } #endif @@ -4488,8 +3500,8 @@ void sqlite3PagerDontWrite(DbPage *pDbPage){ if( MEMDB ) return; pagerEnter(pPager); - pPg->alwaysRollback = 1; - if( pPg->dirty && !pPager->stmtInUse ){ + pPg->flags |= PGHDR_ALWAYS_ROLLBACK; + if( (pPg->flags&PGHDR_DIRTY) && !pPager->stmtInUse ){ assert( pPager->state>=PAGER_SHARED ); if( pPager->dbSize==(int)pPg->pgno && pPager->origDbSizedbSize ){ /* If this pages is the last page in the file and the file has grown @@ -4533,7 +3545,9 @@ void sqlite3PagerDontRollback(DbPage *pPg){ ** this page (DontWrite() sets the alwaysRollback flag), then this ** function is a no-op. */ - if( pPager->journalOpen==0 || pPg->alwaysRollback || pPager->alwaysRollback ){ + if( pPager->journalOpen==0 || (pPg->flags&PGHDR_ALWAYS_ROLLBACK) + || pPager->alwaysRollback + ){ pagerLeave(pPager); return; } @@ -4541,6 +3555,7 @@ void sqlite3PagerDontRollback(DbPage *pPg){ #ifdef SQLITE_SECURE_DELETE if( pPg->inJournal || (int)pPg->pgno > pPager->origDbSize ){ + pagerLeave(pPager); return; } #endif @@ -4559,8 +3574,8 @@ void sqlite3PagerDontRollback(DbPage *pPg){ assert( pPager->pInJournal!=0 ); sqlite3BitvecSet(pPager->pInJournal, pPg->pgno); - pPg->inJournal = 1; - pPg->needRead = 0; + pPg->flags |= PGHDR_IN_JOURNAL; + pPg->flags &= ~PGHDR_NEED_READ; if( pPager->stmtInUse ){ assert( pPager->stmtSize >= pPager->origDbSize ); sqlite3BitvecSet(pPager->pInStmt, pPg->pgno); @@ -4599,11 +3614,11 @@ static int pager_incr_changecounter(Pager *pPager, int isDirect){ /* Increment the value just read and write it back to byte 24. */ change_counter = sqlite3Get4byte((u8*)pPager->dbFileVers); change_counter++; - put32bits(((char*)PGHDR_TO_DATA(pPgHdr))+24, change_counter); + put32bits(((char*)pPgHdr->pData)+24, change_counter); #ifdef SQLITE_ENABLE_ATOMIC_WRITE if( isDirect && pPager->fd->pMethods ){ - const void *zBuf = PGHDR_TO_DATA(pPgHdr); + const void *zBuf = pPgHdr->pData; rc = sqlite3OsWrite(pPager->fd, zBuf, pPager->pageSize, 0); } #endif @@ -4691,12 +3706,13 @@ int sqlite3PagerCommitPhaseOne( ** If the optimization can be used, then the journal file will never ** be created for this transaction. */ + pPg = sqlite3PcacheDirtyList(pPager->pPCache); int useAtomicWrite = ( !zMaster && pPager->journalOpen && pPager->journalOff==jrnlBufferSize(pPager) && nTrunc==0 && - (0==pPager->pDirty || 0==pPager->pDirty->pDirty) + (pPg==0 || pPg->pDirty==0) ); assert( pPager->journalOpen || pPager->journalMode==PAGER_JOURNALMODE_OFF ); if( useAtomicWrite ){ @@ -4765,7 +3781,7 @@ int sqlite3PagerCommitPhaseOne( #endif /* Write all dirty pages to the database file */ - pPg = pager_get_all_dirty_pages(pPager); + pPg = sqlite3PcacheDirtyList(pPager->pPCache); rc = pager_write_pagelist(pPg); if( rc!=SQLITE_OK ){ assert( rc!=SQLITE_IOERR_BLOCKED ); @@ -4778,7 +3794,7 @@ int sqlite3PagerCommitPhaseOne( */ goto sync_exit; } - pPager->pDirty = 0; + sqlite3PcacheCleanAll(pPager->pPCache); /* Sync the database file. */ if( !pPager->noSync && !noSync ){ @@ -4813,8 +3829,7 @@ sync_exit: ** is returned. */ int sqlite3PagerCommitPhaseTwo(Pager *pPager){ - int rc; - PgHdr *pPg; + int rc = SQLITE_OK; if( pPager->errCode ){ return pPager->errCode; @@ -4831,34 +3846,17 @@ int sqlite3PagerCommitPhaseTwo(Pager *pPager){ pagerEnter(pPager); PAGERTRACE2("COMMIT %d\n", PAGERID(pPager)); if( MEMDB ){ - pPg = pager_get_all_dirty_pages(pPager); - while( pPg ){ - PgHistory *pHist = PGHDR_TO_HIST(pPg, pPager); - clearHistory(pHist); - pPg->dirty = 0; - pPg->inJournal = 0; - pHist->inStmt = 0; - pPg->needSync = 0; - pHist->pPrevStmt = pHist->pNextStmt = 0; - pPg = pPg->pDirty; - } - pPager->pDirty = 0; -#ifndef NDEBUG - for(pPg=pPager->pAll; pPg; pPg=pPg->pNextAll){ - PgHistory *pHist = PGHDR_TO_HIST(pPg, pPager); - assert( !pPg->alwaysRollback ); - assert( !pHist->pOrig ); - assert( !pHist->pStmt ); - } -#endif - pPager->pStmt = 0; + sqlite3PcacheCommit(pPager->pPCache, 0); + sqlite3PcacheCleanAll(pPager->pPCache); + sqlite3PcacheSetFlags(pPager->pPCache, + ~(PGHDR_IN_JOURNAL | PGHDR_NEED_SYNC | PGHDR_ALWAYS_ROLLBACK), 0 + ); pPager->state = PAGER_SHARED; - pagerLeave(pPager); - return SQLITE_OK; + }else{ + assert( pPager->state==PAGER_SYNCED || !pPager->dirtyCache ); + rc = pager_end_transaction(pPager, pPager->setMaster); + rc = pager_error(pPager, rc); } - assert( pPager->state==PAGER_SYNCED || !pPager->dirtyCache ); - rc = pager_end_transaction(pPager, pPager->setMaster); - rc = pager_error(pPager, rc); pagerLeave(pPager); return rc; } @@ -4876,76 +3874,47 @@ int sqlite3PagerCommitPhaseTwo(Pager *pPager){ ** SQLITE_OK is returned. */ int sqlite3PagerRollback(Pager *pPager){ - int rc; + int rc = SQLITE_OK; PAGERTRACE2("ROLLBACK %d\n", PAGERID(pPager)); + pagerEnter(pPager); if( MEMDB ){ - PgHdr *p; - for(p=pPager->pAll; p; p=p->pNextAll){ - PgHistory *pHist; - assert( !p->alwaysRollback ); - if( !p->dirty ){ - assert( !((PgHistory *)PGHDR_TO_HIST(p, pPager))->pOrig ); - assert( !((PgHistory *)PGHDR_TO_HIST(p, pPager))->pStmt ); - continue; - } - - pHist = PGHDR_TO_HIST(p, pPager); - if( pHist->pOrig ){ - memcpy(PGHDR_TO_DATA(p), pHist->pOrig, pPager->pageSize); - PAGERTRACE3("ROLLBACK-PAGE %d of %d\n", p->pgno, PAGERID(pPager)); - }else{ - PAGERTRACE3("PAGE %d is clean on %d\n", p->pgno, PAGERID(pPager)); - } - clearHistory(pHist); - p->dirty = 0; - p->inJournal = 0; - pHist->inStmt = 0; - pHist->pPrevStmt = pHist->pNextStmt = 0; - if( pPager->xReiniter ){ - pPager->xReiniter(p, pPager->pageSize); - } - } - pPager->pDirty = 0; - pPager->pStmt = 0; + sqlite3PcacheRollback(pPager->pPCache, 1); + sqlite3PcacheRollback(pPager->pPCache, 0); + sqlite3PcacheCleanAll(pPager->pPCache); + sqlite3PcacheSetFlags(pPager->pPCache, + ~(PGHDR_IN_JOURNAL | PGHDR_NEED_SYNC | PGHDR_ALWAYS_ROLLBACK), 0 + ); pPager->dbSize = pPager->origDbSize; pager_truncate_cache(pPager); pPager->stmtInUse = 0; pPager->state = PAGER_SHARED; - return SQLITE_OK; - } - - pagerEnter(pPager); - if( !pPager->dirtyCache || !pPager->journalOpen ){ + }else if( !pPager->dirtyCache || !pPager->journalOpen ){ rc = pager_end_transaction(pPager, pPager->setMaster); - pagerLeave(pPager); - return rc; - } - - if( pPager->errCode && pPager->errCode!=SQLITE_FULL ){ + }else if( pPager->errCode && pPager->errCode!=SQLITE_FULL ){ if( pPager->state>=PAGER_EXCLUSIVE ){ pager_playback(pPager, 0); } - pagerLeave(pPager); - return pPager->errCode; - } - if( pPager->state==PAGER_RESERVED ){ - int rc2; - rc = pager_playback(pPager, 0); - rc2 = pager_end_transaction(pPager, pPager->setMaster); - if( rc==SQLITE_OK ){ - rc = rc2; - } + rc = pPager->errCode; }else{ - rc = pager_playback(pPager, 0); - } - /* pager_reset(pPager); */ - pPager->dbSize = -1; + if( pPager->state==PAGER_RESERVED ){ + int rc2; + rc = pager_playback(pPager, 0); + rc2 = pager_end_transaction(pPager, pPager->setMaster); + if( rc==SQLITE_OK ){ + rc = rc2; + } + }else{ + rc = pager_playback(pPager, 0); + } - /* If an error occurs during a ROLLBACK, we can no longer trust the pager - ** cache. So call pager_error() on the way out to make any error - ** persistent. - */ - rc = pager_error(pPager, rc); + pPager->dbSize = -1; + + /* If an error occurs during a ROLLBACK, we can no longer trust the pager + ** cache. So call pager_error() on the way out to make any error + ** persistent. + */ + rc = pager_error(pPager, rc); + } pagerLeave(pPager); return rc; } @@ -4962,7 +3931,7 @@ int sqlite3PagerIsreadonly(Pager *pPager){ ** Return the number of references to the pager. */ int sqlite3PagerRefcount(Pager *pPager){ - return pPager->nRef; + return sqlite3PcacheRefCount(pPager->pPCache); } #ifdef SQLITE_TEST @@ -4971,9 +3940,10 @@ int sqlite3PagerRefcount(Pager *pPager){ */ int *sqlite3PagerStats(Pager *pPager){ static int a[11]; - a[0] = pPager->nRef; - a[1] = pPager->nPage; - a[2] = pPager->mxPage; + pagerEnter(pPager); + a[0] = sqlite3PcacheRefCount(pPager->pPCache); + a[1] = sqlite3PcachePagecount(pPager->pPCache); + a[2] = sqlite3PcacheGetCachesize(pPager->pPCache); a[3] = pPager->dbSize; a[4] = pPager->state; a[5] = pPager->errCode; @@ -4982,6 +3952,7 @@ int *sqlite3PagerStats(Pager *pPager){ a[8] = 0; /* Used to be pPager->nOvfl */ a[9] = pPager->nRead; a[10] = pPager->nWrite; + pagerLeave(pPager); return a; } int sqlite3PagerIsMemdb(Pager *pPager){ @@ -5012,10 +3983,8 @@ static int pagerStmtBegin(Pager *pPager){ return SQLITE_OK; } assert( pPager->journalOpen ); - pagerLeave(pPager); assert( pPager->pInStmt==0 ); pPager->pInStmt = sqlite3BitvecCreate(pPager->dbSize); - pagerEnter(pPager); if( pPager->pInStmt==0 ){ /* sqlite3OsLock(pPager->fd, SHARED_LOCK); */ return SQLITE_NOMEM; @@ -5056,26 +4025,16 @@ int sqlite3PagerStmtBegin(Pager *pPager){ int sqlite3PagerStmtCommit(Pager *pPager){ pagerEnter(pPager); if( pPager->stmtInUse ){ - PgHdr *pPg, *pNext; PAGERTRACE2("STMT-COMMIT %d\n", PAGERID(pPager)); if( !MEMDB ){ /* sqlite3OsTruncate(pPager->stfd, 0); */ sqlite3BitvecDestroy(pPager->pInStmt); pPager->pInStmt = 0; }else{ - for(pPg=pPager->pStmt; pPg; pPg=pNext){ - PgHistory *pHist = PGHDR_TO_HIST(pPg, pPager); - pNext = pHist->pNextStmt; - assert( pHist->inStmt ); - pHist->inStmt = 0; - pHist->pPrevStmt = pHist->pNextStmt = 0; - sqlite3PageFree(pHist->pStmt); - pHist->pStmt = 0; - } + sqlite3PcacheCommit(pPager->pPCache, 1); } pPager->stmtNRec = 0; pPager->stmtInUse = 0; - pPager->pStmt = 0; } pPager->stmtAutoopen = 0; pagerLeave(pPager); @@ -5091,16 +4050,7 @@ int sqlite3PagerStmtRollback(Pager *pPager){ if( pPager->stmtInUse ){ PAGERTRACE2("STMT-ROLLBACK %d\n", PAGERID(pPager)); if( MEMDB ){ - PgHdr *pPg; - PgHistory *pHist; - for(pPg=pPager->pStmt; pPg; pPg=pHist->pNextStmt){ - pHist = PGHDR_TO_HIST(pPg, pPager); - if( pHist->pStmt ){ - memcpy(PGHDR_TO_DATA(pPg), pHist->pStmt, pPager->pageSize); - sqlite3PageFree(pHist->pStmt); - pHist->pStmt = 0; - } - } + sqlite3PcacheRollback(pPager->pPCache, 1); pPager->dbSize = pPager->stmtSize; pager_truncate_cache(pPager); rc = SQLITE_OK; @@ -5200,16 +4150,16 @@ void sqlite3PagerSetCodec( */ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ PgHdr *pPgOld; /* The page being overwritten. */ - int h; Pgno needSyncPgno = 0; - pagerEnter(pPager); assert( pPg->nRef>0 ); PAGERTRACE5("MOVE %d page %d (needSync=%d) moves to %d\n", - PAGERID(pPager), pPg->pgno, pPg->needSync, pgno); + PAGERID(pPager), pPg->pgno, (pPg->flags&PGHDR_NEED_SYNC)?1:0, pgno); IOTRACE(("MOVE %p %d %d\n", pPager, pPg->pgno, pgno)) + pagerEnter(pPager); + pager_get_content(pPg); /* If the journal needs to be sync()ed before page pPg->pgno can @@ -5219,44 +4169,33 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ ** the journal needs to be sync()ed before database page pPg->pgno ** can be written to. The caller has already promised not to write to it. */ - if( pPg->needSync && !isCommit ){ + if( (pPg->flags&PGHDR_NEED_SYNC) && !isCommit ){ needSyncPgno = pPg->pgno; - assert( pPg->inJournal || (int)pgno>pPager->origDbSize ); - assert( pPg->dirty ); + assert( (pPg->flags&PGHDR_IN_JOURNAL) || (int)pgno>pPager->origDbSize ); + assert( pPg->flags&PGHDR_DIRTY ); assert( pPager->needSync ); } - /* Unlink pPg from its hash-chain */ - unlinkHashChain(pPager, pPg); - /* If the cache contains a page with page-number pgno, remove it ** from its hash chain. Also, if the PgHdr.needSync was set for ** page pgno before the 'move' operation, it needs to be retained ** for the page moved there. */ - pPg->needSync = 0; + pPg->flags &= ~(PGHDR_NEED_SYNC|PGHDR_IN_JOURNAL); pPgOld = pager_lookup(pPager, pgno); + assert( !pPgOld || pPgOld->nRef==1 ); if( pPgOld ){ - assert( pPgOld->nRef==0 ); - unlinkHashChain(pPager, pPgOld); - makeClean(pPgOld); - pPg->needSync = pPgOld->needSync; - }else{ - pPg->needSync = 0; + pPg->flags |= (pPgOld->flags&PGHDR_NEED_SYNC); + } + if( sqlite3BitvecTest(pPager->pInJournal, pgno) ){ + pPg->flags |= PGHDR_IN_JOURNAL; } - pPg->inJournal = sqlite3BitvecTest(pPager->pInJournal, pgno); - /* Change the page number for pPg and insert it into the new hash-chain. */ - assert( pgno!=0 ); - pPg->pgno = pgno; - h = pgno & (pPager->nHash-1); - if( pPager->aHash[h] ){ - assert( pPager->aHash[h]->pPrevHash==0 ); - pPager->aHash[h]->pPrevHash = pPg; + sqlite3PcacheMove(pPg, pgno); + if( pPgOld ){ + sqlite3PcacheMove(pPgOld, 0); + sqlite3PcacheRelease(pPgOld); } - pPg->pNextHash = pPager->aHash[h]; - pPager->aHash[h] = pPg; - pPg->pPrevHash = 0; makeDirty(pPg); pPager->dirtyCache = 1; @@ -5292,8 +4231,8 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ return rc; } pPager->needSync = 1; - pPgHdr->needSync = 1; - pPgHdr->inJournal = 1; + pPgHdr->flags |= PGHDR_NEED_SYNC; + pPgHdr->flags |= PGHDR_IN_JOURNAL; makeDirty(pPgHdr); sqlite3PagerUnref(pPgHdr); } @@ -5307,7 +4246,8 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ ** Return a pointer to the data for the specified page. */ void *sqlite3PagerGetData(DbPage *pPg){ - return PGHDR_TO_DATA(pPg); + assert( pPg->nRef>0 ); + return pPg->pData; } /* @@ -5316,7 +4256,7 @@ void *sqlite3PagerGetData(DbPage *pPg){ */ void *sqlite3PagerGetExtra(DbPage *pPg){ Pager *pPager = pPg->pPager; - return (pPager?PGHDR_TO_EXTRA(pPg, pPager):0); + return (pPager?pPg->pExtra:0); } /* @@ -5374,4 +4314,8 @@ i64 sqlite3PagerJournalSizeLimit(Pager *pPager, i64 iLimit){ return pPager->journalSizeLimit; } +void sqlite3PagerAlwaysRollback(Pager *pPager){ + pPager->alwaysRollback = 1; +} + #endif /* SQLITE_OMIT_DISKIO */ diff --git a/src/pager.h b/src/pager.h index b62b089f49..d6a200b41f 100644 --- a/src/pager.h +++ b/src/pager.h @@ -13,7 +13,7 @@ ** subsystem. The page cache subsystem reads and writes a file a page ** at a time and provides a journal for rollback. ** -** @(#) $Id: pager.h,v 1.77 2008/07/16 18:17:56 danielk1977 Exp $ +** @(#) $Id: pager.h,v 1.78 2008/08/20 14:49:25 danielk1977 Exp $ */ #ifndef _PAGER_H_ @@ -70,7 +70,7 @@ typedef struct PgHdr DbPage; ** See source code comments for a detailed description of the following ** routines: */ -int sqlite3PagerOpen(sqlite3_vfs *, Pager **ppPager, const char*, int,int,int); +int sqlite3PagerOpen(sqlite3_vfs *, Pager **ppPager, const char*, void(*)(DbPage*), int,int,int); void sqlite3PagerSetBusyhandler(Pager*, BusyHandler *pBusyHandler); void sqlite3PagerSetDestructor(Pager*, void(*)(DbPage*,int)); void sqlite3PagerSetReiniter(Pager*, void(*)(DbPage*,int)); @@ -113,6 +113,7 @@ int sqlite3PagerJournalMode(Pager *, int); i64 sqlite3PagerJournalSizeLimit(Pager *, i64); void *sqlite3PagerTempSpace(Pager*); int sqlite3PagerSync(Pager *pPager); +void sqlite3PagerAlwaysRollback(Pager *pPager); #if defined(SQLITE_ENABLE_MEMORY_MANAGEMENT) && !defined(SQLITE_OMIT_DISKIO) int sqlite3PagerReleaseMemory(int); diff --git a/src/pcache.c b/src/pcache.c new file mode 100644 index 0000000000..e207e0ed6a --- /dev/null +++ b/src/pcache.c @@ -0,0 +1,1132 @@ +/* +** 2008 August 05 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file implements that page cache. +** +** @(#) $Id: pcache.c,v 1.1 2008/08/20 14:49:25 danielk1977 Exp $ +*/ +#include "sqliteInt.h" + +/* +** A complete page cache is an instance of this structure. +*/ +struct PCache { + PCache *pNextAll, *pPrevAll; /* List of all page caches */ + int szPage; /* Size of every page in this cache */ + int szExtra; /* Size of extra space for each page */ + int nHash; /* Number of slots in apHash[] */ + int nPage; /* Total number of pages in apHash */ + int nMax; /* Configured cache size */ + PgHdr **apHash; /* Hash table for fast lookup by pgno */ + int bPurgeable; /* True if pages are on backing store */ + void (*xDestroy)(PgHdr*); /* Called when refcnt goes 1->0 */ + int (*xStress)(void*); /* Call to try to make pages clean */ + void *pStress; /* Argument to xStress */ + PgHdr *pClean; /* List of clean pages in use */ + PgHdr *pDirty; /* List of dirty pages */ + int nRef; /* Number of outstanding page refs */ + + int iInUseMM; + int iInUseDB; +}; + +/* +** Free slots in the page block allocator +*/ +typedef struct PgFreeslot PgFreeslot; +struct PgFreeslot { + PgFreeslot *pNext; /* Next free slot */ +}; + +/* +** Global data for the page cache. +** +** The maximum number of cached pages stored by the system is determined +** by the pcache.mxPage and pcache.mxPagePurgeable variables. If +** mxPage is non-zero, then the system tries to limit the number of +** cached pages stored to mxPage. In this case mxPagePurgeable is not +** used. +** +** If mxPage is zero, then the system tries to limit the number of +** pages held by purgable caches to mxPagePurgeable. +*/ +static struct PCacheGlobal { + int isInit; /* True when initialized */ + sqlite3_mutex *mutex_mem2; /* static mutex MUTEX_STATIC_MEM2 */ + sqlite3_mutex *mutex_lru; /* static mutex MUTEX_STATIC_LRU */ + PCache *pAll; /* list of all page caches */ + int nPage; /* Number of pages */ + int nPurgeable; /* Number of pages in purgable caches */ + int mxPage; /* Globally configured page maximum */ + int mxPagePurgeable; /* Purgeable page maximum */ + PgHdr *pLruHead, *pLruTail; /* Global LRU list of unused pages */ + int szSlot; /* Size of each free slot */ + void *pStart, *pEnd; /* Bounds of pagecache malloc range */ + PgFreeslot *pFree; /* Free page blocks */ +} pcache = {0}; + +/* +** All global variables used by this module (most of which are grouped +** together in global structure "pcache" above) except the list of all +** pager-caches starting with pcache.pAll, are protected by the static +** SQLITE_MUTEX_STATIC_LRU mutex. A pointer to this mutex is stored in +** variable "pcache.mutex_lru". +** +** The list of all pager-caches (PCache structures) headed by pcache.pAll +** is protected by SQLITE_MUTEX_STATIC_MEM2. +** +** Access to the contents of the individual PCache structures is not +** protected. It is the job of the caller to ensure that these structures +** are accessed in a thread-safe manner. However, this module provides the +** functions sqlite3PcacheLock() and sqlite3PcacheUnlock() that may be used +** by the caller to increment/decrement a lock-count on an individual +** pager-cache object. This module guarantees that the xStress() callback +** will not be invoked on a pager-cache with a non-zero lock-count except +** from within a call to sqlite3PcacheFetch() on the same pager. A call +** to sqlite3PcacheLock() may block if such an xStress() call is currently +** underway. +** +** Before the xStress callback of a pager-cache (PCache) is invoked, the +** SQLITE_MUTEX_STATIC_MEM2 mutex is obtained and the SQLITE_MUTEX_STATIC_LRU +** mutex released (in that order) before making the call. +*/ + +#define enterPcacheGlobal() sqlite3_mutex_enter(pcache.mutex_lru) +#define exitPcacheGlobal() sqlite3_mutex_leave(pcache.mutex_lru) + +/* +** Increment the reference count on both page p and its cache by n. +*/ +static void page_ref(PgHdr *p, int n){ + /* This next block assert()s that the number of references to the + ** PCache is the sum of the number of references to all pages in + ** the PCache. This is a bit expensive to leave turned on all the + ** time, even in debugging builds. + */ +#if 0 + PgHdr *pHdr; + int nRef = 0; + for(pHdr=p->pCache->pClean; pHdr; pHdr=pHdr->pNext) nRef += pHdr->nRef; + for(pHdr=p->pCache->pDirty; pHdr; pHdr=pHdr->pNext) nRef += pHdr->nRef; + assert( p->pCache->nRef==nRef ); +#endif + p->nRef += n; + p->pCache->nRef += n; +} + +/********************************** Linked List Management ********************/ + +static int check_hash_count(PCache *pCache){ + int i; + int nPage = 0; + for(i=0; inHash; i++){ + PgHdr *p; + for(p=pCache->apHash[i]; p; p=p->pNextHash){ + nPage++; + } + } + assert( nPage==pCache->nPage ); + return 1; +} + +/* +** Remove a page from its hash table (PCache.apHash[]). +*/ +static void pcacheRemoveFromHash(PgHdr *pPage){ + if( pPage->pPrevHash ){ + pPage->pPrevHash->pNextHash = pPage->pNextHash; + }else{ + PCache *pCache = pPage->pCache; + u32 h = pPage->pgno % pCache->nHash; + assert( pCache->apHash[h]==pPage ); + pCache->apHash[h] = pPage->pNextHash; + } + if( pPage->pNextHash ){ + pPage->pNextHash->pPrevHash = pPage->pPrevHash; + } + pPage->pCache->nPage--; + assert( check_hash_count(pPage->pCache) ); +} + +/* +** Insert a page into the hash table +*/ +static void pcacheAddToHash(PgHdr *pPage){ + PCache *pCache = pPage->pCache; + u32 h = pPage->pgno % pCache->nHash; + pPage->pNextHash = pCache->apHash[h]; + pPage->pPrevHash = 0; + if( pCache->apHash[h] ){ + pCache->apHash[h]->pPrevHash = pPage; + } + pCache->apHash[h] = pPage; + pCache->nPage++; + assert( check_hash_count(pCache) ); +} + +/* +** Resize the hash table to nHash buckets. +*/ +static int pcacheResizeHash(PCache *pCache, int nHash){ +#ifdef SQLITE_MALLOC_SOFT_LIMIT + if( nHash*sizeof(PgHdr *)>SQLITE_MALLOC_SOFT_LIMIT ){ + nHash = SQLITE_MALLOC_SOFT_LIMIT/sizeof(PgHdr *); + } +#endif + if( nHash>pCache->nHash ){ + PgHdr *p; + PgHdr **pNew = (PgHdr **)sqlite3_malloc(sizeof(PgHdr *)*nHash); + if( !pNew ){ + return SQLITE_NOMEM; + } + memset(pNew, 0, sizeof(PgHdr *)*nHash); + sqlite3_free(pCache->apHash); + pCache->apHash = pNew; + pCache->nHash = nHash; + pCache->nPage = 0; + + for(p=pCache->pClean; p; p=p->pNext){ + pcacheAddToHash(p); + } + for(p=pCache->pDirty; p; p=p->pNext){ + pcacheAddToHash(p); + } + } + return SQLITE_OK; +} + +/* +** Remove a page from a linked list that is headed by *ppHead. +** *ppHead is either PCache.pClean or PCache.pDirty. +*/ +static void pcacheRemoveFromList(PgHdr **ppHead, PgHdr *pPage){ + if( pPage->pPrev ){ + pPage->pPrev->pNext = pPage->pNext; + }else{ + assert( *ppHead==pPage ); + *ppHead = pPage->pNext; + } + if( pPage->pNext ){ + pPage->pNext->pPrev = pPage->pPrev; + } +} + +/* +** Add a page from a linked list that is headed by *ppHead. +** *ppHead is either PCache.pClean or PCache.pDirty. +*/ +static void pcacheAddToList(PgHdr **ppHead, PgHdr *pPage){ + if( (*ppHead) ){ + (*ppHead)->pPrev = pPage; + } + pPage->pNext = *ppHead; + pPage->pPrev = 0; + *ppHead = pPage; +} + +/* +** Remove a page from the global LRU list +*/ +static void pcacheRemoveFromLruList(PgHdr *pPage){ + assert( sqlite3_mutex_held(pcache.mutex_lru) ); + if( pPage->pCache->bPurgeable==0 ) return; + if( pPage->pNextLru ){ + pPage->pNextLru->pPrevLru = pPage->pPrevLru; + }else{ + assert( pcache.pLruTail==pPage ); + pcache.pLruTail = pPage->pPrevLru; + } + if( pPage->pPrevLru ){ + pPage->pPrevLru->pNextLru = pPage->pNextLru; + }else{ + assert( pcache.pLruHead==pPage ); + pcache.pLruHead = pPage->pNextLru; + } +} + +/* +** Add a page to the global LRU list. The page is normally added +** to the front of the list so that it will be the last page recycled. +** However, if the PGHDR_REUSE_UNLIKELY bit is set, the page is added +** to the end of the LRU list so that it will be the next to be recycled. +*/ +static void pcacheAddToLruList(PgHdr *pPage){ + assert( sqlite3_mutex_held(pcache.mutex_lru) ); + if( pPage->pCache->bPurgeable==0 ) return; + if( pcache.pLruTail && (pPage->flags & PGHDR_REUSE_UNLIKELY)!=0 ){ + /* If reuse is unlikely. Put the page at the end of the LRU list + ** where it will be recycled sooner rather than later. + */ + assert( pcache.pLruHead ); + pPage->pNextLru = 0; + pPage->pPrevLru = pcache.pLruTail; + pcache.pLruTail->pNextLru = pPage; + pcache.pLruTail = pPage; + pPage->flags &= ~PGHDR_REUSE_UNLIKELY; + }else{ + /* If reuse is possible. the page goes at the beginning of the LRU + ** list so that it will be the last to be recycled. + */ + if( pcache.pLruHead ){ + pcache.pLruHead->pPrevLru = pPage; + } + pPage->pNextLru = pcache.pLruHead; + pcache.pLruHead = pPage; + pPage->pPrevLru = 0; + if( pcache.pLruTail==0 ){ + pcache.pLruTail = pPage; + } + } +} + +/*********************************************** Memory Allocation *********** +** +** Initialize the page cache memory pool. +** +** This must be called at start-time when no page cache lines are +** checked out. This function is not threadsafe. +*/ +void sqlite3PCacheBufferSetup(void *pBuf, int sz, int n){ + PgFreeslot *p; + sz &= ~7; + pcache.szSlot = sz; + pcache.pStart = pBuf; + pcache.pFree = 0; + while( n-- ){ + p = (PgFreeslot*)pBuf; + p->pNext = pcache.pFree; + pcache.pFree = p; + pBuf = (void*)&((char*)pBuf)[sz]; + } + pcache.pEnd = pBuf; +} + +/* +** Allocate a page cache line. Look in the page cache memory pool first +** and use an element from it first if available. If nothing is available +** in the page cache memory pool, go to the general purpose memory allocator. +*/ +void *pcacheMalloc(int sz){ + assert( sqlite3_mutex_held(pcache.mutex_lru) ); + if( sz<=pcache.szSlot && pcache.pFree ){ + PgFreeslot *p = pcache.pFree; + pcache.pFree = p->pNext; + sqlite3StatusSet(SQLITE_STATUS_PAGECACHE_SIZE, sz); + sqlite3StatusAdd(SQLITE_STATUS_PAGECACHE_USED, 1); + return (void*)p; + }else{ + void *p = sqlite3Malloc(sz); + if( p ){ + sz = sqlite3MallocSize(p); + sqlite3StatusAdd(SQLITE_STATUS_PAGECACHE_OVERFLOW, sz); + } + return p; + } +} +void *sqlite3PageMalloc(sz){ + void *p; + enterPcacheGlobal(); + p = pcacheMalloc(sz); + exitPcacheGlobal(); + return p; +} + +/* +** Release a pager memory allocation +*/ +void pcacheFree(void *p){ + assert( sqlite3_mutex_held(pcache.mutex_lru) ); + if( p==0 ) return; + if( p>=pcache.pStart && ppNext = pcache.pFree; + pcache.pFree = pSlot; + }else{ + int iSize = sqlite3MallocSize(p); + sqlite3StatusAdd(SQLITE_STATUS_PAGECACHE_OVERFLOW, -iSize); + sqlite3_free(p); + } +} +void sqlite3PageFree(void *p){ + enterPcacheGlobal(); + pcacheFree(p); + exitPcacheGlobal(); +} + +/* +** Allocate a new page. +*/ +static PgHdr *pcachePageAlloc(int szPage, int szExtra, int bPurgeable){ + PgHdr *p; + int sz = sizeof(*p) + szPage + szExtra; + assert( sqlite3_mutex_held(pcache.mutex_lru) ); + p = pcacheMalloc( sz ); + if( p==0 ) return 0; + memset(p, 0, sizeof(PgHdr)); + p->pData = (void*)&p[1]; + p->pExtra = (void*)&((char*)p->pData)[szPage]; + + pcache.nPage++; + if( bPurgeable ){ + pcache.nPurgeable++; + } + + return p; +} + +/* +** Deallocate a page +*/ +static void pcachePageFree(PgHdr *p){ + assert( sqlite3_mutex_held(pcache.mutex_lru) ); + pcache.nPage--; + if( p->pCache->bPurgeable ){ + pcache.nPurgeable--; + } + pcacheFree(p->apSave[0]); + pcacheFree(p->apSave[1]); + pcacheFree(p); +} + +/* +** Obtain space for a page. Try to recycle an old page if the limit on the +** number of pages has been reached. If the limit has not been reached or +** there are no pages eligible for recycling, allocate a new page. +** +** Return a pointer to the new page, or NULL if an OOM condition occurs. +*/ +static PgHdr *pcacheRecycleOrAlloc(PCache *pCache){ + PgHdr *p = 0; + + int szPage = pCache->szPage; + int szExtra = pCache->szExtra; + int bPurg = pCache->bPurgeable; + + assert( pcache.isInit ); + assert( sqlite3_mutex_notheld(pcache.mutex_lru) ); + + enterPcacheGlobal(); + + if( (pcache.mxPage && pcache.nPage>=pcache.mxPage) + || (!pcache.mxPage && bPurg && pcache.nPurgeable>=pcache.mxPagePurgeable) + ){ + PCache *pCsr; + + /* If the above test succeeds, then a page will be obtained by recycling + ** an existing page. + */ + if( !pcache.pLruTail && SQLITE_OK==sqlite3_mutex_try(pcache.mutex_mem2) ){ + + /* Invoke xStress() callbacks until the LRU list contains at least one + ** page that can be reused or until the xStress() callback of all + ** caches has been invoked. + */ + for(pCsr=pcache.pAll; pCsr&&!pcache.pLruTail; pCsr=pCsr->pNextAll){ + assert( pCsr->iInUseMM==0 ); + pCsr->iInUseMM = 1; + if( pCsr->xStress && (pCsr->iInUseDB==0 || pCache==pCsr) ){ + exitPcacheGlobal(); + pCsr->xStress(pCsr->pStress); + enterPcacheGlobal(); + } + pCsr->iInUseMM = 0; + } + + sqlite3_mutex_leave(pcache.mutex_mem2); + } + + p = pcache.pLruTail; + } + + if( p ){ + pcacheRemoveFromLruList(p); + pcacheRemoveFromHash(p); + pcacheRemoveFromList(&p->pCache->pClean, p); + + /* If the always-rollback flag is set on the page being recycled, set + ** the always-rollback flag on the corresponding pager. + */ + if( p->flags&PGHDR_ALWAYS_ROLLBACK ){ + assert(p->pPager); + sqlite3PagerAlwaysRollback(p->pPager); + } + + if( p->pCache->szPage!=szPage || p->pCache->szExtra!=szExtra ){ + pcachePageFree(p); + p = 0; + } + } + + if( !p ){ + /* Allocate a new page object. */ + p = pcachePageAlloc(szPage, szExtra, bPurg); + } + + exitPcacheGlobal(); + return p; +} + +/*************************************************** General Interfaces ****** +** +** Initialize and shutdown the page cache subsystem. Neither of these +** functions are threadsafe. +*/ +int sqlite3PcacheInitialize(void){ + assert( pcache.isInit==0 ); + memset(&pcache, 0, sizeof(pcache)); + if( sqlite3Config.bCoreMutex ){ + pcache.mutex_lru = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_LRU); + pcache.mutex_mem2 = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MEM2); + if( pcache.mutex_lru==0 || pcache.mutex_mem2==0 ){ + return SQLITE_NOMEM; + } + } + pcache.isInit = 1; + return SQLITE_OK; +} +void sqlite3PcacheShutdown(void){ + memset(&pcache, 0, sizeof(pcache)); +} + +/* +** Return the size in bytes of a PCache object. +*/ +int sqlite3PcacheSize(void){ return sizeof(PCache); } + +/* +** Create a new PCache object. Storage space to hold the object +** has already been allocated and is passed in as the p pointer. +*/ +void sqlite3PcacheOpen( + int szPage, /* Size of every page */ + int szExtra, /* Extra space associated with each page */ + int bPurgeable, /* True if pages are on backing store */ + void (*xDestroy)(PgHdr*), /* Called to destroy a page */ + int (*xStress)(void*), /* Call to try to make pages clean */ + void *pStress, /* Argument to xStress */ + PCache *p /* Preallocated space for the PCache */ +){ + assert( pcache.isInit ); + memset(p, 0, sizeof(PCache)); + p->szPage = szPage; + p->szExtra = szExtra; + p->bPurgeable = bPurgeable; + p->xDestroy = xDestroy; + p->xStress = xStress; + p->pStress = pStress; + p->nMax = 100; + + if( bPurgeable ){ + enterPcacheGlobal(); + pcache.mxPagePurgeable += p->nMax; + exitPcacheGlobal(); + } + + /* Add the new pager-cache to the list of caches starting at pcache.pAll */ + sqlite3_mutex_enter(pcache.mutex_mem2); + p->pNextAll = pcache.pAll; + if( pcache.pAll ){ + pcache.pAll->pPrevAll = p; + } + p->pPrevAll = 0; + pcache.pAll = p; + sqlite3_mutex_leave(pcache.mutex_mem2); +} + +void sqlite3PcacheSetPageSize(PCache *pCache, int szPage){ + assert(pCache->nPage==0); + pCache->szPage = szPage; +} + +/* +** Try to obtain a page from the cache. +*/ +int sqlite3PcacheFetch( + PCache *pCache, /* Obtain the page from this cache */ + Pgno pgno, /* Page number to obtain */ + int createFlag, /* If true, create page if it does not exist already */ + PgHdr **ppPage /* Write the page here */ +){ + PgHdr *pPage; + assert( pcache.isInit ); + assert( pCache!=0 ); + assert( pgno>0 ); + assert( pCache->iInUseDB || pCache->iInUseMM ); + + /* Search the hash table for the requested page. Exit early if it is found. */ + if( pCache->apHash ){ + u32 h = pgno % pCache->nHash; + for(pPage=pCache->apHash[h]; pPage; pPage=pPage->pNextHash){ + if( pPage->pgno==pgno ){ + if( pPage->nRef==0 && (pPage->flags & PGHDR_DIRTY)==0 ){ + enterPcacheGlobal(); + pcacheRemoveFromLruList(pPage); + exitPcacheGlobal(); + } + page_ref(pPage, 1); + *ppPage = pPage; + return SQLITE_OK; + } + } + } + + if( createFlag ){ + if( pCache->nHash<=pCache->nPage ){ + int rc = pcacheResizeHash(pCache, pCache->nHash<256?256:pCache->nHash*2); + if( rc!=SQLITE_OK ){ + return rc; + } + } + + pPage = pcacheRecycleOrAlloc(pCache); + *ppPage = pPage; + if( pPage==0 ){ + return SQLITE_NOMEM; + } + + pPage->pPager = 0; + pPage->flags = 0; + pPage->pDirty = 0; + pPage->nRef = 0; + pPage->pgno = pgno; + pPage->pCache = pCache; + page_ref(pPage, 1); + pcacheAddToList(&pCache->pClean, pPage); + pcacheAddToHash(pPage); + }else{ + *ppPage = 0; + } + + return SQLITE_OK; +} + +/* +** Dereference a page. When the reference count reaches zero, +** move the page to the LRU list if it is clean. +*/ +void sqlite3PcacheRelease(PgHdr *p){ + assert( p->nRef>0 ); + assert( p->pCache->iInUseDB || p->pCache->iInUseMM ); + page_ref(p, -1); + if( p->nRef!=0 ) return; + if( p->pCache->xDestroy ){ + p->pCache->xDestroy(p); + } + if( (p->flags & PGHDR_DIRTY)!=0 ) return; + enterPcacheGlobal(); + pcacheAddToLruList(p); + exitPcacheGlobal(); +} + +void sqlite3PcacheRef(PgHdr *p){ + assert(p->nRef>=0); + page_ref(p, 1); +} + +/* +** Drop a page from the cache. This should be the only reference to +** the page. +*/ +void sqlite3PcacheDrop(PgHdr *p){ + PCache *pCache; + assert( p->pCache->iInUseDB ); + assert( p->nRef==1 ); + pCache = p->pCache; + pCache->nRef--; + if( p->flags & PGHDR_DIRTY ){ + pcacheRemoveFromList(&pCache->pDirty, p); + }else{ + pcacheRemoveFromList(&pCache->pClean, p); + } + pcacheRemoveFromHash(p); + enterPcacheGlobal(); + pcachePageFree(p); + exitPcacheGlobal(); +} + +/* +** Make sure the page is marked as dirty. If it isn't dirty already, +** make it so. +*/ +void sqlite3PcacheMakeDirty(PgHdr *p){ + PCache *pCache; + assert( p->pCache->iInUseDB ); + if( p->flags & PGHDR_DIRTY ) return; + assert( (p->flags & PGHDR_DIRTY)==0 ); + assert( p->nRef>0 ); + pCache = p->pCache; + pcacheRemoveFromList(&pCache->pClean, p); + pcacheAddToList(&pCache->pDirty, p); + p->flags |= PGHDR_DIRTY; +} + +/* +** Make sure the page is marked as clean. If it isn't clean already, +** make it so. +*/ +void sqlite3PcacheMakeClean(PgHdr *p){ + PCache *pCache; + assert( p->pCache->iInUseDB || p->pCache->iInUseMM ); + if( (p->flags & PGHDR_DIRTY)==0 ) return; + assert( p->apSave[0]==0 && p->apSave[1]==0 ); + assert( p->flags & PGHDR_DIRTY ); + /* assert( p->nRef>0 ); */ + pCache = p->pCache; + pcacheRemoveFromList(&pCache->pDirty, p); + pcacheAddToList(&pCache->pClean, p); + p->flags &= ~PGHDR_DIRTY; + if( p->nRef==0 ){ + enterPcacheGlobal(); + pcacheAddToLruList(p); + exitPcacheGlobal(); + } +} + +/* +** Make every page in the cache clean. +*/ +void sqlite3PcacheCleanAll(PCache *pCache){ + PgHdr *p; + assert( pCache->iInUseDB ); + while( (p = pCache->pDirty)!=0 ){ + assert( p->apSave[0]==0 && p->apSave[1]==0 ); + pcacheRemoveFromList(&pCache->pDirty, p); + pcacheAddToList(&pCache->pClean, p); + p->flags &= ~PGHDR_DIRTY; + if( p->nRef==0 ){ + enterPcacheGlobal(); + pcacheAddToLruList(p); + exitPcacheGlobal(); + } + } +} + +/* +** Change the page number of page p to newPgno. If newPgno is 0, then the +** page object is added to the clean-list and the PGHDR_REUSE_UNLIKELY +** flag set. +*/ +void sqlite3PcacheMove(PgHdr *p, Pgno newPgno){ + assert( p->pCache->iInUseDB ); + pcacheRemoveFromHash(p); + p->pgno = newPgno; + if( newPgno==0 ){ + p->flags |= PGHDR_REUSE_UNLIKELY; + enterPcacheGlobal(); + pcacheFree(p->apSave[0]); + pcacheFree(p->apSave[1]); + exitPcacheGlobal(); + p->apSave[0] = 0; + p->apSave[1] = 0; + sqlite3PcacheMakeClean(p); + } + pcacheAddToHash(p); +} + +/* +** Set the global maximum number of pages. Return the previous value. +*/ +void sqlite3PcacheGlobalMax(int mx){ + enterPcacheGlobal(); + pcache.mxPage = mx; + exitPcacheGlobal(); +} + +/* +** Remove all content from a page cache +*/ +void pcacheClear(PCache *pCache){ + PgHdr *p, *pNext; + assert( sqlite3_mutex_held(pcache.mutex_lru) ); + for(p=pCache->pClean; p; p=pNext){ + pNext = p->pNext; + pcacheRemoveFromLruList(p); + pcachePageFree(p); + } + for(p=pCache->pDirty; p; p=pNext){ + pNext = p->pNext; + pcachePageFree(p); + } + pCache->pClean = 0; + pCache->pDirty = 0; + pCache->nPage = 0; + memset(pCache->apHash, 0, pCache->nHash*sizeof(pCache->apHash[0])); +} + + +/* +** Drop every cache entry whose page number is greater than "pgno". +*/ +void sqlite3PcacheTruncate(PCache *pCache, Pgno pgno){ + PgHdr *p, *pNext; + PgHdr *pDirty = pCache->pDirty; + assert( pCache->iInUseDB ); + enterPcacheGlobal(); + for(p=pCache->pClean; p||pDirty; p=pNext){ + if( !p ){ + p = pDirty; + pDirty = 0; + } + pNext = p->pNext; + if( p->pgno>pgno ){ + if( p->nRef==0 ){ + pcacheRemoveFromHash(p); + if( p->flags&PGHDR_DIRTY ){ + pcacheRemoveFromList(&pCache->pDirty, p); + }else{ + pcacheRemoveFromLruList(p); + pcacheRemoveFromList(&pCache->pClean, p); + } + pcachePageFree(p); + }else{ + /* If there are references to the page, it cannot be freed. In this + ** case, zero the page content instead. + */ + memset(p->pData, 0, pCache->szPage); + } + } + } + exitPcacheGlobal(); +} + + +/* +** Close a cache. +*/ +void sqlite3PcacheClose(PCache *pCache){ + assert( pCache->iInUseDB==1 ); + + /* Free all the pages used by this pager and remove them from the LRU + ** list. This requires the protection of the MUTEX_STATIC_LRU mutex. + */ + enterPcacheGlobal(); + pcacheClear(pCache); + if( pCache->bPurgeable ){ + pcache.mxPagePurgeable -= pCache->nMax; + } + sqlite3_free(pCache->apHash); + exitPcacheGlobal(); + + /* Now remove the pager-cache structure itself from the list of + ** all such structures headed by pcache.pAll. This required the + ** MUTEX_STATIC_MEM2 mutex. + */ + sqlite3_mutex_enter(pcache.mutex_mem2); + assert(pCache==pcache.pAll || pCache->pPrevAll); + assert(pCache->pNextAll==0 || pCache->pNextAll->pPrevAll==pCache); + assert(pCache->pPrevAll==0 || pCache->pPrevAll->pNextAll==pCache); + if( pCache->pPrevAll ){ + pCache->pPrevAll->pNextAll = pCache->pNextAll; + }else{ + pcache.pAll = pCache->pNextAll; + } + if( pCache->pNextAll ){ + pCache->pNextAll->pPrevAll = pCache->pPrevAll; + } + sqlite3_mutex_leave(pcache.mutex_mem2); +} + +/* +** Preserve the content of the page, if it has not been preserved +** already. If idJournal==0 then this is for the overall transaction. +** If idJournal==1 then this is for the statement journal. +** +** This routine is used for in-memory databases only. +** +** Return SQLITE_OK or SQLITE_NOMEM if a memory allocation fails. +*/ +int sqlite3PcachePreserve(PgHdr *p, int idJournal){ + void *x; + int sz; + assert( p->pCache->iInUseDB ); + assert( p->pCache->bPurgeable==0 ); + if( !p->apSave[idJournal] ){ + sz = p->pCache->szPage; + p->apSave[idJournal] = x = sqlite3PageMalloc( sz ); + if( x==0 ) return SQLITE_NOMEM; + memcpy(x, p->pData, sz); + } + return SQLITE_OK; +} + +/* +** Commit a change previously preserved. +*/ +void sqlite3PcacheCommit(PCache *pCache, int idJournal){ + PgHdr *p; + assert( pCache->iInUseDB ); + enterPcacheGlobal(); /* Mutex is required to call pcacheFree() */ + for(p=pCache->pDirty; p; p=p->pNext){ + if( p->apSave[idJournal] ){ + pcacheFree(p->apSave[idJournal]); + p->apSave[idJournal] = 0; + } + } + exitPcacheGlobal(); +} + +/* +** Rollback a change previously preserved. +*/ +void sqlite3PcacheRollback(PCache *pCache, int idJournal){ + PgHdr *p; + int sz; + assert( pCache->iInUseDB ); + enterPcacheGlobal(); /* Mutex is required to call pcacheFree() */ + sz = pCache->szPage; + for(p=pCache->pDirty; p; p=p->pNext){ + if( p->apSave[idJournal] ){ + memcpy(p->pData, p->apSave[idJournal], sz); + pcacheFree(p->apSave[idJournal]); + p->apSave[idJournal] = 0; + } + } + exitPcacheGlobal(); +} + +/* +** Assert flags settings on all pages. Debugging only. +*/ +void sqlite3PcacheAssertFlags(PCache *pCache, int trueMask, int falseMask){ + PgHdr *p; + assert( pCache->iInUseDB || pCache->iInUseMM ); + for(p=pCache->pDirty; p; p=p->pNext){ + assert( (p->flags&trueMask)==trueMask ); + assert( (p->flags&falseMask)==0 ); + } + for(p=pCache->pClean; p; p=p->pNext){ + assert( (p->flags&trueMask)==trueMask ); + assert( (p->flags&falseMask)==0 ); + } +} + +/* +** Discard the contents of the cache. +*/ +int sqlite3PcacheClear(PCache *pCache){ + assert( pCache->iInUseDB ); + assert(pCache->nRef==0); + enterPcacheGlobal(); + pcacheClear(pCache); + exitPcacheGlobal(); + return SQLITE_OK; +} + +/* +** Merge two lists of pages connected by pDirty and in pgno order. +** Do not both fixing the pPrevDirty pointers. +*/ +static PgHdr *pcacheMergeDirtyList(PgHdr *pA, PgHdr *pB){ + PgHdr result, *pTail; + pTail = &result; + while( pA && pB ){ + if( pA->pgnopgno ){ + pTail->pDirty = pA; + pTail = pA; + pA = pA->pDirty; + }else{ + pTail->pDirty = pB; + pTail = pB; + pB = pB->pDirty; + } + } + if( pA ){ + pTail->pDirty = pA; + }else if( pB ){ + pTail->pDirty = pB; + }else{ + pTail->pDirty = 0; + } + return result.pDirty; +} + +/* +** Sort the list of pages in accending order by pgno. Pages are +** connected by pDirty pointers. The pPrevDirty pointers are +** corrupted by this sort. +*/ +#define N_SORT_BUCKET_ALLOC 25 +#define N_SORT_BUCKET 25 +#ifdef SQLITE_TEST + int sqlite3_pager_n_sort_bucket = 0; + #undef N_SORT_BUCKET + #define N_SORT_BUCKET \ + (sqlite3_pager_n_sort_bucket?sqlite3_pager_n_sort_bucket:N_SORT_BUCKET_ALLOC) +#endif +static PgHdr *pcacheSortDirtyList(PgHdr *pIn){ + PgHdr *a[N_SORT_BUCKET_ALLOC], *p; + int i; + memset(a, 0, sizeof(a)); + while( pIn ){ + p = pIn; + pIn = p->pDirty; + p->pDirty = 0; + for(i=0; iiInUseDB ); + for(p=pCache->pDirty; p; p=p->pNext){ + p->pDirty = p->pNext; + } + return pcacheSortDirtyList(pCache->pDirty); +} + +/* +** This function searches cache pCache for a dirty page for which the +** reference count is zero. If such a page can be found, the PgHdr.pDirty +** pointer is set to 0 and a pointer to the page is returned. If no +** such page is found, 0 is returned. +** +** This is used by the pager module to implement the xStress callback. +*/ +PgHdr *sqlite3PcacheDirtyPage(PCache *pCache){ + PgHdr *p = 0; + assert( pCache->iInUseMM ); +#if 1 + PgHdr *pIter; + Pgno min_pgno; + for(pIter=pCache->pDirty; pIter; pIter=pIter->pNext){ + if( pIter->nRef==0 && (p==0 || pIter->pgnopgno; + } + } +#else + for(p=pCache->pDirty; p && p->nRef; p=p->pNext); +#endif + if( p ){ + p->pDirty = 0; + } + return p; +} + +/* +** Return the total number of outstanding page references. +*/ +int sqlite3PcacheRefCount(PCache *pCache){ + return pCache->nRef; +} + +/* +** Return the total number of pages in the cache. +*/ +int sqlite3PcachePagecount(PCache *pCache){ + assert( pCache->iInUseDB || pCache->iInUseMM ); + assert( pCache->nPage>=0 ); + return pCache->nPage; +} + +#ifdef SQLITE_CHECK_PAGES +/* +** This function is used by the pager.c module to iterate through all +** pages in the cache. At present, this is only required if the +** SQLITE_CHECK_PAGES macro (used for debugging) is specified. +*/ +void sqlite3PcacheIterate(PCache *pCache, void (*xIter)(PgHdr *)){ + PgHdr *p; + assert( pCache->iInUseDB || pCache->iInUseMM ); + for(p=pCache->pClean; p; p=p->pNext){ + xIter(p); + } + for(p=pCache->pDirty; p; p=p->pNext){ + xIter(p); + } +} +#endif + +/* +** Set flags on all pages in the page cache +*/ +void sqlite3PcacheSetFlags(PCache *pCache, int andMask, int orMask){ + PgHdr *p; + assert( pCache->iInUseDB || pCache->iInUseMM ); + for(p=pCache->pDirty; p; p=p->pNext){ + p->flags = (p->flags&andMask)|orMask; + } + for(p=pCache->pClean; p; p=p->pNext){ + p->flags = (p->flags&andMask)|orMask; + } +} + +/* +** Set the suggested cache-size value. +*/ +int sqlite3PcacheGetCachesize(PCache *pCache){ + return pCache->nMax; +} + +/* +** Set the suggested cache-size value. +*/ +void sqlite3PcacheSetCachesize(PCache *pCache, int mxPage){ + if( mxPage<10 ){ + mxPage = 10; + } + if( pCache->bPurgeable ){ + enterPcacheGlobal(); + pcache.mxPagePurgeable -= pCache->nMax; + pcache.mxPagePurgeable += mxPage; + exitPcacheGlobal(); + } + pCache->nMax = mxPage; +} + +/* +** Lock a pager-cache. +*/ +void sqlite3PcacheLock(PCache *pCache){ + pCache->iInUseDB++; + if( pCache->iInUseMM && pCache->iInUseDB==1 ){ + pCache->iInUseDB = 0; + sqlite3_mutex_enter(pcache.mutex_mem2); + assert( pCache->iInUseMM==0 && pCache->iInUseDB==0 ); + pCache->iInUseDB = 1; + sqlite3_mutex_leave(pcache.mutex_mem2); + } +} + +/* +** Unlock a pager-cache. +*/ +void sqlite3PcacheUnlock(PCache *pCache){ + pCache->iInUseDB--; + assert( pCache->iInUseDB>=0 ); +} + diff --git a/src/pcache.h b/src/pcache.h new file mode 100644 index 0000000000..15772f8bde --- /dev/null +++ b/src/pcache.h @@ -0,0 +1,175 @@ +/* +** 2008 August 05 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This header file defines the interface that the sqlite page cache +** subsystem. +** +** @(#) $Id: pcache.h,v 1.1 2008/08/20 14:49:25 danielk1977 Exp $ +*/ + +#ifndef _PCACHE_H_ + +typedef struct PgHdr PgHdr; +typedef struct PCache PCache; + +/* +** Every page in the cache is controlled by an instance of the following +** structure. +*/ +struct PgHdr { + u32 flags; /* PGHDR flags defined below */ + void *pData; /* Content of this page */ + void *pExtra; /* Extra content */ + PgHdr *pDirty; /* Transient list of dirty pages */ + Pgno pgno; /* Page number for this page */ + Pager *pPager; +#ifdef SQLITE_CHECK_PAGES + u32 pageHash; +#endif + /*** Public data is above. All that follows is private to pcache.c ***/ + PCache *pCache; /* Cache that owns this page */ + PgHdr *pNextHash, *pPrevHash; /* Hash collision chain for PgHdr.pgno */ + PgHdr *pNext, *pPrev; /* List of clean or dirty pages */ + PgHdr *pNextLru, *pPrevLru; /* Part of global LRU list */ + int nRef; /* Number of users of this page */ + void *apSave[2]; /* Journal entries for in-memory databases */ +}; + +/* Bit values for PgHdr.flags */ +#define PGHDR_IN_JOURNAL 0x001 /* Page is in rollback journal */ +#define PGHDR_IN_STMTJRNL 0x002 /* Page is in the statement journal */ +#define PGHDR_DIRTY 0x004 /* Page has changed */ +#define PGHDR_NEED_SYNC 0x008 /* Peed to fsync this page */ +#define PGHDR_ALWAYS_ROLLBACK 0x010 /* Force writing to journal */ +#define PGHDR_NEED_READ 0x020 /* Content is unread */ +#define PGHDR_IS_INIT 0x040 /* pData is initialized */ +#define PGHDR_REUSE_UNLIKELY 0x080 /* Hint: Reuse is unlikely */ + + +/* Initialize and shutdown the page cache subsystem */ +int sqlite3PcacheInitialize(void); +void sqlite3PcacheShutdown(void); + +/* Page cache buffer management: +** These routines implement SQLITE_CONFIG_PAGECACHE. +*/ +void sqlite3PCacheBufferSetup(void *, int sz, int n); +void *sqlite3PCacheMalloc(int sz); +void sqlite3PCacheFree(void*); + +/* Create a new pager cache. +** Under memory stress, invoke xStress to try to make pages clean. +** Only clean and unpinned pages can be reclaimed. +*/ +void sqlite3PcacheOpen( + int szPage, /* Size of every page */ + int szExtra, /* Extra space associated with each page */ + int bPurgeable, /* True if pages are on backing store */ + void (*xDestroy)(PgHdr *), /* Called to destroy a page */ + int (*xStress)(void*), /* Call to try to make pages clean */ + void *pStress, /* Argument to xStress */ + PCache *pToInit /* Preallocated space for the PCache */ +); + +/* Modify the page-size after the cache has been created. */ +void sqlite3PcacheSetPageSize(PCache *, int); + +/* Return the size in bytes of a PCache object. Used to preallocate +** storage space. +*/ +int sqlite3PcacheSize(void); + +/* One release per successful fetch. Page is pinned until released. +** Reference counted. +*/ +int sqlite3PcacheFetch(PCache*, Pgno, int createFlag, PgHdr**); +void sqlite3PcacheRelease(PgHdr*); + +void sqlite3PcacheDrop(PgHdr*); /* Remove page from cache */ +void sqlite3PcacheMakeDirty(PgHdr*); /* Make sure page is marked dirty */ +void sqlite3PcacheMakeClean(PgHdr*); /* Mark a single page as clean */ +void sqlite3PcacheCleanAll(PCache*); /* Mark all dirty list pages as clean */ + +/* Change a page number. Used by incr-vacuum. */ +void sqlite3PcacheMove(PgHdr*, Pgno); + +/* Set a global maximum page count for all page caches. +** If the sum of individual cache maxes exceed the global max, the +** individuals are scaled down proportionally. +*/ +void sqlite3PcacheGlobalMax(int N); + +/* Remove all pages with pgno>x. Reset the cache if x==0 */ +void sqlite3PcacheTruncate(PCache*, Pgno x); + +/* Routines used to implement transactions on memory-only databases. */ +int sqlite3PcachePreserve(PgHdr*, int); /* Preserve current page content */ +void sqlite3PcacheCommit(PCache*, int); /* Drop preserved copy */ +void sqlite3PcacheRollback(PCache*, int); /* Rollback to preserved copy */ + +/* Get a list of all dirty pages in the cache, sorted by page number */ +PgHdr *sqlite3PcacheDirtyList(PCache*); + +/* Query the cache for a dirty page with a zero ref-count */ +PgHdr *sqlite3PcacheDirtyPage(PCache *); + +/* Reset and close the cache object */ +void sqlite3PcacheClose(PCache*); + +/* Set flags on all pages in the page cache */ +void sqlite3PcacheSetFlags(PCache*, int andMask, int orMask); + +/* Assert flags settings on all pages. Debugging only */ +void sqlite3PcacheAssertFlags(PCache*, int trueMask, int falseMask); + +/* Return true if the number of dirty pages is 0 or 1 */ +int sqlite3PcacheZeroOrOneDirtyPages(PCache*); + +/* Discard the contents of the cache */ +int sqlite3PcacheClear(PCache*); + +/* Return the total number of outstanding page references */ +int sqlite3PcacheRefCount(PCache*); + +/* Increment the reference count of an existing page */ +void sqlite3PcacheRef(PgHdr*); + +/* Return the total number of pages stored in the cache */ +int sqlite3PcachePagecount(PCache*); + +/* Iterate through all pages currently stored in the cache. This interface +** is only available if SQLITE_CHECK_PAGES is defined when the library is +** built. +*/ +void sqlite3PcacheIterate(PCache *pCache, void (*xIter)(PgHdr *)); + +/* Set and get the suggested cache-size for the specified pager-cache. If +** a global maximum on the number of pages cached by the system is +** configured via the sqlite3PcacheGlobalMax() API, then the suggested +** cache-sizes are not used at all. +** +** If no global maximum is configured, then the system attempts to limit +** the total number of pages cached by purgeable pager-caches to the sum +** of the suggested cache-sizes. +*/ +int sqlite3PcacheGetCachesize(PCache *); +void sqlite3PcacheSetCachesize(PCache *, int); + +/* Lock and unlock a pager-cache object. The PcacheLock() function may +** block if the lock is temporarily available. While a pager-cache is locked, +** the system guarantees that any configured xStress() callback will not +** be invoked by any thread other than the one holding the lock. +*/ +void sqlite3PcacheLock(PCache *); +void sqlite3PcacheUnlock(PCache *); + +#endif /* _PCACHE_H_ */ + diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 1851b41522..455aec62b8 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -11,7 +11,7 @@ ************************************************************************* ** Internal interface definitions for SQLite. ** -** @(#) $Id: sqliteInt.h,v 1.755 2008/08/13 19:11:48 drh Exp $ +** @(#) $Id: sqliteInt.h,v 1.756 2008/08/20 14:49:25 danielk1977 Exp $ */ #ifndef _SQLITEINT_H_ #define _SQLITEINT_H_ @@ -482,6 +482,7 @@ typedef struct WhereLevel WhereLevel; #include "btree.h" #include "vdbe.h" #include "pager.h" +#include "pcache.h" #include "os.h" #include "mutex.h" @@ -757,7 +758,7 @@ struct FuncDef { void (*xFunc)(sqlite3_context*,int,sqlite3_value**); /* Regular function */ void (*xStep)(sqlite3_context*,int,sqlite3_value**); /* Aggregate step */ void (*xFinalize)(sqlite3_context*); /* Aggregate finializer */ - char zName[1]; /* SQL name of the function. MUST BE LAST */ + char *zName; /* SQL name of the function. */ }; /* @@ -1932,6 +1933,7 @@ int sqlite3IsNaN(double); void sqlite3VXPrintf(StrAccum*, int, const char*, va_list); char *sqlite3MPrintf(sqlite3*,const char*, ...); +char *sqlite3MAppendf(sqlite3 *, char *, const char *, ...); char *sqlite3VMPrintf(sqlite3*,const char*, va_list); char *sqlite3MAppendf(sqlite3*,char*,const char*,...); #if defined(SQLITE_TEST) || defined(SQLITE_DEBUG) @@ -2081,6 +2083,7 @@ Select *sqlite3SelectDup(sqlite3*,Select*); FuncDef *sqlite3FindFunction(sqlite3*,const char*,int,int,u8,int); void sqlite3RegisterBuiltinFunctions(sqlite3*); void sqlite3RegisterDateTimeFunctions(sqlite3*); +int sqlite3GetBuiltinFunction(const char *, int, FuncDef **); #ifdef SQLITE_DEBUG int sqlite3SafetyOn(sqlite3*); int sqlite3SafetyOff(sqlite3*); diff --git a/src/test1.c b/src/test1.c index 9ce4df47d4..cff82a6623 100644 --- a/src/test1.c +++ b/src/test1.c @@ -13,7 +13,7 @@ ** is not included in the SQLite library. It is used for automated ** testing of the SQLite library. ** -** $Id: test1.c,v 1.318 2008/08/12 15:04:59 danielk1977 Exp $ +** $Id: test1.c,v 1.319 2008/08/20 14:49:25 danielk1977 Exp $ */ #include "sqliteInt.h" #include "tcl.h" @@ -4686,7 +4686,6 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ extern int sqlite3_pager_readdb_count; extern int sqlite3_pager_writedb_count; extern int sqlite3_pager_writej_count; - extern int sqlite3_pager_pgfree_count; #if SQLITE_OS_UNIX && defined(SQLITE_TEST) && SQLITE_THREADSAFE extern int threadsOverrideEachOthersLocks; #endif @@ -4733,8 +4732,6 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ (char*)&sqlite3_pager_writedb_count, TCL_LINK_INT); Tcl_LinkVar(interp, "sqlite3_pager_writej_count", (char*)&sqlite3_pager_writej_count, TCL_LINK_INT); - Tcl_LinkVar(interp, "sqlite3_pager_pgfree_count", - (char*)&sqlite3_pager_pgfree_count, TCL_LINK_INT); #ifndef SQLITE_OMIT_UTF16 Tcl_LinkVar(interp, "unaligned_string_counter", (char*)&unaligned_string_counter, TCL_LINK_INT); diff --git a/src/test2.c b/src/test2.c index c27a9fdf8e..18b128b7b1 100644 --- a/src/test2.c +++ b/src/test2.c @@ -13,7 +13,7 @@ ** is not included in the SQLite library. It is used for automated ** testing of the SQLite library. ** -** $Id: test2.c,v 1.59 2008/07/12 14:52:20 drh Exp $ +** $Id: test2.c,v 1.60 2008/08/20 14:49:25 danielk1977 Exp $ */ #include "sqliteInt.h" #include "tcl.h" @@ -78,7 +78,7 @@ static int pager_open( return TCL_ERROR; } if( Tcl_GetInt(interp, argv[2], &nPage) ) return TCL_ERROR; - rc = sqlite3PagerOpen(sqlite3_vfs_find(0), &pPager, argv[1], 0, 0, + rc = sqlite3PagerOpen(sqlite3_vfs_find(0), &pPager, argv[1], 0, 0, 0, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_MAIN_DB); if( rc!=SQLITE_OK ){ Tcl_AppendResult(interp, errorName(rc), 0); @@ -382,6 +382,26 @@ static int page_lookup( return TCL_OK; } +/* +** Usage: pcache_global_max NPAGE +*/ +static int pcache_global_max( + void *NotUsed, + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int argc, /* Number of arguments */ + const char **argv /* Text of each argument */ +){ + int nPage; + if( argc!=2 ){ + Tcl_AppendResult(interp, "wrong # args: should be \"", argv[0], + " NPAGE\"", 0); + return TCL_ERROR; + } + if( Tcl_GetInt(interp, argv[1], &nPage) ) return TCL_ERROR; + sqlite3PcacheGlobalMax(nPage); + return TCL_OK; +} + /* ** Usage: pager_truncate ID PGNO */ @@ -630,6 +650,7 @@ int Sqlitetest2_Init(Tcl_Interp *interp){ { "page_write", (Tcl_CmdProc*)page_write }, { "page_number", (Tcl_CmdProc*)page_number }, { "pager_truncate", (Tcl_CmdProc*)pager_truncate }, + { "pcache_global_max", (Tcl_CmdProc*)pcache_global_max }, #ifndef SQLITE_OMIT_DISKIO { "fake_big_file", (Tcl_CmdProc*)fake_big_file }, #endif diff --git a/src/test8.c b/src/test8.c index feb80bf305..d7d448b8d9 100644 --- a/src/test8.c +++ b/src/test8.c @@ -13,7 +13,7 @@ ** is not included in the SQLite library. It is used for automated ** testing of the SQLite library. ** -** $Id: test8.c,v 1.73 2008/08/12 14:48:41 danielk1977 Exp $ +** $Id: test8.c,v 1.74 2008/08/20 14:49:25 danielk1977 Exp $ */ #include "sqliteInt.h" #include "tcl.h" @@ -96,8 +96,7 @@ static int simulateVtabError(echo_vtab *p, const char *zMethod){ const char *zErr; char zVarname[128]; zVarname[127] = '\0'; - sqlite3_snprintf(127, zVarname, - "echo_module_fail(%s,%s)", zMethod, p->zTableName); + snprintf(zVarname, 127, "echo_module_fail(%s,%s)", zMethod, p->zTableName); zErr = Tcl_GetVar(p->interp, zVarname, TCL_GLOBAL_ONLY); if( zErr ){ p->base.zErrMsg = sqlite3_mprintf("echo-vtab-error: %s", zErr); diff --git a/src/vtab.c b/src/vtab.c index bd18c479a5..6013f61de1 100644 --- a/src/vtab.c +++ b/src/vtab.c @@ -11,7 +11,7 @@ ************************************************************************* ** This file contains code used to help implement virtual tables. ** -** $Id: vtab.c,v 1.74 2008/08/02 03:50:39 drh Exp $ +** $Id: vtab.c,v 1.75 2008/08/20 14:49:25 danielk1977 Exp $ */ #ifndef SQLITE_OMIT_VIRTUALTABLE #include "sqliteInt.h" @@ -807,6 +807,7 @@ FuncDef *sqlite3VtabOverloadFunction( return pDef; } *pNew = *pDef; + pNew->zName = (char *)&pNew[1]; memcpy(pNew->zName, pDef->zName, strlen(pDef->zName)+1); pNew->xFunc = xFunc; pNew->pUserData = pArg; diff --git a/test/io.test b/test/io.test index 12c756acac..0dccd503b5 100644 --- a/test/io.test +++ b/test/io.test @@ -13,7 +13,7 @@ # IO traffic generated by SQLite (making sure SQLite is not writing out # more database pages than it has to, stuff like that). # -# $Id: io.test,v 1.17 2008/07/30 17:28:04 drh Exp $ +# $Id: io.test,v 1.18 2008/08/20 14:49:25 danielk1977 Exp $ set testdir [file dirname $argv0] source $testdir/tester.tcl @@ -383,6 +383,7 @@ ifcapable pager_pragmas { execsql { CREATE TABLE abc(a, b) } nSync execsql { + PRAGMA temp_store = memory; PRAGMA cache_size = 10; BEGIN; INSERT INTO abc VALUES('hello', 'world'); diff --git a/test/ioerr2.test b/test/ioerr2.test index 05b78cd7a7..e5f101b035 100644 --- a/test/ioerr2.test +++ b/test/ioerr2.test @@ -15,7 +15,7 @@ # The tests in this file use special facilities that are only # available in the SQLite test fixture. # -# $Id: ioerr2.test,v 1.8 2008/07/08 15:59:52 danielk1977 Exp $ +# $Id: ioerr2.test,v 1.9 2008/08/20 14:49:25 danielk1977 Exp $ set testdir [file dirname $argv0] source $testdir/tester.tcl @@ -115,11 +115,12 @@ foreach bPersist [list 0 1] { do_test ioerr2-5 { execsql { CREATE TABLE t2 AS SELECT * FROM t1; + PRAGMA temp_store = memory; } set ::sqlite_io_error_persist 0 set ::go 1 set rc [catch { - for {set ::N 1} {$::N<200} {incr ::N} { + for {set ::N 2} {$::N<200} {incr ::N} { db eval {SELECT * FROM t1 WHERE rowid IN (1, 5, 10, 15, 20)} { set ::sqlite_io_error_hit 0 set ::sqlite_io_error_pending $::N diff --git a/test/memsubsys1.test b/test/memsubsys1.test index fafb701415..5cd93f3f58 100644 --- a/test/memsubsys1.test +++ b/test/memsubsys1.test @@ -11,7 +11,7 @@ # # This file contains tests of the memory allocation subsystem # -# $Id: memsubsys1.test,v 1.9 2008/08/12 15:21:12 drh Exp $ +# $Id: memsubsys1.test,v 1.10 2008/08/20 14:49:25 danielk1977 Exp $ set testdir [file dirname $argv0] source $testdir/tester.tcl @@ -60,6 +60,8 @@ proc reset_highwater_marks {} { sqlite3_status SQLITE_STATUS_PARSER_STACK 1 } +set xtra_size 256 + # Test 1: Both PAGECACHE and SCRATCH are shut down. # db close @@ -81,7 +83,7 @@ set max_pagecache [lindex [sqlite3_status SQLITE_STATUS_PAGECACHE_OVERFLOW 0] 2] # db close sqlite3_shutdown -sqlite3_config_pagecache 1024 20 +sqlite3_config_pagecache [expr 1024+$xtra_size] 20 sqlite3_initialize reset_highwater_marks build_test_db memsubsys1-2 {PRAGMA page_size=1024} @@ -89,8 +91,11 @@ build_test_db memsubsys1-2 {PRAGMA page_size=1024} do_test memsubsys1-2.3 { set pg_ovfl [lindex [sqlite3_status SQLITE_STATUS_PAGECACHE_OVERFLOW 0] 2] set pg_used [lindex [sqlite3_status SQLITE_STATUS_PAGECACHE_USED 0] 2] - expr {$pg_used*1024 + $pg_ovfl} -} $max_pagecache + expr { + ($pg_used*1024 + $pg_ovfl) < $max_pagecache && + ($pg_used*(1024+$xtra_size) + $pg_ovfl) >= $max_pagecache + } +} 1 do_test memsubsys1-2.4 { set pg_used [lindex [sqlite3_status SQLITE_STATUS_PAGECACHE_USED 0] 2] } 19 @@ -103,7 +108,7 @@ do_test memsubsys1-2.5 { # db close sqlite3_shutdown -sqlite3_config_pagecache 512 20 +sqlite3_config_pagecache [expr 512+$xtra_size] 20 sqlite3_initialize reset_highwater_marks build_test_db memsubsys1-3.1 {PRAGMA page_size=1024} @@ -119,7 +124,7 @@ do_test memsubsys1-3.1.5 { } 0 db close sqlite3_shutdown -sqlite3_config_pagecache 2048 20 +sqlite3_config_pagecache [expr 2048+$xtra_size] 20 sqlite3_initialize reset_highwater_marks build_test_db memsubsys1-3.2 {PRAGMA page_size=2048} @@ -138,7 +143,7 @@ do_test memsubsys1-3.2.5 { # db close sqlite3_shutdown -sqlite3_config_pagecache 1024 50 +sqlite3_config_pagecache [expr 1024+$xtra_size] 50 sqlite3_config_scratch 6000 2 sqlite3_initialize reset_highwater_marks @@ -150,8 +155,11 @@ do_test memsubsys1-4.3 { do_test memsubsys1-4.4 { set pg_ovfl [lindex [sqlite3_status SQLITE_STATUS_PAGECACHE_OVERFLOW 0] 2] set pg_used [lindex [sqlite3_status SQLITE_STATUS_PAGECACHE_USED 0] 2] - expr {$pg_used*1024 + $pg_ovfl} -} $max_pagecache + expr { + ($pg_used*1024 + $pg_ovfl) < $max_pagecache && + ($pg_used*(1024+$xtra_size) + $pg_ovfl) >= $max_pagecache + } +} 1 do_test memsubsys1-4.5 { set maxreq [lindex [sqlite3_status SQLITE_STATUS_MALLOC_SIZE 0] 2] expr {$maxreq<7000} @@ -165,7 +173,7 @@ do_test memsubsys1-4.6 { # db close sqlite3_shutdown -sqlite3_config_pagecache 4096 24 +sqlite3_config_pagecache [expr 4096+$xtra_size] 24 sqlite3_config_scratch 6000 2 sqlite3_initialize reset_highwater_marks @@ -191,7 +199,7 @@ do_test memsubsys1-5.6 { # db close sqlite3_shutdown -sqlite3_config_pagecache 4096 24 +sqlite3_config_pagecache [expr 4096+$xtra_size] 24 sqlite3_config_scratch 25000 1 sqlite3_initialize reset_highwater_marks @@ -202,7 +210,8 @@ do_test memsubsys1-6.3 { } 23 do_test memsubsys1-6.4 { set maxreq [lindex [sqlite3_status SQLITE_STATUS_MALLOC_SIZE 0] 2] -} 4096 + expr {$maxreq>4096 && $maxreq<(4096+$xtra_size)} +} 1 do_test memsubsys1-6.5 { set s_used [lindex [sqlite3_status SQLITE_STATUS_SCRATCH_USED 0] 2] } 1 @@ -216,14 +225,13 @@ do_test memsubsys1-6.6 { # db close sqlite3_shutdown -sqlite3_config_pagecache 4096 24 +sqlite3_config_pagecache [expr 4096+$xtra_size] 24 sqlite3_config_scratch 25000 1 sqlite3_initialize +pcache_global_max 15 reset_highwater_marks build_test_db memsubsys1-7 { PRAGMA page_size=4096; - PRAGMA cache_size=10; - PRAGMA temp_cache_size=10; } #show_memstats do_test memsubsys1-7.3 { @@ -243,6 +251,7 @@ do_test memsubsys1-7.6 { do_test memsubsys1-7.7 { set s_ovfl [lindex [sqlite3_status SQLITE_STATUS_SCRATCH_OVERFLOW 0] 2] } 0 +pcache_global_max 0 db close sqlite3_shutdown diff --git a/test/mutex1.test b/test/mutex1.test index f52d4fe003..af5db2e1e9 100644 --- a/test/mutex1.test +++ b/test/mutex1.test @@ -9,7 +9,7 @@ # #*********************************************************************** # -# $Id: mutex1.test,v 1.10 2008/07/15 00:27:35 drh Exp $ +# $Id: mutex1.test,v 1.11 2008/08/20 14:49:25 danielk1977 Exp $ set testdir [file dirname $argv0] source $testdir/tester.tcl @@ -96,8 +96,8 @@ set enable_shared_cache [sqlite3_enable_shared_cache 1] ifcapable threadsafe { foreach {mode mutexes} { singlethread {} - multithread {fast static_master static_mem static_prng} - serialized {fast recursive static_master static_mem static_prng} + multithread {fast static_lru static_master static_mem static_mem2 static_prng } + serialized {fast recursive static_lru static_master static_mem static_mem2 static_prng } } { ifcapable memorymanage { if {$mode ne "singlethread"} { diff --git a/test/pager.test b/test/pager.test index bb92617594..3b199c4b43 100644 --- a/test/pager.test +++ b/test/pager.test @@ -11,7 +11,7 @@ # This file implements regression tests for SQLite library. The # focus of this script is page cache subsystem. # -# $Id: pager.test,v 1.30 2007/08/24 16:29:24 drh Exp $ +# $Id: pager.test,v 1.31 2008/08/20 14:49:25 danielk1977 Exp $ set testdir [file dirname $argv0] @@ -76,13 +76,13 @@ do_test pager-2.3.4 { expr {$::gx!=""} } {1} do_test pager-2.3.5 { + page_unref $::gx pager_stats $::p1 } {ref 1 page 1 max 10 size 0 state 1 err 0 hit 0 miss 1 ovfl 0} do_test pager-2.3.6 { expr {$::g1==$::gx} } {1} do_test pager-2.3.7 { - page_unref $::gx pager_stats $::p1 } {ref 1 page 1 max 10 size 0 state 1 err 0 hit 0 miss 1 ovfl 0} do_test pager-2.4 { @@ -184,6 +184,7 @@ do_test pager-2.29 { page_read $::g1 } {Page-One} do_test pager-2.99 { + page_unref $::g1 pager_close $::p1 } {} @@ -214,6 +215,7 @@ do_test pager-3.5 { page_unref $gx } pager_commit $::p1 + page_unref $::g(1) } {} for {set i 2} {$i<=20} {incr i} { do_test pager-3.6.[expr {$i-1}] [subst { @@ -427,6 +429,7 @@ ifcapable memorydb { } do_test pager-4.99 { + page_unref $::g1 pager_close $::p1 } {} diff --git a/test/pager2.test b/test/pager2.test index 52dfe73e55..5b65785a61 100644 --- a/test/pager2.test +++ b/test/pager2.test @@ -11,7 +11,7 @@ # This file implements regression tests for SQLite library. The # focus of this script is page cache subsystem. # -# $Id: pager2.test,v 1.6 2007/03/23 18:12:07 danielk1977 Exp $ +# $Id: pager2.test,v 1.7 2008/08/20 14:49:25 danielk1977 Exp $ set testdir [file dirname $argv0] @@ -75,6 +75,7 @@ do_test pager2-2.3.3 { } {ref 1 page 1 max 10 size 0 state 1 err 0 hit 0 miss 1 ovfl 0} do_test pager2-2.3.4 { set ::gx [page_lookup $::p1 1] + page_unref $::gx expr {$::gx!=""} } {1} do_test pager2-2.3.5 { @@ -84,7 +85,6 @@ do_test pager2-2.3.6 { expr {$::g1==$::gx} } {1} do_test pager2-2.3.7 { - page_unref $::gx pager_stats $::p1 } {ref 1 page 1 max 10 size 0 state 1 err 0 hit 0 miss 1 ovfl 0} do_test pager2-2.4 { @@ -181,9 +181,9 @@ do_test pager2-2.29 { set ::g1 [page_get $::p1 1] page_read $::g1 } {Page-One} -#do_test pager2-2.99 { -# pager_close $::p1 -#} {} +do_test pager2-2.99 { + page_unref $::g1 +} {} #do_test pager2-3.1 { # set v [catch { @@ -212,6 +212,7 @@ do_test pager2-3.5 { page_unref $gx } pager_commit $::p1 + page_unref $::g(1) } {} for {set i 2} {$i<=20} {incr i} { do_test pager2-3.6.[expr {$i-1}] [subst { @@ -337,6 +338,7 @@ for {set i 1} {$i<20} {incr i} { } set res } {} +breakpoint do_test pager2-4.5.$i.5 { page_write $g1 "Page-1 v$i" lrange [pager_stats $p1] 8 9 @@ -398,6 +400,7 @@ for {set i 1} {$i<20} {incr i} { } do_test pager2-4.99 { + page_unref $::g1 pager_close $::p1 } {} diff --git a/test/pageropt.test b/test/pageropt.test index 3ab1ff5ccd..c28b97c964 100644 --- a/test/pageropt.test +++ b/test/pageropt.test @@ -12,7 +12,7 @@ # The focus of the tests in this file are to verify that the # pager optimizations implemented in version 3.3.14 work. # -# $Id: pageropt.test,v 1.4 2008/04/14 01:00:58 drh Exp $ +# $Id: pageropt.test,v 1.5 2008/08/20 14:49:25 danielk1977 Exp $ set testdir [file dirname $argv0] source $testdir/tester.tcl @@ -39,12 +39,10 @@ proc pagercount_sql {sql {db db}} { set sqlite3_pager_readdb_count 0 set sqlite3_pager_writedb_count 0 set sqlite3_pager_writej_count 0 - set sqlite3_pager_pgfree_count 0 set r [$db eval $sql] set cnt [list $sqlite3_pager_readdb_count \ $sqlite3_pager_writedb_count \ - $sqlite3_pager_writej_count \ - $sqlite3_pager_pgfree_count] + $sqlite3_pager_writej_count ] return [concat $cnt $r] } @@ -59,12 +57,12 @@ do_test pageropt-1.1 { pagercount_sql { CREATE TABLE t1(x); } -} {0 2 0 0} +} {0 2 0} do_test pageropt-1.2 { pagercount_sql { INSERT INTO t1 VALUES(randomblob(5000)); } -} {0 6 2 0} +} {0 6 2} # Verify that values remain in cache on for subsequent reads. # We should not have to go back to disk. @@ -73,7 +71,7 @@ do_test pageropt-1.3 { pagercount_sql { SELECT length(x) FROM t1 } -} {0 0 0 0 5000} +} {0 0 0 5000} # If another thread reads the database, the original cache # remains valid. @@ -84,7 +82,7 @@ do_test pageropt-1.4 { pagercount_sql { SELECT hex(x) FROM t1 } -} [list 0 0 0 0 $blobcontent] +} [list 0 0 0 $blobcontent] # But if the other thread modifies the database, then the cache # must refill. @@ -94,12 +92,12 @@ do_test pageropt-1.5 { pagercount_sql { SELECT hex(x) FROM t1 } -} [list 6 0 0 6 $blobcontent] +} [list 6 0 0 $blobcontent] do_test pageropt-1.6 { pagercount_sql { SELECT hex(x) FROM t1 } -} [list 0 0 0 0 $blobcontent] +} [list 0 0 0 $blobcontent] # Verify that the last page of an overflow chain is not read from # disk when deleting a row. The one row of t1(x) has four pages @@ -117,7 +115,7 @@ do_test pageropt-2.1 { pagercount_sql { DELETE FROM t1 WHERE rowid=1 } -} {5 3 3 0} +} {5 3 3} # When pulling pages off of the freelist, there is no reason # to actually bring in the old content. @@ -128,12 +126,12 @@ do_test pageropt-2.2 { pagercount_sql { INSERT INTO t1 VALUES(randomblob(1500)); } -} {3 4 3 0} +} {3 4 3} do_test pageropt-2.3 { pagercount_sql { INSERT INTO t1 VALUES(randomblob(1500)); } -} {0 4 3 0} +} {0 4 3} # Note the new optimization that when pulling the very last page off of the # freelist we do not read the content of that page. @@ -142,7 +140,7 @@ do_test pageropt-2.4 { pagercount_sql { INSERT INTO t1 VALUES(randomblob(1500)); } -} {0 5 3 0} +} {0 5 3} # Appending a large quantity of data does not involve writing much # to the journal file. @@ -151,7 +149,7 @@ do_test pageropt-3.1 { pagercount_sql { INSERT INTO t2 SELECT * FROM t1; } -} {1 7 2 0} +} {1 7 2} # Once again, we do not need to read the last page of an overflow chain # while deleting. @@ -160,12 +158,12 @@ do_test pageropt-3.2 { pagercount_sql { DROP TABLE t2; } -} {0 2 3 0} +} {0 2 3} do_test pageropt-3.3 { pagercount_sql { DELETE FROM t1; } -} {0 3 3 0} +} {0 3 3} # There are now 11 pages on the freelist. Move them all into an # overflow chain by inserting a single large record. Starting from @@ -180,7 +178,7 @@ do_test pageropt-4.1 { pagercount_sql { INSERT INTO t1 VALUES(randomblob(11300)) } -} {3 13 3 0} +} {3 13 3} # Now we delete that big entries starting from a cold cache and an # empty freelist. The first 10 of the 11 pages overflow chain have @@ -194,7 +192,7 @@ do_test pageropt-4.2 { pagercount_sql { DELETE FROM t1 } -} {12 3 3 0} +} {12 3 3} sqlite3_soft_heap_limit $soft_limit catch {db2 close} diff --git a/test/shared3.test b/test/shared3.test index 0f05aac11c..d9d5fa8ccb 100644 --- a/test/shared3.test +++ b/test/shared3.test @@ -9,7 +9,7 @@ # #*********************************************************************** # -# $Id: shared3.test,v 1.3 2008/06/23 09:50:52 danielk1977 Exp $ +# $Id: shared3.test,v 1.4 2008/08/20 14:49:25 danielk1977 Exp $ set testdir [file dirname $argv0] source $testdir/tester.tcl @@ -86,9 +86,11 @@ do_test shared3-2.7 { catchsql {select count(*) from sqlite_master} db3 } {0 1} do_test shared3-2.8 { + db3 close execsql { INSERT INTO t1 VALUES(10, randomblob(10000)) } db1 + sqlite3 db3 $alternative_name # If the pager-cache is really still limited to 10 pages, then the INSERT # statement above should have caused the pager to grab an exclusive lock diff --git a/test/tkt2409.test b/test/tkt2409.test index 54e82652fa..fcf180cf3a 100644 --- a/test/tkt2409.test +++ b/test/tkt2409.test @@ -16,7 +16,7 @@ # any statement other than a COMMIT, an I/O error is returned instead # of SQLITE_BUSY. # -# $Id: tkt2409.test,v 1.3 2007/09/12 17:01:45 danielk1977 Exp $ +# $Id: tkt2409.test,v 1.4 2008/08/20 14:49:25 danielk1977 Exp $ # Test Outline: # @@ -25,6 +25,11 @@ # Verify that the transaction is automatically rolled back # and SQLITE_IOERR_BLOCKED is returned # +# UPDATE: As of the pcache modifications, failing to upgrade to +# an exclusive lock when attempting a cache-spill is no longer an +# error. The pcache module allocates more space and keeps working +# in memory if this occurs. +# # tkt-2409-2.*: Cause a cache-spill while updating the change-counter # during a database COMMIT. Verify that the transaction is not # rolled back and SQLITE_BUSY is returned. @@ -32,9 +37,14 @@ # tkt-2409-3.*: Similar to 2409-1.*, but using many INSERT statements # within a transaction instead of just one. # +# UPDATE: Again, pcache now just keeps working in main memory. +# # tkt-2409-4.*: Similar to 2409-1.*, but rig it so that the # INSERT statement starts a statement transaction. Verify that -# SQLOTE_BUSY is returned and the transaction is not rolled back. +# SQLITE_BUSY is returned and the transaction is not rolled back. +# +# UPDATE: This time, SQLITE_BUSY is not returned. pcache just uses +# more malloc()'d memory. # set testdir [file dirname $argv0] @@ -69,6 +79,8 @@ proc unread_lock_db {} { } } +pcache_global_max 10 + # Open the db handle used by [read_lock_db]. # sqlite3 db2 test.db @@ -86,11 +98,11 @@ do_test tkt2409-1.1 { BEGIN; INSERT INTO t1 VALUES($::zShort, $::zLong); } -} {1 {disk I/O error}} +} {0 {}} do_test tkt2409-1.2 { sqlite3_errcode $::DB -} {SQLITE_IOERR+11} +} {SQLITE_OK} # Check the integrity of the cache. # @@ -103,8 +115,7 @@ integrity_check tkt2409-1.3 do_test tkt2409-1.4 { unread_lock_db catchsql { ROLLBACK } -} {1 {cannot rollback - no transaction is active}} - +} {0 {}} set ::zShort [string repeat 0123456789 1] set ::zLong [string repeat 0123456789 1500] @@ -139,6 +150,7 @@ do_test tkt2409-2.3 { } } {0 {}} + do_test tkt2409-3.1 { db close set ::DB [sqlite3 db test.db; sqlite3_connection_pointer db] @@ -154,11 +166,11 @@ do_test tkt2409-3.1 { BEGIN; INSERT INTO t1 SELECT $::zShort, $::zLong; } -} {1 {database is locked}} +} {0 {}} do_test tkt2409-3.2 { sqlite3_errcode $::DB -} {SQLITE_BUSY} +} {SQLITE_OK} # Check the integrity of the cache. # @@ -197,11 +209,11 @@ do_test tkt2409-4.1 { read_lock_db execsql BEGIN catchsql $sql -} {1 {disk I/O error}} +} {0 {}} do_test tkt2409-4.2 { sqlite3_errcode $::DB -} {SQLITE_IOERR+11} +} {SQLITE_OK} # Check the integrity of the cache. # @@ -209,8 +221,9 @@ integrity_check tkt2409-4.3 do_test tkt2409-4.4 { catchsql { ROLLBACK } -} {1 {cannot rollback - no transaction is active}} +} {0 {}} +pcache_global_max 0 unread_lock_db db2 close diff --git a/tool/mkfunction.c b/tool/mkfunction.c new file mode 100644 index 0000000000..1fc6812805 --- /dev/null +++ b/tool/mkfunction.c @@ -0,0 +1,181 @@ + +/* +** This file contains a standalone program used to generate C code that +** implements a static hash table to store the definitions of built-in +** SQL functions in SQLite. +*/ + +#include +#include +#include + +/* +** The SQLite source file "func.c" is included below. +** +** By defining the 4 macros and typedef below before including "func.c", +** most of the code is excluded. What is left is an array of constant +** strings, aBuiltinFunc[], containing the names of SQLite's built-in +** SQL functions. i.e.: +** +** const char aBuiltinFunc[] = { "like", "glob", "min", "max" ... }; +** +** The data from aBuiltinFunc[] is used by this program to create the +** static hash table. +*/ +#define CREATE_BUILTIN_HASHTABLE 1 +#define FUNCTION(zName,w,x,y,z) #zName +#define AGGREGATE(zName,v,w,x,y,z) #zName +#define LIKEFUNC(zName,x,y,z) #zName +#define FuncDef const char * + +#include "func.c" + +/* The number of buckets in the static hash table. */ +#define HASHSIZE 127 + +typedef unsigned char u8; + +/* An array to map all upper-case characters into their corresponding +** lower-case character. +*/ +static const u8 sqlite3UpperToLower[] = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, + 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, + 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, + 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 97, 98, 99,100,101,102,103, + 104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121, + 122, 91, 92, 93, 94, 95, 96, 97, 98, 99,100,101,102,103,104,105,106,107, + 108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125, + 126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143, + 144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161, + 162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179, + 180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197, + 198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215, + 216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233, + 234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251, + 252,253,254,255 +}; +#define UpperToLower sqlite3UpperToLower + +int sqlite3StrICmp(const char *zLeft, const char *zRight){ + register unsigned char *a, *b; + a = (unsigned char *)zLeft; + b = (unsigned char *)zRight; + while( *a!=0 && UpperToLower[*a]==UpperToLower[*b]){ a++; b++; } + return UpperToLower[*a] - UpperToLower[*b]; +} + +static int hashstring(const char *zName){ + int ii; + unsigned int iKey = 0; + for(ii=0; zName[ii]; ii++){ + iKey = (iKey<<3) + (u8)sqlite3UpperToLower[(u8)zName[ii]]; + } + iKey = iKey%HASHSIZE; + return iKey; +} + +static void printarray(const char *zName, u8 *aArray, int nArray){ + int ii; + printf(" static u8 %s[%d] = {", zName, nArray); + for(ii=0; ii=0 && 0==sqlite3StrICmp(aBuiltinFunc[ii], aBuiltinFunc[iHead]) ){ + anFunc[iHead]++; + continue; + }else{ + /* The routine generated by this program assumes that if there are + ** two or more entries in the aBuiltinFunc[] array with the same + ** name (i.e. two versions of the "max" function), then they must + ** be stored in adjacent slots. The following block detects the + ** problem if this is not the case. + */ + int jj; + for(jj=0; jjzName, zName, nName) ){\n" + " pFunc = &aBuiltinFunc[iKey = aNext[iKey]];\n" + " }\n" + "\n" + " *paFunc = pFunc;\n" + " return anFunc[iKey];\n" + "}\n" + ); + + return 0; +} + diff --git a/tool/mksqlite3c.tcl b/tool/mksqlite3c.tcl index 98aede2a79..e3b8a777c5 100644 --- a/tool/mksqlite3c.tcl +++ b/tool/mksqlite3c.tcl @@ -101,6 +101,7 @@ foreach hdr { os_os2.h pager.h parse.h + pcache.h rtree.h sqlite3ext.h sqlite3.h @@ -231,6 +232,7 @@ foreach file { os_win.c bitvec.c + pcache.c pager.c btmutex.c @@ -252,7 +254,7 @@ foreach file { build.c callback.c delete.c - func.c + func2.c insert.c legacy.c loadext.c