diff --git a/manifest b/manifest index dce40a05fc..ebd18074c8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C If\sthe\sSQLITE_TRACE_SIZE_LIMIT\scompile-time\sparameter\sis\sset\sto\sa\spositive\ninteger\sthen\slimit\sthe\sexpansion\sof\sstrings\sand\sblobs\sin\strace\soutput\sto\napproximately\sthat\smany\sbytes. -D 2013-04-02T13:56:53.795 +C Try\sto\suse\smmap()\sto\sspeed\saccess\sto\sthe\sdatabase\sfile\son\swindows,\slinux,\nand\smac. +D 2013-04-04T00:40:17.148 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in aafa71d66bab7e87fb2f348152340645f79f0244 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -118,23 +118,23 @@ F src/alter.c f8db986c03eb0bfb221523fc9bbb9d0b70de3168 F src/analyze.c d5f895810e8ff9737c9ec7b76abc3dcff5860335 F src/attach.c 1816f5a9eea8d2010fc2b22b44f0f63eb3a62704 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 -F src/backup.c b2cac9f7993f3f9588827b824b1501d0c820fa68 +F src/backup.c b266767351ae2d847716c56fcb2a1fea7c761c03 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 62ba5954765efc711c873a20a53f60d9fc2843ba -F src/btree.h 3ad7964d6c5b1c7bff569aab6adfa075f8bf06cd +F src/btree.c 5f2c4fe72f663bba837c5d01a732799c1d1c93f8 +F src/btree.h d9490cd37aaeb530a41b07f06e1262950b1be916 F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 F src/build.c 083da8466fd7e481cb8bd5264398f537507f6176 F src/callback.c d7e46f40c3cf53c43550b7da7a1d0479910b62cc F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac -F src/ctime.c 2a5f251fcd7393808df77ccfc817e7058df08c4c +F src/ctime.c 16658a257bc6a3ca8d8961f574cf61a57e4d6faf F src/date.c 067a81c9942c497aafd2c260e13add8a7d0c7dd4 F src/delete.c aeabdabeeeaa0584127f291baa9617153d334778 F src/expr.c 48048fca951eedbc74aa32262154410d56c83812 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c e16942bd5c8a868ac53287886464a5ed0e72b179 F src/func.c 48987c025d69399f59a1c2a553cea5da41bf105d -F src/global.c e59ecd2c553ad0d4bfbc84ca71231336f8993a7a +F src/global.c d2494a1cea8f66a2cab8258449df07f8f0ae6330 F src/hash.c ac3470bbf1ca4ae4e306a8ecb0fdf1731810ffe4 F src/hash.h 8890a25af81fb85a9ad7790d32eedab4b994da22 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 @@ -143,7 +143,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b F src/loadext.c 1422eba4aa2b1fb5f7b3aef574752272477d21e2 -F src/main.c 379160ec3680e3009aa4978eac47027c3ef27ac5 +F src/main.c 54a841854734b6731c4d026834788cac6a19f3d1 F src/malloc.c fe085aa851b666b7c375c1ff957643dc20a04bf6 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 437c7c4af964895d4650f29881df63535caaa1fa @@ -157,34 +157,34 @@ F src/mutex_noop.c 7682796b7d8d39bf1c138248858efcd10c9e1553 F src/mutex_unix.c c3a4e00f96ba068a8dbef34084465979aaf369cc F src/mutex_w32.c 32a9b3841e2d757355f0012b860b1bc5e01eafa0 F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 -F src/os.c e1acdc09ff3ac2412945cca9766e2dcf4675f31c -F src/os.h 027491c77d2404c0a678bb3fb06286f331eb9b57 +F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d +F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 21a36fa0b3753609b6606b30d9338d4bb6b24696 -F src/os_win.c 9fe5356f943425ab8431237bd3a4297044928b70 -F src/pager.c 3e9a15939684b0af441325f05335331b15979c9d -F src/pager.h 1109a06578ec5574dc2c74cf8d9f69daf36fe3e0 +F src/os_unix.c f0ecce40d92469d5cc737ae883e776eb3e5c0af5 +F src/os_win.c 3265df8c762e0b8caf8d02b3352fa8c22e91ae6b +F src/pager.c 2e68df46d4086027cb6b527d47a6dedbf1a6b7ec +F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 -F src/pcache.h 1b5dcc3dc8103d03e625b177023ee67764fa6b7c +F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 9fd22671c270b35131ef480bbc00392b8b5f8ab9 -F src/pragma.c 9f0ee3d74a7f33eeeff40a4b014fc3abf8182ce2 +F src/pragma.c 682e97f3e3b77fd6c9b569eabfbf4a14c987aca3 F src/prepare.c 310eaff1ee5f3c700b3545afb095cfe9346efc3a F src/printf.c 4a9f882f1c1787a8b494a2987765acf9d97ac21f F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 F src/resolve.c 9079da7d59aed2bb14ec8315bc7f720dd85b5b65 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 01540bcd3df3c8f1187158e77986028b1c667258 -F src/shell.c 7c41bfcd9e5bf9d96b9215f79b03a5b2b44a3bca -F src/sqlite.h.in 8d9e83d965f364ff99ebf4b653d0683d05df3900 +F src/shell.c 319b7791cee6c763b60fde1b590bfaf62613cf37 +F src/sqlite.h.in faeb6b3470193e599d79289f77f984b8a78136e1 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 -F src/sqliteInt.h 0f8f05ee4db4ba9120b38f7a3992b325698f6e8a -F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d +F src/sqliteInt.h 1664dc5ad6f8d4dab871416628aa3271044d66c0 +F src/sqliteLimit.h 1097d2c541147d303e66515040067c6bfcf8cf21 F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c 9a716c737590d2f129d71c8fc7065e5aba0e7222 -F src/test1.c ff3e68eedfbd858c9b89cf03e3db233cd29be1d0 +F src/test1.c 26e66b839f42c2eed6833f9023e0098f0d863f35 F src/test2.c 4178056dd1e7d70f954ad8a1e3edb71a2a784daf F src/test3.c 3c3c2407fa6ec7a19e24ae23f7cb439d0275a60d F src/test4.c bf9fa9bece01de08e6f5e02314e4af5c13590dfa @@ -226,7 +226,7 @@ F src/test_spellfix.c 56dfa6d583ac34f61af0834d7b58d674e7e18e13 F src/test_sqllog.c bc50e5afeb7fb50e77b4594e42302df9d05446aa F src/test_stat.c d1569c7a4839f13e80187e2c26b2ab4da2d03935 F src/test_superlock.c 2b97936ca127d13962c3605dbc9a4ef269c424cd -F src/test_syscall.c a992d8c80ea91fbf21fb2dd570db40e77dd7e6ae +F src/test_syscall.c 7e8293e4e6971b0f44c7f7f37b1315a8cc9f6018 F src/test_tclvar.c f4dc67d5f780707210d6bb0eb6016a431c04c7fa F src/test_thread.c e286f2173563f2a1747c24bcda6b9d030bf4f4e4 F src/test_vfs.c fb16b2d9938cf0c1afc5a423b55b952fcc024275 @@ -249,8 +249,8 @@ F src/vdbemem.c 833005f1cbbf447289f1973dba2a0c2228c7b8ab F src/vdbesort.c 4fad64071ae120c25f39dcac572d716b9cadeb7f F src/vdbetrace.c a22263ab47f6ba4fcd176515cec1e732866b25f0 F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 -F src/wal.c f5c7b5027d0ed0e9bc9afeb4a3a8dfea762ec7d2 -F src/wal.h 29c197540b19044e6cd73487017e5e47a1d3dac6 +F src/wal.c 94b5fed2df988fb12f5bf17256e2840e56957a85 +F src/wal.h a4d3da523d55a226a0b28e9058ef88d0a8051887 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b F src/where.c 4ad2329c439a30ddb915a780f6f80bdffafe3a64 F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 @@ -314,6 +314,7 @@ F test/boundary3.tcl 8901d6a503d0bf64251dd81cc74e5ad3add4b119 F test/boundary3.test 56ef82096b4329aca2be74fa1e2b0f762ea0eb45 F test/boundary4.tcl 0bb4b1a94f4fc5ae59b79b9a2b7a140c405e2983 F test/boundary4.test 89e02fa66397b8a325d5eb102b5806f961f8ec4b +F test/btreefault.test 06899a377f31a8c1a3048ec69831522d4e5c6045 F test/busy.test 76b4887f8b9160ba903c1ac22e8ff406ad6ae2f0 F test/cache.test 13bc046b26210471ca6f2889aceb1ea52dc717de F test/capi2.test e8b18cc61090b6e5e388f54d6b125d711d1b265a @@ -369,7 +370,7 @@ F test/cse.test 277350a26264495e86b1785f34d2d0c8600e021c F test/ctime.test 7bd009071e242aac4f18521581536b652b789a47 F test/date.test f3228180c87bbe5d39c9397bf001c0095c3821b9 F test/dbstatus.test 207e5b63fcb7b9c3bb8e1fdf38ebd4654ad0e54b -F test/dbstatus2.test bf7396af964b89e39435babbcdf296ae8fc5f10a +F test/dbstatus2.test f329941d5f4a8bc0ba6ec5735897ef0cf34e2f5f F test/default.test 6faf23ccb300114924353007795aa9a8ec0aa9dc F test/delete.test a065b05d2ebf60fd16639c579a4adfb7c381c701 F test/delete2.test 3a03f2cca1f9a67ec469915cb8babd6485db43fa @@ -403,7 +404,7 @@ F test/eqp.test 46aa946dd55c90635327898275d3e533d23a9845 F test/errmsg.test 050717f1c6a5685de9c79f5f9f6b83d7c592f73a F test/eval.test bc269c365ba877554948441e91ad5373f9f91be3 F test/exclusive.test a1b324cb21834a490cd052d409d34789cfef57cb -F test/exclusive2.test 372be98f6de44dd78734e364b7b626ea211761a6 +F test/exclusive2.test 354bdabe299a2546c898dff42f79079ff1590d88 F test/exec.test e949714dc127eaa5ecc7d723efec1ec27118fdd7 F test/exists.test 8f7b27b61c2fbe5822f0a1f899c715d14e416e30 F test/expr.test 67c9fd6f8f829e239dc8b0f4a08a73c08b09196d @@ -508,7 +509,7 @@ F test/fts4merge2.test 5faa558d1b672f82b847d2a337465fa745e46891 F test/fts4merge3.test aab02a09f50fe6baaddc2e159c3eabc116d45fc7 F test/fts4unicode.test 25ccad45896f8e50f6a694cff738a35f798cdb40 F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d -F test/func.test b058483c17952eff7797b837bbb61e27e6b05606 +F test/func.test a4f24707e6af0bcd4cb5b856af4462db40b047b1 F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f F test/func3.test 001021e5b88bd02a3b365a5c5fd8f6f49d39744a F test/fuzz-oss1.test 4912e528ec9cf2f42134456933659d371c9e0d74 @@ -526,7 +527,7 @@ F test/in2.test 5d4c61d17493c832f7d2d32bef785119e87bde75 F test/in3.test 3cbf58c87f4052cee3a58b37b6389777505aa0c0 F test/in4.test 64f3cc1acde1b9161ccdd8e5bde3daefdb5b2617 F test/in5.test 99f9a40af01711b06d2d614ecfe96129f334fba3 -F test/incrblob.test e7ef2a6094d9b5eb7284c21af2c07644eefffe7d +F test/incrblob.test 2100cb8964e4a106e5ca9bf80e2c5c3e8be33f77 F test/incrblob2.test edc3a96e557bd61fb39acc8d2edd43371fbbaa19 F test/incrblob3.test aedbb35ea1b6450c33b98f2b6ed98e5020be8dc7 F test/incrblob4.test 09be37d3dd996a31ea6993bba7837ece549414a8 @@ -611,7 +612,7 @@ F test/mallocH.test 79b65aed612c9b3ed2dcdaa727c85895fd1bfbdb F test/mallocI.test a88c2b9627c8506bf4703d8397420043a786cdb6 F test/mallocJ.test b5d1839da331d96223e5f458856f8ffe1366f62e F test/mallocK.test d79968641d1b70d88f6c01bdb9a7eb4a55582cc9 -F test/malloc_common.tcl 2930895b0962823ec679853e67e58dd6d8198b3c +F test/malloc_common.tcl 9a98856549bfb3fab205edbc1317216edc52e70d F test/manydb.test 28385ae2087967aa05c38624cec7d96ec74feb3e F test/mem5.test c6460fba403c5703141348cd90de1c294188c68f F test/memdb.test 708a028d6d373e5b3842e4bdc8ba80998c9a4da6 @@ -630,6 +631,8 @@ F test/misc5.test 528468b26d03303b1f047146e5eefc941b9069f5 F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test dd82ec9250b89178b96cd28b2aca70639d21e5b3 F test/misuse.test ba4fb5d1a6101d1c171ea38b3c613d0661c83054 +F test/mmap1.test df5105f08e6000e57b4de7e748f8c2ae3fed75da +F test/mmap2.test 62dbb5d718e66d654d232116c5a2d96e26a071a5 F test/multiplex.test e08cc7177bd6d85990ee1d71100bb6c684c02256 F test/multiplex2.test 580ca5817c7edbe4cc68fa150609c9473393003a F test/multiplex3.test d228f59eac91839a977eac19f21d053f03e4d101 @@ -651,14 +654,14 @@ F test/oserror.test 50417780d0e0d7cd23cf12a8277bb44024765df3 F test/pager1.test 31c04bec797dda1bde337810b52efa08d1f1f08e F test/pager2.test 745b911dde3d1f24ae0870bd433dfa83d7c658c1 F test/pager3.test 3856d9c80839be0668efee1b74811b1b7f7fc95f -F test/pagerfault.test 452f2cc23e3bfcfa935f4442aec1da4fe1dc0442 +F test/pagerfault.test fc2e37b2da626826dd54bb13720b4d721719b660 F test/pagerfault2.test 1f79ea40d1133b2683a2f811b00f2399f7ec2401 F test/pagerfault3.test f16e2efcb5fc9996d1356f7cbc44c998318ae1d7 -F test/pageropt.test 9191867ed19a2b3db6c42d1b36b6fbc657cd1ab0 +F test/pageropt.test 32cb5a6ed7ccddf8e8c842cb44240bd9340223ce F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16 F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 -F test/permutations.test 360b92859c0af814b3fe10b68746936389606501 +F test/permutations.test b6cb45ce4d3193b831231025c9d30c77317dd240 F test/pragma.test 60d29cd3d8098a2c20bf4c072810f99e3bf2757a F test/pragma2.test 3a55f82b954242c642f8342b17dffc8b47472947 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 @@ -731,7 +734,7 @@ F test/softheap1.test c16709a16ad79fa43b32929b2e623d1d117ccf53 F test/sort.test 0e4456e729e5a92a625907c63dcdedfbe72c5dc5 F test/speed1.test f2974a91d79f58507ada01864c0e323093065452 F test/speed1p.explain d841e650a04728b39e6740296b852dccdca9b2cb -F test/speed1p.test c4a469f29f135f4d76c55b1f2a52f36e209466cc +F test/speed1p.test b180e98609c7677382cf618c0ec9b69f789033a8 F test/speed2.test 53177056baf6556dcbdcf032bbdfc41c1aa74ded F test/speed3.test d32043614c08c53eafdc80f33191d5bd9b920523 F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 @@ -747,8 +750,8 @@ F test/subselect.test d24fd8757daf97dafd2e889c73ea4c4272dcf4e4 F test/substr.test 18f57c4ca8a598805c4d64e304c418734d843c1a F test/superlock.test 1cde669f68d2dd37d6c9bd35eee1d95491ae3fc2 F test/sync.test a34cd43e98b7fb84eabbf38f7ed8f7349b3f3d85 -F test/syscall.test bea9bf329bff733c791310244617c2a76974e64a -F test/sysfault.test c79441d88d23696fbec7b147dba98d42a04f523f +F test/syscall.test a653783d985108c4912cc64d341ffbbb55ad2806 +F test/sysfault.test 503f72712b2b21cb80dc9899e53c2e39484d0313 F test/table.test a59d985ca366e39b17b175f387f9d5db5a18d4e2 F test/tableapi.test 2674633fa95d80da917571ebdd759a14d9819126 F test/tclsqlite.test 37a61c2da7e3bfe3b8c1a2867199f6b860df5d43 @@ -952,11 +955,11 @@ F test/vtabF.test fd5ad376f5a34fe0891df1f3cddb4fe7c3eb077e F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 82f463886e18d7f8395a4b6167c91815efe54839 -F test/wal.test a040047d7f2b9f34bc4d597964e5e7c09609c635 +F test/wal.test 62454b2cf00b311e9e65f898aad0fef536ed561a F test/wal2.test d4b470f13c87f6d8268b004380afa04c3c67cb90 F test/wal3.test b22eb662bcbc148c5f6d956eaf94b047f7afe9c0 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c -F test/wal5.test f58ed4b8b542f71c7441da12fbd769d99b362437 +F test/wal5.test f4d0aee6a2cf09e326ed2459011d396b4fdf661a F test/wal6.test 2e3bc767d9c2ce35c47106148d43fcbd072a93b3 F test/wal7.test 2ae8f427d240099cc4b2dfef63cff44e2a68a1bd F test/wal8.test b3ee739fe8f7586aaebdc2367f477ebcf3e3b034 @@ -1041,7 +1044,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 5687e5ee7bafa00d2b353c3eda1e5dfb219cb185 -R b688b887865b3d8981922c40426941a0 +P e5b710849dd66673ba0e0d935b103cb29abfcc4b b2a72be9bab77f050bef75477a278a6294d3e854 +R 60af9b4d9e73e7757090b3abb77cbb41 U drh -Z 66f86621cb15eccea274cb5b5e71ec26 +Z d05e425b384f57133414b300483bca27 diff --git a/manifest.uuid b/manifest.uuid index ec266fb546..4f646cb565 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e5b710849dd66673ba0e0d935b103cb29abfcc4b \ No newline at end of file +fff2be60779571c8fb89158db976ec3755e9a223 \ No newline at end of file diff --git a/src/backup.c b/src/backup.c index 71a8a1a3e7..252f61cfca 100644 --- a/src/backup.c +++ b/src/backup.c @@ -397,7 +397,8 @@ int sqlite3_backup_step(sqlite3_backup *p, int nPage){ const Pgno iSrcPg = p->iNext; /* Source page number */ if( iSrcPg!=PENDING_BYTE_PAGE(p->pSrc->pBt) ){ DbPage *pSrcPg; /* Source page object */ - rc = sqlite3PagerGet(pSrcPager, iSrcPg, &pSrcPg); + rc = sqlite3PagerAcquire(pSrcPager, iSrcPg, &pSrcPg, + PAGER_ACQUIRE_READONLY); if( rc==SQLITE_OK ){ rc = backupOnePage(p, iSrcPg, sqlite3PagerGetData(pSrcPg), 0); sqlite3PagerUnref(pSrcPg); diff --git a/src/btree.c b/src/btree.c index 96140d68c6..c13f0194f6 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1581,13 +1581,17 @@ static int btreeGetPage( BtShared *pBt, /* The btree */ Pgno pgno, /* Number of the page to fetch */ MemPage **ppPage, /* Return the page in this parameter */ - int noContent /* Do not load page content if true */ + int noContent, /* Do not load page content if true */ + int bReadonly /* True if a read-only (mmap) page is ok */ ){ int rc; DbPage *pDbPage; + int flags = (noContent ? PAGER_ACQUIRE_NOCONTENT : 0) + | (bReadonly ? PAGER_ACQUIRE_READONLY : 0); + assert( noContent==0 || bReadonly==0 ); assert( sqlite3_mutex_held(pBt->mutex) ); - rc = sqlite3PagerAcquire(pBt->pPager, pgno, (DbPage**)&pDbPage, noContent); + rc = sqlite3PagerAcquire(pBt->pPager, pgno, (DbPage**)&pDbPage, flags); if( rc ) return rc; *ppPage = btreePageFromDbPage(pDbPage, pgno, pBt); return SQLITE_OK; @@ -1630,9 +1634,10 @@ u32 sqlite3BtreeLastPage(Btree *p){ ** may remain unchanged, or it may be set to an invalid value. */ static int getAndInitPage( - BtShared *pBt, /* The database file */ - Pgno pgno, /* Number of the page to get */ - MemPage **ppPage /* Write the page pointer here */ + BtShared *pBt, /* The database file */ + Pgno pgno, /* Number of the page to get */ + MemPage **ppPage, /* Write the page pointer here */ + int bReadonly /* True if a read-only (mmap) page is ok */ ){ int rc; assert( sqlite3_mutex_held(pBt->mutex) ); @@ -1640,7 +1645,7 @@ static int getAndInitPage( if( pgno>btreePagecount(pBt) ){ rc = SQLITE_CORRUPT_BKPT; }else{ - rc = btreeGetPage(pBt, pgno, ppPage, 0); + rc = btreeGetPage(pBt, pgno, ppPage, 0, bReadonly); if( rc==SQLITE_OK ){ rc = btreeInitPage(*ppPage); if( rc!=SQLITE_OK ){ @@ -1871,6 +1876,7 @@ int sqlite3BtreeOpen( rc = sqlite3PagerOpen(pVfs, &pBt->pPager, zFilename, EXTRA_SIZE, flags, vfsFlags, pageReinit); if( rc==SQLITE_OK ){ + sqlite3PagerSetMmapLimit(pBt->pPager, db->mxMmap); rc = sqlite3PagerReadFileheader(pBt->pPager,sizeof(zDbHeader),zDbHeader); } if( rc!=SQLITE_OK ){ @@ -2137,6 +2143,19 @@ int sqlite3BtreeSetCacheSize(Btree *p, int mxPage){ return SQLITE_OK; } +/* +** Change the limit on the amount of the database file that may be +** memory mapped. +*/ +int sqlite3BtreeSetMmapLimit(Btree *p, sqlite3_int64 mxMmap){ + BtShared *pBt = p->pBt; + assert( sqlite3_mutex_held(p->db->mutex) ); + sqlite3BtreeEnter(p); + sqlite3PagerSetMmapLimit(pBt->pPager, mxMmap); + sqlite3BtreeLeave(p); + return SQLITE_OK; +} + /* ** Change the way data is synced to disk in order to increase or decrease ** how well the database resists damage due to OS crashes and power @@ -2362,7 +2381,7 @@ static int lockBtree(BtShared *pBt){ assert( pBt->pPage1==0 ); rc = sqlite3PagerSharedLock(pBt->pPager); if( rc!=SQLITE_OK ) return rc; - rc = btreeGetPage(pBt, 1, &pPage1, 0); + rc = btreeGetPage(pBt, 1, &pPage1, 0, 0); if( rc!=SQLITE_OK ) return rc; /* Do some checking to help insure the file we opened really is @@ -2921,7 +2940,7 @@ static int relocatePage( ** iPtrPage. */ if( eType!=PTRMAP_ROOTPAGE ){ - rc = btreeGetPage(pBt, iPtrPage, &pPtrPage, 0); + rc = btreeGetPage(pBt, iPtrPage, &pPtrPage, 0, 0); if( rc!=SQLITE_OK ){ return rc; } @@ -3005,7 +3024,7 @@ static int incrVacuumStep(BtShared *pBt, Pgno nFin, Pgno iLastPg, int bCommit){ u8 eMode = BTALLOC_ANY; /* Mode parameter for allocateBtreePage() */ Pgno iNear = 0; /* nearby parameter for allocateBtreePage() */ - rc = btreeGetPage(pBt, iLastPg, &pLastPg, 0); + rc = btreeGetPage(pBt, iLastPg, &pLastPg, 0, 0); if( rc!=SQLITE_OK ){ return rc; } @@ -3097,8 +3116,11 @@ int sqlite3BtreeIncrVacuum(Btree *p){ if( nOrig0 ){ - invalidateAllOverflowCache(pBt); - rc = incrVacuumStep(pBt, nFin, nOrig, 0); + rc = saveAllCursors(pBt, 0, 0); + if( rc==SQLITE_OK ){ + invalidateAllOverflowCache(pBt); + rc = incrVacuumStep(pBt, nFin, nOrig, 0); + } if( rc==SQLITE_OK ){ rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); put4byte(&pBt->pPage1->aData[28], pBt->nPage); @@ -3146,7 +3168,9 @@ static int autoVacuumCommit(BtShared *pBt){ nFree = get4byte(&pBt->pPage1->aData[36]); nFin = finalDbSize(pBt, nOrig, nFree); if( nFin>nOrig ) return SQLITE_CORRUPT_BKPT; - + if( nFinnFin && rc==SQLITE_OK; iFree--){ rc = incrVacuumStep(pBt, nFin, iFree, 1); } @@ -3163,7 +3187,7 @@ static int autoVacuumCommit(BtShared *pBt){ } } - assert( nRef==sqlite3PagerRefcount(pPager) ); + assert( nRef>=sqlite3PagerRefcount(pPager) ); return rc; } @@ -3419,7 +3443,7 @@ int sqlite3BtreeRollback(Btree *p, int tripCode){ /* The rollback may have destroyed the pPage1->aData value. So ** call btreeGetPage() on page 1 again to make ** sure pPage1->aData is set correctly. */ - if( btreeGetPage(pBt, 1, &pPage1, 0)==SQLITE_OK ){ + if( btreeGetPage(pBt, 1, &pPage1, 0, 0)==SQLITE_OK ){ int nPage = get4byte(28+(u8*)pPage1->aData); testcase( nPage==0 ); if( nPage==0 ) sqlite3PagerPagecount(pBt->pPager, &nPage); @@ -3853,7 +3877,7 @@ static int getOverflowPage( assert( next==0 || rc==SQLITE_DONE ); if( rc==SQLITE_OK ){ - rc = btreeGetPage(pBt, ovfl, &pPage, 0); + rc = btreeGetPage(pBt, ovfl, &pPage, 0, (ppPage==0)); assert( rc==SQLITE_OK || pPage==0 ); if( rc==SQLITE_OK ){ next = get4byte(pPage->aData); @@ -4074,7 +4098,9 @@ static int accessPayload( { DbPage *pDbPage; - rc = sqlite3PagerGet(pBt->pPager, nextPage, &pDbPage); + rc = sqlite3PagerAcquire(pBt->pPager, nextPage, &pDbPage, + (eOp==0 ? PAGER_ACQUIRE_READONLY : 0) + ); if( rc==SQLITE_OK ){ aPayload = sqlite3PagerGetData(pDbPage); nextPage = get4byte(aPayload); @@ -4253,10 +4279,11 @@ static int moveToChild(BtCursor *pCur, u32 newPgno){ assert( cursorHoldsMutex(pCur) ); assert( pCur->eState==CURSOR_VALID ); assert( pCur->iPageiPage>=0 ); if( pCur->iPage>=(BTCURSOR_MAX_DEPTH-1) ){ return SQLITE_CORRUPT_BKPT; } - rc = getAndInitPage(pBt, newPgno, &pNewPage); + rc = getAndInitPage(pBt, newPgno, &pNewPage, (pCur->wrFlag==0)); if( rc ) return rc; pCur->apPage[i+1] = pNewPage; pCur->aiIdx[i+1] = 0; @@ -4373,7 +4400,7 @@ static int moveToRoot(BtCursor *pCur){ pCur->eState = CURSOR_INVALID; return SQLITE_OK; }else{ - rc = getAndInitPage(pBt, pCur->pgnoRoot, &pCur->apPage[0]); + rc = getAndInitPage(pBt, pCur->pgnoRoot, &pCur->apPage[0], pCur->wrFlag==0); if( rc!=SQLITE_OK ){ pCur->eState = CURSOR_INVALID; return rc; @@ -4987,7 +5014,7 @@ static int allocateBtreePage( if( iTrunk>mxPage ){ rc = SQLITE_CORRUPT_BKPT; }else{ - rc = btreeGetPage(pBt, iTrunk, &pTrunk, 0); + rc = btreeGetPage(pBt, iTrunk, &pTrunk, 0, 0); } if( rc ){ pTrunk = 0; @@ -5051,7 +5078,7 @@ static int allocateBtreePage( goto end_allocate_page; } testcase( iNewTrunk==mxPage ); - rc = btreeGetPage(pBt, iNewTrunk, &pNewTrunk, 0); + rc = btreeGetPage(pBt, iNewTrunk, &pNewTrunk, 0, 0); if( rc!=SQLITE_OK ){ goto end_allocate_page; } @@ -5131,7 +5158,7 @@ static int allocateBtreePage( } put4byte(&aData[4], k-1); noContent = !btreeGetHasContent(pBt, *pPgno); - rc = btreeGetPage(pBt, *pPgno, ppPage, noContent); + rc = btreeGetPage(pBt, *pPgno, ppPage, noContent, 0); if( rc==SQLITE_OK ){ rc = sqlite3PagerWrite((*ppPage)->pDbPage); if( rc!=SQLITE_OK ){ @@ -5179,7 +5206,7 @@ static int allocateBtreePage( MemPage *pPg = 0; TRACE(("ALLOCATE: %d from end of file (pointer-map page)\n", pBt->nPage)); assert( pBt->nPage!=PENDING_BYTE_PAGE(pBt) ); - rc = btreeGetPage(pBt, pBt->nPage, &pPg, bNoContent); + rc = btreeGetPage(pBt, pBt->nPage, &pPg, bNoContent, 0); if( rc==SQLITE_OK ){ rc = sqlite3PagerWrite(pPg->pDbPage); releasePage(pPg); @@ -5193,7 +5220,7 @@ static int allocateBtreePage( *pPgno = pBt->nPage; assert( *pPgno!=PENDING_BYTE_PAGE(pBt) ); - rc = btreeGetPage(pBt, *pPgno, ppPage, bNoContent); + rc = btreeGetPage(pBt, *pPgno, ppPage, bNoContent, 0); if( rc ) return rc; rc = sqlite3PagerWrite((*ppPage)->pDbPage); if( rc!=SQLITE_OK ){ @@ -5261,7 +5288,7 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ /* If the secure_delete option is enabled, then ** always fully overwrite deleted information with zeros. */ - if( (!pPage && ((rc = btreeGetPage(pBt, iPage, &pPage, 0))!=0) ) + if( (!pPage && ((rc = btreeGetPage(pBt, iPage, &pPage, 0, 0))!=0) ) || ((rc = sqlite3PagerWrite(pPage->pDbPage))!=0) ){ goto freepage_out; @@ -5288,7 +5315,7 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ u32 nLeaf; /* Initial number of leaf cells on trunk page */ iTrunk = get4byte(&pPage1->aData[32]); - rc = btreeGetPage(pBt, iTrunk, &pTrunk, 0); + rc = btreeGetPage(pBt, iTrunk, &pTrunk, 0, 0); if( rc!=SQLITE_OK ){ goto freepage_out; } @@ -5334,7 +5361,7 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ ** first trunk in the free-list is full. Either way, the page being freed ** will become the new first trunk page in the free-list. */ - if( pPage==0 && SQLITE_OK!=(rc = btreeGetPage(pBt, iPage, &pPage, 0)) ){ + if( pPage==0 && SQLITE_OK!=(rc = btreeGetPage(pBt, iPage, &pPage, 0, 0)) ){ goto freepage_out; } rc = sqlite3PagerWrite(pPage->pDbPage); @@ -6135,7 +6162,7 @@ static int balance_nonroot( } pgno = get4byte(pRight); while( 1 ){ - rc = getAndInitPage(pBt, pgno, &apOld[i]); + rc = getAndInitPage(pBt, pgno, &apOld[i], 0); if( rc ){ memset(apOld, 0, (i+1)*sizeof(MemPage*)); goto balance_cleanup; @@ -7223,10 +7250,17 @@ static int btreeCreateTable(Btree *p, int *piTable, int createTabFlags){ u8 eType = 0; Pgno iPtrPage = 0; + /* Save the positions of any open cursors. This is required in + ** case they are holding a reference to an xFetch reference + ** corresponding to page pgnoRoot. */ + rc = saveAllCursors(pBt, 0, 0); releasePage(pPageMove); + if( rc!=SQLITE_OK ){ + return rc; + } /* Move the page currently at pgnoRoot to pgnoMove. */ - rc = btreeGetPage(pBt, pgnoRoot, &pRoot, 0); + rc = btreeGetPage(pBt, pgnoRoot, &pRoot, 0, 0); if( rc!=SQLITE_OK ){ return rc; } @@ -7247,7 +7281,7 @@ static int btreeCreateTable(Btree *p, int *piTable, int createTabFlags){ if( rc!=SQLITE_OK ){ return rc; } - rc = btreeGetPage(pBt, pgnoRoot, &pRoot, 0); + rc = btreeGetPage(pBt, pgnoRoot, &pRoot, 0, 0); if( rc!=SQLITE_OK ){ return rc; } @@ -7323,7 +7357,7 @@ static int clearDatabasePage( return SQLITE_CORRUPT_BKPT; } - rc = getAndInitPage(pBt, pgno, &pPage); + rc = getAndInitPage(pBt, pgno, &pPage, 0); if( rc ) return rc; for(i=0; inCell; i++){ pCell = findCell(pPage, i); @@ -7425,7 +7459,7 @@ static int btreeDropTable(Btree *p, Pgno iTable, int *piMoved){ return SQLITE_LOCKED_SHAREDCACHE; } - rc = btreeGetPage(pBt, (Pgno)iTable, &pPage, 0); + rc = btreeGetPage(pBt, (Pgno)iTable, &pPage, 0, 0); if( rc ) return rc; rc = sqlite3BtreeClearTable(p, iTable, 0); if( rc ){ @@ -7460,7 +7494,7 @@ static int btreeDropTable(Btree *p, Pgno iTable, int *piMoved){ */ MemPage *pMove; releasePage(pPage); - rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0); + rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0, 0); if( rc!=SQLITE_OK ){ return rc; } @@ -7470,7 +7504,7 @@ static int btreeDropTable(Btree *p, Pgno iTable, int *piMoved){ return rc; } pMove = 0; - rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0); + rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0, 0); freePage(pMove, &rc); releasePage(pMove); if( rc!=SQLITE_OK ){ @@ -7882,7 +7916,7 @@ static int checkTreePage( usableSize = pBt->usableSize; if( iPage==0 ) return 0; if( checkRef(pCheck, iPage, zParentContext) ) return 0; - if( (rc = btreeGetPage(pBt, (Pgno)iPage, &pPage, 0))!=0 ){ + if( (rc = btreeGetPage(pBt, (Pgno)iPage, &pPage, 0, 0))!=0 ){ checkAppendMsg(pCheck, zContext, "unable to get the page. error code=%d", rc); return 0; @@ -8354,6 +8388,17 @@ int sqlite3BtreePutData(BtCursor *pCsr, u32 offset, u32 amt, void *z){ return SQLITE_ABORT; } + /* Save the positions of all other cursors open on this table. This is + ** required in case any of them are holding references to an xFetch + ** version of the b-tree page modified by the accessPayload call below. + ** + ** Note that pCsr must be open on a BTREE_INTKEY table and saveCursorPosition() + ** and hence saveAllCursors() cannot fail on a BTREE_INTKEY table, hence + ** saveAllCursors can only return SQLITE_OK. + */ + VVA_ONLY(rc =) saveAllCursors(pCsr->pBt, pCsr->pgnoRoot, pCsr); + assert( rc==SQLITE_OK ); + /* Check some assumptions: ** (a) the cursor is open for writing, ** (b) there is a read/write transaction open, diff --git a/src/btree.h b/src/btree.h index d4c9fe37d7..23f6ad7068 100644 --- a/src/btree.h +++ b/src/btree.h @@ -63,6 +63,7 @@ int sqlite3BtreeOpen( int sqlite3BtreeClose(Btree*); int sqlite3BtreeSetCacheSize(Btree*,int); +int sqlite3BtreeSetMmapLimit(Btree*,sqlite3_int64); int sqlite3BtreeSetSafetyLevel(Btree*,int,int,int); int sqlite3BtreeSyncDisabled(Btree*); int sqlite3BtreeSetPageSize(Btree *p, int nPagesize, int nReserve, int eFix); diff --git a/src/ctime.c b/src/ctime.c index c42454ca71..ee6d4cbe2e 100644 --- a/src/ctime.c +++ b/src/ctime.c @@ -57,6 +57,9 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_DEFAULT_LOCKING_MODE "DEFAULT_LOCKING_MODE=" CTIMEOPT_VAL(SQLITE_DEFAULT_LOCKING_MODE), #endif +#ifdef SQLITE_DEFAULT_MMAP_LIMIT + "DEFAULT_MMAP_LIMIT=" CTIMEOPT_VAL(SQLITE_DEFAULT_MMAP_LIMIT), +#endif #ifdef SQLITE_DISABLE_DIRSYNC "DISABLE_DIRSYNC", #endif diff --git a/src/global.c b/src/global.c index f5da7e7f1f..aec3958c1f 100644 --- a/src/global.c +++ b/src/global.c @@ -156,6 +156,7 @@ SQLITE_WSD struct Sqlite3Config sqlite3Config = { (void*)0, /* pHeap */ 0, /* nHeap */ 0, 0, /* mnHeap, mxHeap */ + SQLITE_DEFAULT_MMAP_LIMIT, /* mxMmap */ (void*)0, /* pScratch */ 0, /* szScratch */ 0, /* nScratch */ diff --git a/src/main.c b/src/main.c index 2fcb236e83..288d4535f3 100644 --- a/src/main.c +++ b/src/main.c @@ -496,6 +496,13 @@ int sqlite3_config(int op, ...){ } #endif + case SQLITE_CONFIG_MMAP_LIMIT: { + sqlite3_int64 mxMmap = va_arg(ap, sqlite3_int64); + if( mxMmap<0 ) mxMmap = SQLITE_DEFAULT_MMAP_LIMIT; + sqlite3GlobalConfig.mxMmap = mxMmap; + break; + } + default: { rc = SQLITE_ERROR; break; @@ -2316,6 +2323,7 @@ static int openDatabase( memcpy(db->aLimit, aHardLimit, sizeof(db->aLimit)); db->autoCommit = 1; db->nextAutovac = -1; + db->mxMmap = sqlite3GlobalConfig.mxMmap; db->nextPagesize = 0; db->flags |= SQLITE_ShortColNames | SQLITE_AutoIndex | SQLITE_EnableTrigger #if SQLITE_DEFAULT_FILE_FORMAT<4 diff --git a/src/os.c b/src/os.c index b5e918a727..2ba857e2aa 100644 --- a/src/os.c +++ b/src/os.c @@ -141,6 +141,14 @@ int sqlite3OsShmMap( return id->pMethods->xShmMap(id, iPage, pgsz, bExtend, pp); } +int sqlite3OsFetch(sqlite3_file *id, i64 iOff, int iAmt, void **pp){ + DO_OS_MALLOC_TEST(id); + return id->pMethods->xFetch(id, iOff, iAmt, pp); +} +int sqlite3OsUnfetch(sqlite3_file *id, i64 iOff, void *p){ + return id->pMethods->xUnfetch(id, iOff, p); +} + /* ** The next group of routines are convenience wrappers around the ** VFS methods. diff --git a/src/os.h b/src/os.h index 1ec7d4ba11..82e07ee1cb 100644 --- a/src/os.h +++ b/src/os.h @@ -259,6 +259,8 @@ int sqlite3OsShmMap(sqlite3_file *,int,int,int,void volatile **); int sqlite3OsShmLock(sqlite3_file *id, int, int, int); void sqlite3OsShmBarrier(sqlite3_file *id); int sqlite3OsShmUnmap(sqlite3_file *id, int); +int sqlite3OsFetch(sqlite3_file *id, i64, int, void **); +int sqlite3OsUnfetch(sqlite3_file *, i64, void *); /* diff --git a/src/os_unix.c b/src/os_unix.c index 84b1594ac3..75f9236013 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -225,6 +225,11 @@ struct unixFile { const char *zPath; /* Name of the file */ unixShm *pShm; /* Shared memory segment information */ int szChunk; /* Configured by FCNTL_CHUNK_SIZE */ + int nFetchOut; /* Number of outstanding xFetch refs */ + sqlite3_int64 mmapSize; /* Usable size of mapping at pMapRegion */ + sqlite3_int64 mmapOrigsize; /* Actual size of mapping at pMapRegion */ + sqlite3_int64 mmapLimit; /* Configured FCNTL_MMAP_LIMIT value */ + void *pMapRegion; /* Memory mapped region */ #ifdef __QNXNTO__ int sectorSize; /* Device sector size */ int deviceCharacteristics; /* Precomputed device characteristics */ @@ -249,7 +254,9 @@ struct unixFile { unsigned char transCntrChng; /* True if the transaction counter changed */ unsigned char dbUpdate; /* True if any part of database file changed */ unsigned char inNormalWrite; /* True if in a normal write operation */ + #endif + #ifdef SQLITE_TEST /* In test mode, increase the size of this structure a bit so that ** it is larger than the struct CrashFile defined in test6.c. @@ -306,6 +313,17 @@ struct unixFile { #define threadid 0 #endif +/* +** HAVE_MREMAP defaults to true on Linux and false everywhere else. +*/ +#if !defined(HAVE_MREMAP) +# if defined(__linux__) && defined(_GNU_SOURCE) +# define HAVE_MREMAP 1 +# else +# define HAVE_MREMAP 0 +# endif +#endif + /* ** Different Unix systems declare open() in different ways. Same use ** open(const char*,int,mode_t). Others use open(const char*,int,...). @@ -437,6 +455,19 @@ static struct unix_syscall { { "fchown", (sqlite3_syscall_ptr)posixFchown, 0 }, #define osFchown ((int(*)(int,uid_t,gid_t))aSyscall[20].pCurrent) + { "mmap", (sqlite3_syscall_ptr)mmap, 0 }, +#define osMmap ((void*(*)(void*,size_t,int,int,int,off_t))aSyscall[21].pCurrent) + + { "munmap", (sqlite3_syscall_ptr)munmap, 0 }, +#define osMunmap ((void*(*)(void*,size_t))aSyscall[22].pCurrent) + +#if HAVE_MREMAP + { "mremap", (sqlite3_syscall_ptr)mremap, 0 }, +#else + { "mremap", (sqlite3_syscall_ptr)0, 0 }, +#endif +#define osMremap ((void*(*)(void*,size_t,size_t,int,...))aSyscall[23].pCurrent) + }; /* End of the overrideable system calls */ /* @@ -1104,7 +1135,6 @@ static int unixLogErrorAtLine( zErr = strerror(iErrno); #endif - assert( errcode!=SQLITE_OK ); if( zPath==0 ) zPath = ""; sqlite3_log(errcode, "os_unix.c:%d: (%d) %s(%s) - %s", @@ -1800,9 +1830,13 @@ end_unlock: ** the requested locking level, this routine is a no-op. */ static int unixUnlock(sqlite3_file *id, int eFileLock){ + assert( eFileLock==SHARED_LOCK || ((unixFile *)id)->nFetchOut==0 ); return posixUnlock(id, eFileLock, 0); } +static int unixMapfile(unixFile *pFd, i64 nByte); +static void unixUnmapfile(unixFile *pFd); + /* ** This function performs the parts of the "close file" operation ** common to all locking schemes. It closes the directory and file @@ -1815,6 +1849,7 @@ static int unixUnlock(sqlite3_file *id, int eFileLock){ */ static int closeUnixFile(sqlite3_file *id){ unixFile *pFile = (unixFile*)id; + unixUnmapfile(pFile); if( pFile->h>=0 ){ robust_close(pFile, pFile->h, __LINE__); pFile->h = -1; @@ -3082,6 +3117,21 @@ static int unixRead( ); #endif + /* Deal with as much of this read request as possible by transfering + ** data from the memory mapping using memcpy(). */ + if( offsetmmapSize ){ + if( offset+amt <= pFile->mmapSize ){ + memcpy(pBuf, &((u8 *)(pFile->pMapRegion))[offset], amt); + return SQLITE_OK; + }else{ + int nCopy = pFile->mmapSize - offset; + memcpy(pBuf, &((u8 *)(pFile->pMapRegion))[offset], nCopy); + pBuf = &((u8 *)pBuf)[nCopy]; + amt -= nCopy; + offset += nCopy; + } + } + got = seekAndRead(pFile, offset, pBuf, amt); if( got==amt ){ return SQLITE_OK; @@ -3186,6 +3236,21 @@ static int unixWrite( } #endif + /* Deal with as much of this write request as possible by transfering + ** data from the memory mapping using memcpy(). */ + if( offsetmmapSize ){ + if( offset+amt <= pFile->mmapSize ){ + memcpy(&((u8 *)(pFile->pMapRegion))[offset], pBuf, amt); + return SQLITE_OK; + }else{ + int nCopy = pFile->mmapSize - offset; + memcpy(&((u8 *)(pFile->pMapRegion))[offset], pBuf, nCopy); + pBuf = &((u8 *)pBuf)[nCopy]; + amt -= nCopy; + offset += nCopy; + } + } + while( amt>0 && (wrote = seekAndWrite(pFile, offset, pBuf, amt))>0 ){ amt -= wrote; offset += wrote; @@ -3468,6 +3533,14 @@ static int unixTruncate(sqlite3_file *id, i64 nByte){ } #endif + /* If the file was just truncated to a size smaller than the currently + ** mapped region, reduce the effective mapping size as well. SQLite will + ** use read() and write() to access data beyond this point from now on. + */ + if( nBytemmapSize ){ + pFile->mmapSize = nByte; + } + return SQLITE_OK; } } @@ -3556,6 +3629,19 @@ static int fcntlSizeHint(unixFile *pFile, i64 nByte){ } } + if( pFile->mmapLimit>0 && nByte>pFile->mmapSize ){ + int rc; + if( pFile->szChunk<=0 ){ + if( robust_ftruncate(pFile->h, nByte) ){ + pFile->lastErrno = errno; + return unixLogError(SQLITE_IOERR_TRUNCATE, "ftruncate", pFile->zPath); + } + } + + rc = unixMapfile(pFile, nByte); + return rc; + } + return SQLITE_OK; } @@ -3623,6 +3709,12 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ } return SQLITE_OK; } + case SQLITE_FCNTL_MMAP_LIMIT: { + i64 newLimit = *(i64*)pArg; + *(i64*)pArg = pFile->mmapLimit; + if( newLimit>=0 ) pFile->mmapLimit = newLimit; + return SQLITE_OK; + } #ifdef SQLITE_DEBUG /* The pager calls this method to signal that it has done ** a rollback and that the database is therefore unchanged and @@ -3935,7 +4027,7 @@ static void unixShmPurge(unixFile *pFd){ sqlite3_mutex_free(p->mutex); for(i=0; inRegion; i++){ if( p->h>=0 ){ - munmap(p->apRegion[i], p->szRegion); + osMunmap(p->apRegion[i], p->szRegion); }else{ sqlite3_free(p->apRegion[i]); } @@ -4208,7 +4300,7 @@ static int unixShmMap( while(pShmNode->nRegion<=iRegion){ void *pMem; if( pShmNode->h>=0 ){ - pMem = mmap(0, szRegion, + pMem = osMmap(0, szRegion, pShmNode->isReadonly ? PROT_READ : PROT_READ|PROT_WRITE, MAP_SHARED, pShmNode->h, szRegion*(i64)pShmNode->nRegion ); @@ -4425,6 +4517,223 @@ static int unixShmUnmap( # define unixShmUnmap 0 #endif /* #ifndef SQLITE_OMIT_WAL */ +/* +** If it is currently memory mapped, unmap file pFd. +*/ +static void unixUnmapfile(unixFile *pFd){ + assert( pFd->nFetchOut==0 ); + if( pFd->pMapRegion ){ + osMunmap(pFd->pMapRegion, pFd->mmapOrigsize); + pFd->pMapRegion = 0; + pFd->mmapSize = 0; + pFd->mmapOrigsize = 0; + } +} + +/* +** Return the system page size. +*/ +static int unixGetPagesize(void){ +#if HAVE_MREMAP + return 512; +#elif defined(_BSD_SOURCE) + return getpagesize(); +#else + return (int)sysconf(_SC_PAGESIZE); +#endif +} + +/* +** Attempt to set the size of the memory mapping maintained by file +** descriptor pFd to nNew bytes. Any existing mapping is discarded. +** +** If successful, this function sets the following variables: +** +** unixFile.pMapRegion +** unixFile.mmapSize +** unixFile.mmapOrigsize +** +** If unsuccessful, an error message is logged via sqlite3_log() and +** the three variables above are zeroed. In this case SQLite should +** continue accessing the database using the xRead() and xWrite() +** methods. +*/ +static void unixRemapfile( + unixFile *pFd, /* File descriptor object */ + i64 nNew /* Required mapping size */ +){ + const char *zErr = "mmap"; + int h = pFd->h; /* File descriptor open on db file */ + u8 *pOrig = (u8 *)pFd->pMapRegion; /* Pointer to current file mapping */ + i64 nOrig = pFd->mmapOrigsize; /* Size of pOrig region in bytes */ + u8 *pNew = 0; /* Location of new mapping */ + int flags = PROT_READ; /* Flags to pass to mmap() */ + + assert( pFd->nFetchOut==0 ); + assert( nNew>pFd->mmapSize ); + assert( nNew<=pFd->mmapLimit ); + assert( nNew>0 ); + assert( pFd->mmapOrigsize>=pFd->mmapSize ); + assert( MAP_FAILED!=0 ); + + if( (pFd->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; + + if( pOrig ){ + const int szSyspage = unixGetPagesize(); + i64 nReuse = (pFd->mmapSize & ~(szSyspage-1)); + u8 *pReq = &pOrig[nReuse]; + + /* Unmap any pages of the existing mapping that cannot be reused. */ + if( nReuse!=nOrig ){ + osMunmap(pReq, nOrig-nReuse); + } + +#if HAVE_MREMAP + pNew = osMremap(pOrig, nReuse, nNew, MREMAP_MAYMOVE); + zErr = "mremap"; +#else + pNew = osMmap(pReq, nNew-nReuse, flags, MAP_SHARED, h, nReuse); + if( pNew!=MAP_FAILED ){ + if( pNew!=pReq ){ + osMunmap(pNew, nNew - nReuse); + pNew = 0; + }else{ + pNew = pOrig; + } + } +#endif + + /* The attempt to extend the existing mapping failed. Free it. */ + if( pNew==MAP_FAILED || pNew==0 ){ + osMunmap(pOrig, nReuse); + } + } + + /* If pNew is still NULL, try to create an entirely new mapping. */ + if( pNew==0 ){ + pNew = osMmap(0, nNew, flags, MAP_SHARED, h, 0); + } + + if( pNew==MAP_FAILED ){ + pNew = 0; + nNew = 0; + unixLogError(SQLITE_OK, zErr, pFd->zPath); + + /* If the mmap() above failed, assume that all subsequent mmap() calls + ** will probably fail too. Fall back to using xRead/xWrite exclusively + ** in this case. */ + pFd->mmapLimit = 0; + } + pFd->pMapRegion = (void *)pNew; + pFd->mmapSize = pFd->mmapOrigsize = nNew; +} + +/* +** Memory map or remap the file opened by file-descriptor pFd (if the file +** is already mapped, the existing mapping is replaced by the new). Or, if +** there already exists a mapping for this file, and there are still +** outstanding xFetch() references to it, this function is a no-op. +** +** If parameter nByte is non-negative, then it is the requested size of +** the mapping to create. Otherwise, if nByte is less than zero, then the +** requested size is the size of the file on disk. The actual size of the +** created mapping is either the requested size or the value configured +** using SQLITE_FCNTL_MMAP_LIMIT, whichever is smaller. +** +** SQLITE_OK is returned if no error occurs (even if the mapping is not +** recreated as a result of outstanding references) or an SQLite error +** code otherwise. +*/ +static int unixMapfile(unixFile *pFd, i64 nByte){ + i64 nMap = nByte; + int rc; + + assert( nMap>=0 || pFd->nFetchOut==0 ); + if( pFd->nFetchOut>0 ) return SQLITE_OK; + + if( nMap<0 ){ + struct stat statbuf; /* Low-level file information */ + rc = osFstat(pFd->h, &statbuf); + if( rc!=SQLITE_OK ){ + return SQLITE_IOERR_FSTAT; + } + nMap = statbuf.st_size; + } + if( nMap>pFd->mmapLimit ){ + nMap = pFd->mmapLimit; + } + + if( nMap!=pFd->mmapSize ){ + if( nMap>0 ){ + unixRemapfile(pFd, nMap); + }else{ + unixUnmapfile(pFd); + } + } + + return SQLITE_OK; +} + +/* +** If possible, return a pointer to a mapping of file fd starting at offset +** iOff. The mapping must be valid for at least nAmt bytes. +** +** If such a pointer can be obtained, store it in *pp and return SQLITE_OK. +** Or, if one cannot but no error occurs, set *pp to 0 and return SQLITE_OK. +** Finally, if an error does occur, return an SQLite error code. The final +** value of *pp is undefined in this case. +** +** If this function does return a pointer, the caller must eventually +** release the reference by calling unixUnfetch(). +*/ +static int unixFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ + unixFile *pFd = (unixFile *)fd; /* The underlying database file */ + *pp = 0; + + if( pFd->mmapLimit>0 ){ + if( pFd->pMapRegion==0 ){ + int rc = unixMapfile(pFd, -1); + if( rc!=SQLITE_OK ) return rc; + } + if( pFd->mmapSize >= iOff+nAmt ){ + *pp = &((u8 *)pFd->pMapRegion)[iOff]; + pFd->nFetchOut++; + } + } + return SQLITE_OK; +} + +/* +** If the third argument is non-NULL, then this function releases a +** reference obtained by an earlier call to unixFetch(). The second +** argument passed to this function must be the same as the corresponding +** argument that was passed to the unixFetch() invocation. +** +** Or, if the third argument is NULL, then this function is being called +** to inform the VFS layer that, according to POSIX, any existing mapping +** may now be invalid and should be unmapped. +*/ +static int unixUnfetch(sqlite3_file *fd, i64 iOff, void *p){ + unixFile *pFd = (unixFile *)fd; /* The underlying database file */ + + /* If p==0 (unmap the entire file) then there must be no outstanding + ** xFetch references. Or, if p!=0 (meaning it is an xFetch reference), + ** then there must be at least one outstanding. */ + assert( (p==0)==(pFd->nFetchOut==0) ); + + /* If p!=0, it must match the iOff value. */ + assert( p==0 || p==&((u8 *)pFd->pMapRegion)[iOff] ); + + if( p ){ + pFd->nFetchOut--; + }else{ + unixUnmapfile(pFd); + } + + assert( pFd->nFetchOut>=0 ); + return SQLITE_OK; +} + /* ** Here ends the implementation of all sqlite3_file methods. ** @@ -4483,7 +4792,9 @@ static const sqlite3_io_methods METHOD = { \ unixShmMap, /* xShmMap */ \ unixShmLock, /* xShmLock */ \ unixShmBarrier, /* xShmBarrier */ \ - unixShmUnmap /* xShmUnmap */ \ + unixShmUnmap, /* xShmUnmap */ \ + unixFetch, /* xFetch */ \ + unixUnfetch, /* xUnfetch */ \ }; \ static const sqlite3_io_methods *FINDER##Impl(const char *z, unixFile *p){ \ UNUSED_PARAMETER(z); UNUSED_PARAMETER(p); \ @@ -4500,7 +4811,7 @@ static const sqlite3_io_methods *(*const FINDER)(const char*,unixFile *p) \ IOMETHODS( posixIoFinder, /* Finder function name */ posixIoMethods, /* sqlite3_io_methods object name */ - 2, /* shared memory is enabled */ + 3, /* shared memory and mmap are enabled */ unixClose, /* xClose method */ unixLock, /* xLock method */ unixUnlock, /* xUnlock method */ @@ -4751,6 +5062,7 @@ static int fillInUnixFile( pNew->pVfs = pVfs; pNew->zPath = zFilename; pNew->ctrlFlags = (u8)ctrlFlags; + pNew->mmapLimit = sqlite3GlobalConfig.mxMmap; if( sqlite3_uri_boolean(((ctrlFlags & UNIXFILE_URI) ? zFilename : 0), "psow", SQLITE_POWERSAFE_OVERWRITE) ){ pNew->ctrlFlags |= UNIXFILE_PSOW; @@ -6988,7 +7300,7 @@ int sqlite3_os_init(void){ /* Double-check that the aSyscall[] array has been constructed ** correctly. See ticket [bb3a86e890c8e96ab] */ - assert( ArraySize(aSyscall)==21 ); + assert( ArraySize(aSyscall)==24 ); /* Register all VFSes defined in the aVfs[] array */ for(i=0; i<(sizeof(aVfs)/sizeof(sqlite3_vfs)); i++){ diff --git a/src/os_win.c b/src/os_win.c index 58ba25f302..a7966171b5 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -150,11 +150,18 @@ struct winFile { winceLock local; /* Locks obtained by this instance of winFile */ winceLock *shared; /* Global shared lock memory for the file */ #endif + int nFetchOut; /* Number of outstanding xFetch references */ + HANDLE hMap; /* Handle for accessing memory mapping */ + void *pMapRegion; /* Area memory mapped */ + sqlite3_int64 mmapSize; /* Usable size of mapped region */ + sqlite3_int64 mmapOrigsize; /* Actual size of mapped region */ + sqlite3_int64 mmapLimit; /* Configured FCNTL_MMAP_LIMIT value */ }; /* ** Allowed values for winFile.ctrlFlags */ +#define WINFILE_RDONLY 0x02 /* Connection is read only */ #define WINFILE_PERSIST_WAL 0x04 /* Persistent WAL mode */ #define WINFILE_PSOW 0x10 /* SQLITE_IOCAP_POWERSAFE_OVERWRITE */ @@ -2061,6 +2068,9 @@ static int seekWinFile(winFile *pFile, sqlite3_int64 iOffset){ #endif } +/* Forward references to VFS methods */ +static int winUnmapfile(winFile*); + /* ** Close a file. ** @@ -2082,6 +2092,10 @@ static int winClose(sqlite3_file *id){ #endif OSTRACE(("CLOSE %d\n", pFile->h)); assert( pFile->h!=NULL && pFile->h!=INVALID_HANDLE_VALUE ); + + rc = winUnmapfile(pFile); + if( rc!=SQLITE_OK ) return rc; + do{ rc = osCloseHandle(pFile->h); /* SimulateIOError( rc=0; cnt=MX_CLOSE_ATTEMPT; ); */ @@ -2130,9 +2144,25 @@ static int winRead( int nRetry = 0; /* Number of retrys */ assert( id!=0 ); + assert( amt>0 ); SimulateIOError(return SQLITE_IOERR_READ); OSTRACE(("READ %d lock=%d\n", pFile->h, pFile->locktype)); + /* Deal with as much of this read request as possible by transfering + ** data from the memory mapping using memcpy(). */ + if( offsetmmapSize ){ + if( offset+amt <= pFile->mmapSize ){ + memcpy(pBuf, &((u8 *)(pFile->pMapRegion))[offset], amt); + return SQLITE_OK; + }else{ + int nCopy = (int)(pFile->mmapSize - offset); + memcpy(pBuf, &((u8 *)(pFile->pMapRegion))[offset], nCopy); + pBuf = &((u8 *)pBuf)[nCopy]; + amt -= nCopy; + offset += nCopy; + } + } + #if SQLITE_OS_WINCE if( seekWinFile(pFile, offset) ){ return SQLITE_FULL; @@ -2182,6 +2212,21 @@ static int winWrite( OSTRACE(("WRITE %d lock=%d\n", pFile->h, pFile->locktype)); + /* Deal with as much of this write request as possible by transfering + ** data from the memory mapping using memcpy(). */ + if( offsetmmapSize ){ + if( offset+amt <= pFile->mmapSize ){ + memcpy(&((u8 *)(pFile->pMapRegion))[offset], pBuf, amt); + return SQLITE_OK; + }else{ + int nCopy = (int)(pFile->mmapSize - offset); + memcpy(&((u8 *)(pFile->pMapRegion))[offset], pBuf, nCopy); + pBuf = &((u8 *)pBuf)[nCopy]; + amt -= nCopy; + offset += nCopy; + } + } + #if SQLITE_OS_WINCE rc = seekWinFile(pFile, offset); if( rc==0 ){ @@ -2249,6 +2294,7 @@ static int winWrite( static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ winFile *pFile = (winFile*)id; /* File handle object */ int rc = SQLITE_OK; /* Return code for this function */ + DWORD lastErrno; assert( pFile ); @@ -2267,11 +2313,20 @@ static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ /* SetEndOfFile() returns non-zero when successful, or zero when it fails. */ if( seekWinFile(pFile, nByte) ){ rc = winLogError(SQLITE_IOERR_TRUNCATE, pFile->lastErrno, - "winTruncate1", pFile->zPath); - }else if( 0==osSetEndOfFile(pFile->h) ){ - pFile->lastErrno = osGetLastError(); + "winTruncate1", pFile->zPath); + }else if( 0==osSetEndOfFile(pFile->h) && + ((lastErrno = osGetLastError())!=ERROR_USER_MAPPED_FILE) ){ + pFile->lastErrno = lastErrno; rc = winLogError(SQLITE_IOERR_TRUNCATE, pFile->lastErrno, - "winTruncate2", pFile->zPath); + "winTruncate2", pFile->zPath); + } + + /* If the file was truncated to a size smaller than the currently + ** mapped region, reduce the effective mapping size as well. SQLite will + ** use read() and write() to access data beyond this point from now on. + */ + if( pFile->pMapRegion && nBytemmapSize ){ + pFile->mmapSize = nByte; } OSTRACE(("TRUNCATE %d %lld %s\n", pFile->h, nByte, rc ? "failed" : "ok")); @@ -2781,6 +2836,12 @@ static int winFileControl(sqlite3_file *id, int op, void *pArg){ } return SQLITE_OK; } + case SQLITE_FCNTL_MMAP_LIMIT: { + i64 newLimit = *(i64*)pArg; + *(i64*)pArg = pFile->mmapLimit; + if( newLimit>=0 ) pFile->mmapLimit = newLimit; + return SQLITE_OK; + } } return SQLITE_NOTFOUND; } @@ -3451,6 +3512,184 @@ shmpage_out: # define winShmUnmap 0 #endif /* #ifndef SQLITE_OMIT_WAL */ +/* +** Cleans up the mapped region of the specified file, if any. +*/ +static int winUnmapfile(winFile *pFile){ + assert( pFile!=0 ); + if( pFile->pMapRegion ){ + if( !osUnmapViewOfFile(pFile->pMapRegion) ){ + pFile->lastErrno = osGetLastError(); + return winLogError(SQLITE_IOERR_MMAP, pFile->lastErrno, + "winUnmap1", pFile->zPath); + } + pFile->pMapRegion = 0; + pFile->mmapSize = 0; + pFile->mmapOrigsize = 0; + } + if( pFile->hMap!=NULL ){ + if( !osCloseHandle(pFile->hMap) ){ + pFile->lastErrno = osGetLastError(); + return winLogError(SQLITE_IOERR_MMAP, pFile->lastErrno, + "winUnmap2", pFile->zPath); + } + pFile->hMap = NULL; + } + return SQLITE_OK; +} + +/* +** Memory map or remap the file opened by file-descriptor pFd (if the file +** is already mapped, the existing mapping is replaced by the new). Or, if +** there already exists a mapping for this file, and there are still +** outstanding xFetch() references to it, this function is a no-op. +** +** If parameter nByte is non-negative, then it is the requested size of +** the mapping to create. Otherwise, if nByte is less than zero, then the +** requested size is the size of the file on disk. The actual size of the +** created mapping is either the requested size or the value configured +** using SQLITE_FCNTL_MMAP_LIMIT, whichever is smaller. +** +** SQLITE_OK is returned if no error occurs (even if the mapping is not +** recreated as a result of outstanding references) or an SQLite error +** code otherwise. +*/ +static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ + sqlite3_int64 nMap = nByte; + int rc; + + assert( nMap>=0 || pFd->nFetchOut==0 ); + if( pFd->nFetchOut>0 ) return SQLITE_OK; + + if( nMap<0 ){ + rc = winFileSize((sqlite3_file*)pFd, &nMap); + if( rc ){ + return SQLITE_IOERR_FSTAT; + } + } + if( nMap>pFd->mmapLimit ){ + nMap = pFd->mmapLimit; + } + nMap &= ~(sqlite3_int64)(winSysInfo.dwPageSize - 1); + + if( nMap==0 && pFd->mmapSize>0 ){ + winUnmapfile(pFd); + } + if( nMap!=pFd->mmapSize ){ + void *pNew = 0; + DWORD protect = PAGE_READONLY; + DWORD flags = FILE_MAP_READ; + + winUnmapfile(pFd); + if( (pFd->ctrlFlags & WINFILE_RDONLY)==0 ){ + protect = PAGE_READWRITE; + flags |= FILE_MAP_WRITE; + } +#if SQLITE_OS_WINRT + pFd->hMap = osCreateFileMappingFromApp(pFd->h, NULL, protect, nMap, NULL); +#elif defined(SQLITE_WIN32_HAS_WIDE) + pFd->hMap = osCreateFileMappingW(pFd->h, NULL, protect, + (DWORD)((nMap>>32) & 0xffffffff), + (DWORD)(nMap & 0xffffffff), NULL); +#elif defined(SQLITE_WIN32_HAS_ANSI) + pFd->hMap = osCreateFileMappingA(pFd->h, NULL, protect, + (DWORD)((nMap>>32) & 0xffffffff), + (DWORD)(nMap & 0xffffffff), NULL); +#endif + if( pFd->hMap==NULL ){ + pFd->lastErrno = osGetLastError(); + rc = winLogError(SQLITE_IOERR_MMAP, pFd->lastErrno, + "winMapfile", pFd->zPath); + /* Log the error, but continue normal operation using xRead/xWrite */ + return SQLITE_OK; + } + assert( (nMap % winSysInfo.dwPageSize)==0 ); +#if SQLITE_OS_WINRT + pNew = osMapViewOfFileFromApp(pFd->hMap, flags, 0, nMap); +#else + assert( sizeof(SIZE_T)==sizeof(sqlite3_int64) || nMap<=0xffffffff ); + pNew = osMapViewOfFile(pFd->hMap, flags, 0, 0, (SIZE_T)nMap); +#endif + if( pNew==NULL ){ + osCloseHandle(pFd->hMap); + pFd->hMap = NULL; + pFd->lastErrno = osGetLastError(); + winLogError(SQLITE_IOERR_MMAP, pFd->lastErrno, + "winMapfile", pFd->zPath); + return SQLITE_OK; + } + pFd->pMapRegion = pNew; + pFd->mmapSize = nMap; + pFd->mmapOrigsize = nMap; + } + + return SQLITE_OK; +} + +/* +** If possible, return a pointer to a mapping of file fd starting at offset +** iOff. The mapping must be valid for at least nAmt bytes. +** +** If such a pointer can be obtained, store it in *pp and return SQLITE_OK. +** Or, if one cannot but no error occurs, set *pp to 0 and return SQLITE_OK. +** Finally, if an error does occur, return an SQLite error code. The final +** value of *pp is undefined in this case. +** +** If this function does return a pointer, the caller must eventually +** release the reference by calling unixUnfetch(). +*/ +static int winFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ + winFile *pFd = (winFile*)fd; /* The underlying database file */ + *pp = 0; + + if( pFd->mmapLimit>0 ){ + if( pFd->pMapRegion==0 ){ + int rc = winMapfile(pFd, -1); + if( rc!=SQLITE_OK ) return rc; + } + if( pFd->mmapSize >= iOff+nAmt ){ + *pp = &((u8 *)pFd->pMapRegion)[iOff]; + pFd->nFetchOut++; + } + } + return SQLITE_OK; +} + +/* +** If the third argument is non-NULL, then this function releases a +** reference obtained by an earlier call to unixFetch(). The second +** argument passed to this function must be the same as the corresponding +** argument that was passed to the unixFetch() invocation. +** +** Or, if the third argument is NULL, then this function is being called +** to inform the VFS layer that, according to POSIX, any existing mapping +** may now be invalid and should be unmapped. +*/ +static int winUnfetch(sqlite3_file *fd, i64 iOff, void *p){ + winFile *pFd = (winFile*)fd; /* The underlying database file */ + + /* If p==0 (unmap the entire file) then there must be no outstanding + ** xFetch references. Or, if p!=0 (meaning it is an xFetch reference), + ** then there must be at least one outstanding. */ + assert( (p==0)==(pFd->nFetchOut==0) ); + + /* If p!=0, it must match the iOff value. */ + assert( p==0 || p==&((u8 *)pFd->pMapRegion)[iOff] ); + + if( p ){ + pFd->nFetchOut--; + }else{ + /* FIXME: If Windows truly always prevents truncating or deleting a + ** file while a mapping is held, then the following winUnmapfile() call + ** is unnecessary can can be omitted - potentially improving + ** performance. */ + winUnmapfile(pFd); + } + + assert( pFd->nFetchOut>=0 ); + return SQLITE_OK; +} + /* ** Here ends the implementation of all sqlite3_file methods. ** @@ -3462,7 +3701,7 @@ shmpage_out: ** sqlite3_file for win32. */ static const sqlite3_io_methods winIoMethod = { - 2, /* iVersion */ + 3, /* iVersion */ winClose, /* xClose */ winRead, /* xRead */ winWrite, /* xWrite */ @@ -3478,7 +3717,9 @@ static const sqlite3_io_methods winIoMethod = { winShmMap, /* xShmMap */ winShmLock, /* xShmLock */ winShmBarrier, /* xShmBarrier */ - winShmUnmap /* xShmUnmap */ + winShmUnmap, /* xShmUnmap */ + winFetch, /* xFetch */ + winUnfetch /* xUnfetch */ }; /**************************************************************************** @@ -3654,9 +3895,7 @@ static int winOpen( int isExclusive = (flags & SQLITE_OPEN_EXCLUSIVE); int isDelete = (flags & SQLITE_OPEN_DELETEONCLOSE); int isCreate = (flags & SQLITE_OPEN_CREATE); -#ifndef NDEBUG int isReadonly = (flags & SQLITE_OPEN_READONLY); -#endif int isReadWrite = (flags & SQLITE_OPEN_READWRITE); #ifndef NDEBUG @@ -3867,11 +4106,19 @@ static int winOpen( pFile->pMethod = &winIoMethod; pFile->pVfs = pVfs; pFile->h = h; + if( isReadonly ){ + pFile->ctrlFlags |= WINFILE_RDONLY; + } if( sqlite3_uri_boolean(zName, "psow", SQLITE_POWERSAFE_OVERWRITE) ){ pFile->ctrlFlags |= WINFILE_PSOW; } pFile->lastErrno = NO_ERROR; pFile->zPath = zName; + pFile->hMap = NULL; + pFile->pMapRegion = 0; + pFile->mmapSize = 0; + pFile->mmapOrigsize = 0; + pFile->mmapLimit = sqlite3GlobalConfig.mxMmap; OpenCounter(+1); return rc; @@ -4500,7 +4747,6 @@ int sqlite3_os_init(void){ ** correctly. See ticket [bb3a86e890c8e96ab] */ assert( ArraySize(aSyscall)==74 ); -#ifndef SQLITE_OMIT_WAL /* get memory map allocation granularity */ memset(&winSysInfo, 0, sizeof(SYSTEM_INFO)); #if SQLITE_OS_WINRT @@ -4508,8 +4754,8 @@ int sqlite3_os_init(void){ #else osGetSystemInfo(&winSysInfo); #endif - assert(winSysInfo.dwAllocationGranularity > 0); -#endif + assert( winSysInfo.dwAllocationGranularity>0 ); + assert( winSysInfo.dwPageSize>0 ); sqlite3_vfs_register(&winVfs, 1); return SQLITE_OK; diff --git a/src/pager.c b/src/pager.c index dfa179d542..ae2f10069f 100644 --- a/src/pager.c +++ b/src/pager.c @@ -655,6 +655,11 @@ struct Pager { PagerSavepoint *aSavepoint; /* Array of active savepoints */ int nSavepoint; /* Number of elements in aSavepoint[] */ char dbFileVers[16]; /* Changes whenever database file changes */ + + u8 bUseFetch; /* True to use xFetch() */ + int nMmapOut; /* Number of mmap pages currently outstanding */ + sqlite3_int64 mxMmap; /* Desired maximum mmap size */ + PgHdr *pMmapFreelist; /* List of free mmap page headers (pDirty) */ /* ** End of the routinely-changing class members ***************************************************************************/ @@ -2252,7 +2257,7 @@ static int pager_playback_one_page( i64 ofst = (pgno-1)*(i64)pPager->pageSize; testcase( !isSavepnt && pPg!=0 && (pPg->flags&PGHDR_NEED_SYNC)!=0 ); assert( !pagerUseWal(pPager) ); - rc = sqlite3OsWrite(pPager->fd, (u8*)aData, pPager->pageSize, ofst); + rc = sqlite3OsWrite(pPager->fd, (u8 *)aData, pPager->pageSize, ofst); if( pgno>pPager->dbFileSize ){ pPager->dbFileSize = pgno; } @@ -2834,11 +2839,10 @@ end_playback: ** If an IO error occurs, then the IO error is returned to the caller. ** Otherwise, SQLITE_OK is returned. */ -static int readDbPage(PgHdr *pPg){ +static int readDbPage(PgHdr *pPg, u32 iFrame){ Pager *pPager = pPg->pPager; /* Pager object associated with page pPg */ Pgno pgno = pPg->pgno; /* Page number to read */ int rc = SQLITE_OK; /* Return code */ - int isInWal = 0; /* True if page is in log file */ int pgsz = pPager->pageSize; /* Number of bytes to read */ assert( pPager->eState>=PAGER_READER && !MEMDB ); @@ -2850,11 +2854,10 @@ static int readDbPage(PgHdr *pPg){ return SQLITE_OK; } - if( pagerUseWal(pPager) ){ + if( iFrame ){ /* Try to pull the page from the write-ahead log. */ - rc = sqlite3WalRead(pPager->pWal, pgno, &isInWal, pgsz, pPg->pData); - } - if( rc==SQLITE_OK && !isInWal ){ + rc = sqlite3WalReadFrame(pPager->pWal, iFrame, pgsz, pPg->pData); + }else{ i64 iOffset = (pgno-1)*(i64)pPager->pageSize; rc = sqlite3OsRead(pPager->fd, pPg->pData, pgsz, iOffset); if( rc==SQLITE_IOERR_SHORT_READ ){ @@ -2933,12 +2936,17 @@ static int pagerUndoCallback(void *pCtx, Pgno iPg){ Pager *pPager = (Pager *)pCtx; PgHdr *pPg; + assert( pagerUseWal(pPager) ); pPg = sqlite3PagerLookup(pPager, iPg); if( pPg ){ if( sqlite3PcachePageRefcount(pPg)==1 ){ sqlite3PcacheDrop(pPg); }else{ - rc = readDbPage(pPg); + u32 iFrame = 0; + rc = sqlite3WalFindFrame(pPager->pWal, pPg->pgno, &iFrame); + if( rc==SQLITE_OK ){ + rc = readDbPage(pPg, iFrame); + } if( rc==SQLITE_OK ){ pPager->xReiniter(pPg); } @@ -3082,6 +3090,7 @@ static int pagerBeginReadTransaction(Pager *pPager){ rc = sqlite3WalBeginReadTransaction(pPager->pWal, &changed); if( rc!=SQLITE_OK || changed ){ pager_reset(pPager); + if( pPager->bUseFetch ) sqlite3OsUnfetch(pPager->fd, 0, 0); } return rc; @@ -3343,6 +3352,27 @@ void sqlite3PagerSetCachesize(Pager *pPager, int mxPage){ sqlite3PcacheSetCachesize(pPager->pPCache, mxPage); } +/* +** Invoke SQLITE_FCNTL_MMAP_LIMIT based on the current value of mxMmap. +*/ +static void pagerFixMaplimit(Pager *pPager){ + sqlite3_file *fd = pPager->fd; + if( isOpen(fd) ){ + sqlite3_int64 mx; + pPager->bUseFetch = (fd->pMethods->iVersion>=3) && pPager->mxMmap>0; + mx = pPager->mxMmap; + sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_MMAP_LIMIT, &mx); + } +} + +/* +** Change the maximum size of any memory mapping made of the database file. +*/ +void sqlite3PagerSetMmapLimit(Pager *pPager, sqlite3_int64 mxMmap){ + pPager->mxMmap = mxMmap; + pagerFixMaplimit(pPager); +} + /* ** Free as much memory as possible from the pager. */ @@ -3578,6 +3608,7 @@ int sqlite3PagerSetPagesize(Pager *pPager, u32 *pPageSize, int nReserve){ assert( nReserve>=0 && nReserve<1000 ); pPager->nReserve = (i16)nReserve; pagerReportSize(pPager); + pagerFixMaplimit(pPager); } return rc; } @@ -3803,6 +3834,81 @@ static int pagerSyncHotJournal(Pager *pPager){ return rc; } +/* +** Obtain a reference to a memory mapped page object for page number pgno. +** The new object will use the pointer pData, obtained from xFetch(). +** If successful, set *ppPage to point to the new page reference +** and return SQLITE_OK. Otherwise, return an SQLite error code and set +** *ppPage to zero. +** +** Page references obtained by calling this function should be released +** by calling pagerReleaseMapPage(). +*/ +static int pagerAcquireMapPage( + Pager *pPager, /* Pager object */ + Pgno pgno, /* Page number */ + void *pData, /* xFetch()'d data for this page */ + PgHdr **ppPage /* OUT: Acquired page object */ +){ + PgHdr *p; /* Memory mapped page to return */ + + if( pPager->pMmapFreelist ){ + *ppPage = p = pPager->pMmapFreelist; + pPager->pMmapFreelist = p->pDirty; + p->pDirty = 0; + memset(p->pExtra, 0, pPager->nExtra); + }else{ + *ppPage = p = (PgHdr *)sqlite3MallocZero(sizeof(PgHdr) + pPager->nExtra); + if( p==0 ){ + sqlite3OsUnfetch(pPager->fd, (i64)(pgno-1) * pPager->pageSize, pData); + return SQLITE_NOMEM; + } + p->pExtra = (void *)&p[1]; + p->flags = PGHDR_MMAP; + p->nRef = 1; + p->pPager = pPager; + } + + assert( p->pExtra==(void *)&p[1] ); + assert( p->pPage==0 ); + assert( p->flags==PGHDR_MMAP ); + assert( p->pPager==pPager ); + assert( p->nRef==1 ); + + p->pgno = pgno; + p->pData = pData; + pPager->nMmapOut++; + + return SQLITE_OK; +} + +/* +** Release a reference to page pPg. pPg must have been returned by an +** earlier call to pagerAcquireMapPage(). +*/ +static void pagerReleaseMapPage(PgHdr *pPg){ + Pager *pPager = pPg->pPager; + pPager->nMmapOut--; + pPg->pDirty = pPager->pMmapFreelist; + pPager->pMmapFreelist = pPg; + + assert( pPager->fd->pMethods->iVersion>=3 ); + sqlite3OsUnfetch(pPager->fd, (i64)(pPg->pgno-1)*pPager->pageSize, pPg->pData); +} + +/* +** Free all PgHdr objects stored in the Pager.pMmapFreelist list. +*/ +static void pagerFreeMapHdrs(Pager *pPager){ + PgHdr *p; + PgHdr *pNext; + for(p=pPager->pMmapFreelist; p; p=pNext){ + pNext = p->pDirty; + sqlite3_free(p); + } +} + + /* ** Shutdown the page cache. Free all memory and close all files. ** @@ -3823,6 +3929,7 @@ int sqlite3PagerClose(Pager *pPager){ assert( assert_pager_state(pPager) ); disable_simulated_io_errors(); sqlite3BeginBenignMalloc(); + pagerFreeMapHdrs(pPager); /* pPager->errCode = 0; */ pPager->exclusiveMode = 0; #ifndef SQLITE_OMIT_WAL @@ -4084,7 +4191,9 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ ** file size will be. */ assert( rc!=SQLITE_OK || isOpen(pPager->fd) ); - if( rc==SQLITE_OK && pPager->dbSize>pPager->dbHintSize ){ + if( rc==SQLITE_OK + && (pList->pDirty ? pPager->dbSize : pList->pgno+1)>pPager->dbHintSize + ){ sqlite3_int64 szFile = pPager->pageSize * (sqlite3_int64)pPager->dbSize; sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_SIZE_HINT, &szFile); pPager->dbHintSize = pPager->dbSize; @@ -4638,6 +4747,7 @@ int sqlite3PagerOpen( /* pPager->pBusyHandlerArg = 0; */ pPager->xReiniter = xReinit; /* memset(pPager->aHash, 0, sizeof(pPager->aHash)); */ + /* pPager->mxMmap = SQLITE_DEFAULT_MMAP_LIMIT // will be set by btree.c */ *ppPager = pPager; return SQLITE_OK; @@ -4929,9 +5039,11 @@ int sqlite3PagerSharedLock(Pager *pPager){ ); } - if( !pPager->tempFile - && (pPager->pBackup || sqlite3PcachePagecount(pPager->pPCache)>0) - ){ + if( !pPager->tempFile && ( + pPager->pBackup + || sqlite3PcachePagecount(pPager->pPCache)>0 + || pPager->bUseFetch + )){ /* The shared-lock has just been acquired on the database file ** and there are already pages in the cache (from a previous ** read or write transaction). Check to see if the database @@ -4957,7 +5069,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ if( nPage>0 ){ IOTRACE(("CKVERS %p %d\n", pPager, sizeof(dbFileVers))); rc = sqlite3OsRead(pPager->fd, &dbFileVers, sizeof(dbFileVers), 24); - if( rc!=SQLITE_OK ){ + if( rc!=SQLITE_OK && rc!=SQLITE_IOERR_SHORT_READ ){ goto failed; } }else{ @@ -4966,6 +5078,16 @@ int sqlite3PagerSharedLock(Pager *pPager){ if( memcmp(pPager->dbFileVers, dbFileVers, sizeof(dbFileVers))!=0 ){ pager_reset(pPager); + + /* Unmap the database file. It is possible that external processes + ** may have truncated the database file and then extended it back + ** to its original size while this process was not holding a lock. + ** In this case there may exist a Pager.pMap mapping that appears + ** to be the right size but is not actually valid. Avoid this + ** possibility by unmapping the db here. */ + if( pPager->bUseFetch ){ + sqlite3OsUnfetch(pPager->fd, 0, 0); + } } } @@ -5007,7 +5129,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ ** nothing to rollback, so this routine is a no-op. */ static void pagerUnlockIfUnused(Pager *pPager){ - if( (sqlite3PcacheRefCount(pPager->pPCache)==0) ){ + if( pPager->nMmapOut==0 && (sqlite3PcacheRefCount(pPager->pPCache)==0) ){ pagerUnlockAndRollback(pPager); } } @@ -5066,13 +5188,24 @@ int sqlite3PagerAcquire( Pager *pPager, /* The pager open on the database file */ Pgno pgno, /* Page number to fetch */ DbPage **ppPage, /* Write a pointer to the page here */ - int noContent /* Do not bother reading content from disk if true */ + int flags /* PAGER_ACQUIRE_XXX flags */ ){ - int rc; - PgHdr *pPg; + int rc = SQLITE_OK; + PgHdr *pPg = 0; + u32 iFrame = 0; /* Frame to read from WAL file */ + const int noContent = (flags & PAGER_ACQUIRE_NOCONTENT); + + /* It is acceptable to use a read-only (mmap) page for any page except + ** page 1 if there is no write-transaction open or the ACQUIRE_READONLY + ** flag was specified by the caller. And so long as the db is not a + ** temporary or in-memory database. */ + const int bMmapOk = (pgno!=1 && pPager->bUseFetch + && (pPager->eState==PAGER_READER || (flags & PAGER_ACQUIRE_READONLY)) + ); assert( pPager->eState>=PAGER_READER ); assert( assert_pager_state(pPager) ); + assert( noContent==0 || bMmapOk==0 ); if( pgno==0 ){ return SQLITE_CORRUPT_BKPT; @@ -5083,6 +5216,39 @@ int sqlite3PagerAcquire( if( pPager->errCode!=SQLITE_OK ){ rc = pPager->errCode; }else{ + + if( bMmapOk && pagerUseWal(pPager) ){ + rc = sqlite3WalFindFrame(pPager->pWal, pgno, &iFrame); + if( rc!=SQLITE_OK ) goto pager_acquire_err; + } + + if( iFrame==0 && bMmapOk ){ + void *pData = 0; + + rc = sqlite3OsFetch(pPager->fd, + (i64)(pgno-1) * pPager->pageSize, pPager->pageSize, &pData + ); + + if( rc==SQLITE_OK && pData ){ + if( pPager->eState>PAGER_READER ){ + (void)sqlite3PcacheFetch(pPager->pPCache, pgno, 0, &pPg); + } + if( pPg==0 ){ + rc = pagerAcquireMapPage(pPager, pgno, pData, &pPg); + }else{ + sqlite3OsUnfetch(pPager->fd, (i64)(pgno-1)*pPager->pageSize, pData); + } + if( pPg ){ + assert( rc==SQLITE_OK ); + *ppPage = pPg; + return SQLITE_OK; + } + } + if( rc!=SQLITE_OK ){ + goto pager_acquire_err; + } + } + rc = sqlite3PcacheFetch(pPager->pPCache, pgno, 1, ppPage); } @@ -5141,9 +5307,13 @@ int sqlite3PagerAcquire( memset(pPg->pData, 0, pPager->pageSize); IOTRACE(("ZERO %p %d\n", pPager, pgno)); }else{ + if( pagerUseWal(pPager) && bMmapOk==0 ){ + rc = sqlite3WalFindFrame(pPager->pWal, pgno, &iFrame); + if( rc!=SQLITE_OK ) goto pager_acquire_err; + } assert( pPg->pPager==pPager ); pPager->aStat[PAGER_STAT_MISS]++; - rc = readDbPage(pPg); + rc = readDbPage(pPg, iFrame); if( rc!=SQLITE_OK ){ goto pager_acquire_err; } @@ -5196,7 +5366,11 @@ DbPage *sqlite3PagerLookup(Pager *pPager, Pgno pgno){ void sqlite3PagerUnref(DbPage *pPg){ if( pPg ){ Pager *pPager = pPg->pPager; - sqlite3PcacheRelease(pPg); + if( pPg->flags & PGHDR_MMAP ){ + pagerReleaseMapPage(pPg); + }else{ + sqlite3PcacheRelease(pPg); + } pagerUnlockIfUnused(pPager); } } @@ -5531,6 +5705,7 @@ int sqlite3PagerWrite(DbPage *pDbPage){ Pager *pPager = pPg->pPager; Pgno nPagePerSector = (pPager->sectorSize/pPager->pageSize); + assert( (pPg->flags & PGHDR_MMAP)==0 ); assert( pPager->eState>=PAGER_WRITER_LOCKED ); assert( pPager->eState!=PAGER_ERROR ); assert( assert_pager_state(pPager) ); @@ -6087,7 +6262,7 @@ int sqlite3PagerRollback(Pager *pPager){ } assert( pPager->eState==PAGER_READER || rc!=SQLITE_OK ); - assert( rc==SQLITE_OK || rc==SQLITE_FULL + assert( rc==SQLITE_OK || rc==SQLITE_FULL || rc==SQLITE_CORRUPT || rc==SQLITE_NOMEM || (rc&0xFF)==SQLITE_IOERR ); /* If an error occurs during a ROLLBACK, we can no longer trust the pager @@ -6821,11 +6996,12 @@ static int pagerOpenWal(Pager *pPager){ ** (e.g. due to malloc() failure), return an error code. */ if( rc==SQLITE_OK ){ - rc = sqlite3WalOpen(pPager->pVfs, + rc = sqlite3WalOpen(pPager->pVfs, pPager->fd, pPager->zWal, pPager->exclusiveMode, pPager->journalSizeLimit, &pPager->pWal ); } + pagerFixMaplimit(pPager); return rc; } @@ -6916,6 +7092,7 @@ int sqlite3PagerCloseWal(Pager *pPager){ rc = sqlite3WalClose(pPager->pWal, pPager->ckptSyncFlags, pPager->pageSize, (u8*)pPager->pTmpSpace); pPager->pWal = 0; + pagerFixMaplimit(pPager); } } return rc; diff --git a/src/pager.h b/src/pager.h index 90f8e6af78..6f659136e2 100644 --- a/src/pager.h +++ b/src/pager.h @@ -78,6 +78,12 @@ typedef struct PgHdr DbPage; #define PAGER_JOURNALMODE_MEMORY 4 /* In-memory journal file */ #define PAGER_JOURNALMODE_WAL 5 /* Use write-ahead logging */ +/* +** Flags that make up the mask passed to sqlite3PagerAcquire(). +*/ +#define PAGER_ACQUIRE_NOCONTENT 0x01 /* Do not load data from disk */ +#define PAGER_ACQUIRE_READONLY 0x02 /* Read-only page is acceptable */ + /* ** The remainder of this file contains the declarations of the functions ** that make up the Pager sub-system API. See source code comments for @@ -102,6 +108,7 @@ void sqlite3PagerSetBusyhandler(Pager*, int(*)(void *), void *); int sqlite3PagerSetPagesize(Pager*, u32*, int); int sqlite3PagerMaxPageCount(Pager*, int); void sqlite3PagerSetCachesize(Pager*, int); +void sqlite3PagerSetMmapLimit(Pager *, sqlite3_int64); void sqlite3PagerShrink(Pager*); void sqlite3PagerSetSafetyLevel(Pager*,int,int,int); int sqlite3PagerLockingMode(Pager *, int); diff --git a/src/pcache.h b/src/pcache.h index b9135fd859..f4d4ad71c1 100644 --- a/src/pcache.h +++ b/src/pcache.h @@ -53,6 +53,8 @@ struct PgHdr { #define PGHDR_REUSE_UNLIKELY 0x010 /* A hint that reuse is unlikely */ #define PGHDR_DONT_WRITE 0x020 /* Do not write content to disk */ +#define PGHDR_MMAP 0x040 /* This is an mmap page object */ + /* Initialize and shutdown the page cache subsystem */ int sqlite3PcacheInitialize(void); void sqlite3PcacheShutdown(void); diff --git a/src/pragma.c b/src/pragma.c index f6dadbd4a8..3527c0974c 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -744,6 +744,40 @@ void sqlite3Pragma( } }else + /* + ** PRAGMA [database.]mmap_limit(N) + ** + ** Used to set mapping size limit. The mapping size limit is + ** used to limit the aggregate size of all memory mapped regions of the + ** database file. If this parameter is set to zero, then memory mapping + ** is not used at all. If N is negative, then the default memory map + ** limit determined by sqlite3_config(SQLITE_CONFIG_MMAP_LIMIT) is set. + ** The parameter N is measured in bytes. + ** + ** This value is advisory. The underlying VFS is free to memory map + ** as little or as much as it wants. Except, if N is set to 0 then the + ** upper layers will never invoke the xFetch interfaces to the VFS. + */ + if( sqlite3StrICmp(zLeft,"mmap_limit")==0 ){ + sqlite3_int64 mx; + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + if( zRight ){ + int ii; + sqlite3Atoi64(zRight, &mx, 1000, SQLITE_UTF8); + if( mx<0 ) mx = sqlite3GlobalConfig.mxMmap; + if( pId2->n==0 ) db->mxMmap = mx; + for(ii=db->nDb-1; ii>=0; ii--){ + if( db->aDb[ii].pBt && (ii==iDb || pId2->n==0) ){ + sqlite3BtreeSetMmapLimit(db->aDb[ii].pBt, mx); + } + } + } + mx = -1; + if( sqlite3_file_control(db,zDb,SQLITE_FCNTL_MMAP_LIMIT,&mx)==SQLITE_OK ){ + returnSingleInt(pParse, "mmap_limit", mx); + } + }else + /* ** PRAGMA temp_store ** PRAGMA temp_store = "default"|"memory"|"file" diff --git a/src/shell.c b/src/shell.c index c6d7fa3d53..6a0e047475 100644 --- a/src/shell.c +++ b/src/shell.c @@ -1552,6 +1552,43 @@ static int booleanValue(char *zArg){ return 0; } +/* +** Interpret zArg as an integer value, possibly with suffixes. +*/ +static sqlite3_int64 integerValue(const char *zArg){ + sqlite3_int64 v = 0; + static const struct { char *zSuffix; int iMult; } aMult[] = { + { "KiB", 1024 }, + { "MiB", 1024*1024 }, + { "GiB", 1024*1024*1024 }, + { "KB", 1000 }, + { "MB", 1000000 }, + { "GB", 1000000000 }, + { "K", 1000 }, + { "M", 1000000 }, + { "G", 1000000000 }, + }; + int i; + int isNeg = 0; + if( zArg[0]=='-' ){ + isNeg = 1; + zArg++; + }else if( zArg[0]=='+' ){ + zArg++; + } + while( isdigit(zArg[0]) ){ + v = v*10 + zArg[0] - '0'; + zArg++; + } + for(i=0; i0x7fff0000 ) szHeap = 0x7fff0000; sqlite3_config(SQLITE_CONFIG_HEAP, malloc((int)szHeap), (int)szHeap, 64); #endif @@ -3026,6 +3059,8 @@ int main(int argc, char **argv){ extern int sqlite3_multiple_initialize(const char*,int); sqlite3_multiplex_initialize(0, 1); #endif + }else if( strcmp(z,"-mmap")==0 ){ + sqlite3_config(SQLITE_CONFIG_MMAP_LIMIT, integerValue(cmdline_option_value(argc,argv,++i))); }else if( strcmp(z,"-vfs")==0 ){ sqlite3_vfs *pVfs = sqlite3_vfs_find(cmdline_option_value(argc,argv,++i)); if( pVfs ){ @@ -3111,6 +3146,8 @@ int main(int argc, char **argv){ stdin_is_interactive = 0; }else if( strcmp(z,"-heap")==0 ){ i++; + }else if( strcmp(z,"-mmap")==0 ){ + i++; }else if( strcmp(z,"-vfs")==0 ){ i++; #ifdef SQLITE_ENABLE_VFSTRACE diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 5c89f7e4e9..a4fb1a8a9a 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -470,6 +470,7 @@ int sqlite3_exec( #define SQLITE_IOERR_SHMMAP (SQLITE_IOERR | (21<<8)) #define SQLITE_IOERR_SEEK (SQLITE_IOERR | (22<<8)) #define SQLITE_IOERR_DELETE_NOENT (SQLITE_IOERR | (23<<8)) +#define SQLITE_IOERR_MMAP (SQLITE_IOERR | (24<<8)) #define SQLITE_LOCKED_SHAREDCACHE (SQLITE_LOCKED | (1<<8)) #define SQLITE_BUSY_RECOVERY (SQLITE_BUSY | (1<<8)) #define SQLITE_CANTOPEN_NOTEMPDIR (SQLITE_CANTOPEN | (1<<8)) @@ -728,6 +729,9 @@ struct sqlite3_io_methods { void (*xShmBarrier)(sqlite3_file*); int (*xShmUnmap)(sqlite3_file*, int deleteFlag); /* Methods above are valid for version 2 */ + int (*xFetch)(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp); + int (*xUnfetch)(sqlite3_file*, sqlite3_int64 iOfst, void *p); + /* Methods above are valid for version 3 */ /* Additional methods may be added in future releases */ }; @@ -882,6 +886,12 @@ struct sqlite3_io_methods { ** written into memory obtained from [sqlite3_malloc()]. The caller should ** invoke [sqlite3_free()] on the result to avoid a memory leak. ** +**
  • [[SQLITE_FCNTL_MMAP_LIMIT]] +** The argument is assumed to pointer to a value of type sqlite3_int64 that +** is an advisory maximum number of bytes in the file to memory map. The +** pointer is overwritten with the old value. The limit is not changed if +** the original value pointed to is negative. +** ** */ #define SQLITE_FCNTL_LOCKSTATE 1 @@ -900,6 +910,7 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_PRAGMA 14 #define SQLITE_FCNTL_BUSYHANDLER 15 #define SQLITE_FCNTL_TEMPFILENAME 16 +#define SQLITE_FCNTL_MMAP_LIMIT 18 /* ** CAPI3REF: Mutex Handle @@ -1628,6 +1639,19 @@ struct sqlite3_mem_methods { ** points to has just been executed. Or, if the fourth parameter is 2, then ** the connection being passed as the second parameter is being closed. The ** third parameter is passed NULL In this case. +** +** [[SQLITE_CONFIG_MMAP_LIMIT]] +**
    SQLITE_CONFIG_MMAP_LIMIT +**
    The sole argument should be a 64-bit integer (an sqlite3_int64) that +** is the default maximum number of bytes of process address space that +** should be used for accessing each database file using memory mapping. +** The default setting can be overridden by each database connection using +** either the [PRAGMA mmap_limit] command or the "mmaplimit" query parameter +** on the [URI filename] when opening the databaes file or by using the +** [SQLITE_FCNTL_MMAP_LIMIT] file control. The value set here overrides the +** compile-time default that is set using the [SQLITE_DEFAULT_MMAP_LIMIT] +** compile-time option. If the argument to this option is negative, then +** the memory map limit is set to the compile-time default. ** */ #define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ @@ -1651,6 +1675,7 @@ struct sqlite3_mem_methods { #define SQLITE_CONFIG_GETPCACHE2 19 /* sqlite3_pcache_methods2* */ #define SQLITE_CONFIG_COVERING_INDEX_SCAN 20 /* int */ #define SQLITE_CONFIG_SQLLOG 21 /* xSqllog, void* */ +#define SQLITE_CONFIG_MMAP_LIMIT 22 /* sqlite3_int64 */ /* ** CAPI3REF: Database Connection Configuration Options diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 129c4c5ea5..d894051790 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -834,6 +834,7 @@ struct sqlite3 { int nDb; /* Number of backends currently in use */ int flags; /* Miscellaneous flags. See below */ i64 lastRowid; /* ROWID of most recent insert (see above) */ + i64 mxMmap; /* Default mmap_limit setting */ unsigned int openFlags; /* Flags passed to sqlite3_vfs.xOpen() */ int errCode; /* Most recent error code (SQLITE_*) */ int errMask; /* & result codes with this before returning */ @@ -2505,6 +2506,7 @@ struct Sqlite3Config { void *pHeap; /* Heap storage space */ int nHeap; /* Size of pHeap[] */ int mnReq, mxReq; /* Min and max heap requests sizes */ + sqlite3_int64 mxMmap; /* Maximum mmap() space per open file */ void *pScratch; /* Scratch memory */ int szScratch; /* Size of each scratch buffer */ int nScratch; /* Number of scratch buffers */ diff --git a/src/sqliteLimit.h b/src/sqliteLimit.h index c7aee53ceb..d3a3b4ac95 100644 --- a/src/sqliteLimit.h +++ b/src/sqliteLimit.h @@ -206,3 +206,23 @@ #ifndef SQLITE_MAX_TRIGGER_DEPTH # define SQLITE_MAX_TRIGGER_DEPTH 1000 #endif + +/* +** Default maximum size of memory used by xFetch in the VFS. +*/ +#ifdef __APPLE__ +# include +# if TARGET_OS_IPHONE +# define SQLITE_DEFAULT_MMAP_LIMIT 0 +# endif +#endif +#ifndef SQLITE_DEFAULT_MMAP_LIMIT +# if defined(__linux__) \ + || defined(_WIN32) \ + || (defined(__APPLE__) && defined(__MACH__)) \ + || defined(__sun) +# define SQLITE_DEFAULT_MMAP_LIMIT 268435456 /* = 256*1024*1024 */ +# else +# define SQLITE_DEFAULT_MMAP_LIMIT 0 +# endif +#endif diff --git a/src/test1.c b/src/test1.c index 0bace84420..fb340f92d0 100644 --- a/src/test1.c +++ b/src/test1.c @@ -5844,6 +5844,31 @@ static int test_test_control( return TCL_OK; } +#if SQLITE_OS_UNIX +#include +#include + +static int test_getrusage( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + char buf[1024]; + struct rusage r; + memset(&r, 0, sizeof(r)); + getrusage(RUSAGE_SELF, &r); + + sprintf(buf, "ru_utime=%d.%06d ru_stime=%d.%06d ru_minflt=%d ru_majflt=%d", + (int)r.ru_utime.tv_sec, (int)r.ru_utime.tv_usec, + (int)r.ru_stime.tv_sec, (int)r.ru_stime.tv_usec, + (int)r.ru_minflt, (int)r.ru_majflt + ); + Tcl_SetObjResult(interp, Tcl_NewStringObj(buf, -1)); + return TCL_OK; +} +#endif + #if SQLITE_OS_WIN /* ** Information passed from the main thread into the windows file locker @@ -6233,6 +6258,9 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ { "print_explain_query_plan", test_print_eqp, 0 }, #endif { "sqlite3_test_control", test_test_control }, +#if SQLITE_OS_UNIX + { "getrusage", test_getrusage }, +#endif }; static int bitmask_size = sizeof(Bitmask)*8; int i; diff --git a/src/test_syscall.c b/src/test_syscall.c index d484f22db4..3d117fb2ab 100644 --- a/src/test_syscall.c +++ b/src/test_syscall.c @@ -23,7 +23,7 @@ ** ** open close access getcwd stat fstat ** ftruncate fcntl read pread pread64 write -** pwrite pwrite64 fchmod fallocate +** pwrite pwrite64 fchmod fallocate mmap ** ** test_syscall uninstall ** Uninstall all wrapper functions. @@ -81,6 +81,7 @@ /* From test1.c */ extern const char *sqlite3TestErrorName(int); +#include #include #include @@ -106,7 +107,8 @@ static int ts_pwrite(int fd, const void *aBuf, size_t nBuf, off_t off); static int ts_pwrite64(int fd, const void *aBuf, size_t nBuf, off_t off); static int ts_fchmod(int fd, mode_t mode); static int ts_fallocate(int fd, off_t off, off_t len); - +static void *ts_mmap(void *, size_t, int, int, int, off_t); +static void *ts_mremap(void*, size_t, size_t, int, ...); struct TestSyscallArray { const char *zName; @@ -131,6 +133,8 @@ struct TestSyscallArray { /* 13 */ { "pwrite64", (sqlite3_syscall_ptr)ts_pwrite64, 0, 0, 0 }, /* 14 */ { "fchmod", (sqlite3_syscall_ptr)ts_fchmod, 0, 0, 0 }, /* 15 */ { "fallocate", (sqlite3_syscall_ptr)ts_fallocate, 0, 0, 0 }, + /* 16 */ { "mmap", (sqlite3_syscall_ptr)ts_mmap, 0, 0, 0 }, + /* 17 */ { "mremap", (sqlite3_syscall_ptr)ts_mremap, 0, 0, 0 }, { 0, 0, 0, 0, 0 } }; @@ -152,6 +156,8 @@ struct TestSyscallArray { aSyscall[13].xOrig) #define orig_fchmod ((int(*)(int,mode_t))aSyscall[14].xOrig) #define orig_fallocate ((int(*)(int,off_t,off_t))aSyscall[15].xOrig) +#define orig_mmap ((void*(*)(void*,size_t,int,int,int,off_t))aSyscall[16].xOrig) +#define orig_mremap ((void*(*)(void*,size_t,size_t,int,...))aSyscall[17].xOrig) /* ** This function is called exactly once from within each invocation of a @@ -377,6 +383,31 @@ static int ts_fallocate(int fd, off_t off, off_t len){ return orig_fallocate(fd, off, len); } +static void *ts_mmap( + void *pAddr, + size_t nByte, + int prot, + int flags, + int fd, + off_t iOff +){ + if( tsIsFailErrno("mmap") ){ + return MAP_FAILED; + } + return orig_mmap(pAddr, nByte, prot, flags, fd, iOff); +} + +static void *ts_mremap(void *a, size_t b, size_t c, int d, ...){ + va_list ap; + void *pArg; + if( tsIsFailErrno("mremap") ){ + return MAP_FAILED; + } + va_start(ap, d); + pArg = va_arg(ap, void *); + return orig_mremap(a, b, c, d, pArg); +} + static int test_syscall_install( void * clientData, Tcl_Interp *interp, diff --git a/src/wal.c b/src/wal.c index 0d7271bf04..881f0a4c78 100644 --- a/src/wal.c +++ b/src/wal.c @@ -1208,7 +1208,7 @@ finished: */ if( pWal->hdr.nPage ){ sqlite3_log(SQLITE_OK, "Recovered %d frames from WAL file %s", - pWal->hdr.nPage, pWal->zWalName + pWal->hdr.mxFrame, pWal->zWalName ); } } @@ -1722,8 +1722,8 @@ static int walCheckpoint( rc = sqlite3OsSync(pWal->pWalFd, sync_flags); } - /* If the database file may grow as a result of this checkpoint, hint - ** about the eventual size of the db file to the VFS layer. + /* If the database may grow as a result of this checkpoint, hint + ** about the eventual size of the db file to the VFS layer. */ if( rc==SQLITE_OK ){ i64 nReq = ((i64)mxPage * szPage); @@ -1733,6 +1733,7 @@ static int walCheckpoint( } } + /* Iterate through the contents of the WAL, copying data to the db file. */ while( rc==SQLITE_OK && 0==walIteratorNext(pIter, &iDbpage, &iFrame) ){ i64 iOffset; @@ -2287,19 +2288,17 @@ void sqlite3WalEndReadTransaction(Wal *pWal){ } /* -** Read a page from the WAL, if it is present in the WAL and if the -** current read transaction is configured to use the WAL. +** Search the wal file for page pgno. If found, set *piRead to the frame that +** contains the page. Otherwise, if pgno is not in the wal file, set *piRead +** to zero. ** -** The *pInWal is set to 1 if the requested page is in the WAL and -** has been loaded. Or *pInWal is set to 0 if the page was not in -** the WAL and needs to be read out of the database. +** Return SQLITE_OK if successful, or an error code if an error occurs. If an +** error does occur, the final value of *piRead is undefined. */ -int sqlite3WalRead( +int sqlite3WalFindFrame( Wal *pWal, /* WAL handle */ Pgno pgno, /* Database page number to read data for */ - int *pInWal, /* OUT: True if data is read from WAL */ - int nOut, /* Size of buffer pOut in bytes */ - u8 *pOut /* Buffer to write page data to */ + u32 *piRead /* OUT: Frame number (or zero) */ ){ u32 iRead = 0; /* If !=0, WAL frame to return data from */ u32 iLast = pWal->hdr.mxFrame; /* Last page in WAL for this reader */ @@ -2315,7 +2314,7 @@ int sqlite3WalRead( ** WAL were empty. */ if( iLast==0 || pWal->readLock==0 ){ - *pInWal = 0; + *piRead = 0; return SQLITE_OK; } @@ -2386,26 +2385,31 @@ int sqlite3WalRead( } #endif - /* If iRead is non-zero, then it is the log frame number that contains the - ** required page. Read and return data from the log file. - */ - if( iRead ){ - int sz; - i64 iOffset; - sz = pWal->hdr.szPage; - sz = (sz&0xfe00) + ((sz&0x0001)<<16); - testcase( sz<=32768 ); - testcase( sz>=65536 ); - iOffset = walFrameOffset(iRead, sz) + WAL_FRAME_HDRSIZE; - *pInWal = 1; - /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL */ - return sqlite3OsRead(pWal->pWalFd, pOut, (nOut>sz ? sz : nOut), iOffset); - } - - *pInWal = 0; + *piRead = iRead; return SQLITE_OK; } +/* +** Read the contents of frame iRead from the wal file into buffer pOut +** (which is nOut bytes in size). Return SQLITE_OK if successful, or an +** error code otherwise. +*/ +int sqlite3WalReadFrame( + Wal *pWal, /* WAL handle */ + u32 iRead, /* Frame to read */ + int nOut, /* Size of buffer pOut in bytes */ + u8 *pOut /* Buffer to write page data to */ +){ + int sz; + i64 iOffset; + sz = pWal->hdr.szPage; + sz = (sz&0xfe00) + ((sz&0x0001)<<16); + testcase( sz<=32768 ); + testcase( sz>=65536 ); + iOffset = walFrameOffset(iRead, sz) + WAL_FRAME_HDRSIZE; + /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL */ + return sqlite3OsRead(pWal->pWalFd, pOut, (nOut>sz ? sz : nOut), iOffset); +} /* ** Return the size of the database in pages (or zero, if unknown). @@ -2952,6 +2956,9 @@ int sqlite3WalCheckpoint( /* Read the wal-index header. */ if( rc==SQLITE_OK ){ rc = walIndexReadHdr(pWal, &isChanged); + if( isChanged && pWal->pDbFd->pMethods->iVersion>=3 ){ + sqlite3OsUnfetch(pWal->pDbFd, 0, 0); + } } /* Copy data from the log to the database file. */ diff --git a/src/wal.h b/src/wal.h index a848de1527..ff7624af63 100644 --- a/src/wal.h +++ b/src/wal.h @@ -31,7 +31,6 @@ # define sqlite3WalClose(w,x,y,z) 0 # define sqlite3WalBeginReadTransaction(y,z) 0 # define sqlite3WalEndReadTransaction(z) -# define sqlite3WalRead(v,w,x,y,z) 0 # define sqlite3WalDbsize(y) 0 # define sqlite3WalBeginWriteTransaction(y) 0 # define sqlite3WalEndWriteTransaction(x) 0 @@ -71,7 +70,8 @@ int sqlite3WalBeginReadTransaction(Wal *pWal, int *); void sqlite3WalEndReadTransaction(Wal *pWal); /* Read a page from the write-ahead log, if it is present. */ -int sqlite3WalRead(Wal *pWal, Pgno pgno, int *pInWal, int nOut, u8 *pOut); +int sqlite3WalFindFrame(Wal *, Pgno, u32 *); +int sqlite3WalReadFrame(Wal *, u32, int, u8 *); /* If the WAL is not empty, return the size of the database. */ Pgno sqlite3WalDbsize(Wal *pWal); diff --git a/test/btreefault.test b/test/btreefault.test new file mode 100644 index 0000000000..45d326da7f --- /dev/null +++ b/test/btreefault.test @@ -0,0 +1,50 @@ +# 2013 April 02 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains fault injection tests designed to test the btree.c +# module. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/malloc_common.tcl +set testprefix btreefault + +do_test 1-pre1 { + execsql { + PRAGMA auto_vacuum = incremental; + PRAGMA journal_mode = DELETE; + CREATE TABLE t1(a PRIMARY KEY, b); + INSERT INTO t1 VALUES(randomblob(1000), randomblob(100)); + INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1; + INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1; + INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1; + INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1; + DELETE FROM t1 WHERE rowid%2; + } + faultsim_save_and_close +} {} + +do_faultsim_test 1 -prep { + faultsim_restore_and_reopen + set ::STMT [sqlite3_prepare db "SELECT * FROM t1 ORDER BY a" -1 DUMMY] + sqlite3_step $::STMT + sqlite3_step $::STMT +} -body { + execsql { PRAGMA incremental_vacuum = 10 } +} -test { + sqlite3_finalize $::STMT + faultsim_test_result {0 {}} + faultsim_integrity_check +} + +finish_test + diff --git a/test/dbstatus2.test b/test/dbstatus2.test index 18bb0870bd..c2d9164fa2 100644 --- a/test/dbstatus2.test +++ b/test/dbstatus2.test @@ -40,6 +40,7 @@ proc db_write {db {reset 0}} { do_test 1.1 { db close sqlite3 db test.db + execsql { PRAGMA mmap_limit = 0 } expr {[file size test.db] / 1024} } 6 diff --git a/test/exclusive2.test b/test/exclusive2.test index 2208da5101..d09097671d 100644 --- a/test/exclusive2.test +++ b/test/exclusive2.test @@ -25,6 +25,14 @@ ifcapable {!pager_pragmas} { return } +# Tests in this file verify that locking_mode=exclusive causes SQLite to +# use cached pages even if the database is changed on disk. This doesn't +# work with mmap. +if {[permutation]!="nommap"} { + finish_test + return +} + # This module does not work right if the cache spills at unexpected # moments. So disable the soft-heap-limit. # diff --git a/test/func.test b/test/func.test index f09ff49805..032becc94f 100644 --- a/test/func.test +++ b/test/func.test @@ -1273,11 +1273,13 @@ do_test func-29.3 { sqlite3_db_status db CACHE_MISS 1 db eval {SELECT typeof(+x) FROM t29 ORDER BY id} } {integer null real blob text} -do_test func-29.4 { - set x [lindex [sqlite3_db_status db CACHE_MISS 1] 1] - if {$x>100} {set x many} - set x -} {many} +if {[permutation] == "nommap"} { + do_test func-29.4 { + set x [lindex [sqlite3_db_status db CACHE_MISS 1] 1] + if {$x>100} {set x many} + set x + } {many} +} do_test func-29.5 { db close sqlite3 db test.db diff --git a/test/incrblob.test b/test/incrblob.test index 6634f90f9c..fa564de0cf 100644 --- a/test/incrblob.test +++ b/test/incrblob.test @@ -123,6 +123,7 @@ foreach AutoVacuumMode [list 0 1] { forcedelete test.db test.db-journal sqlite3 db test.db + execsql "PRAGMA mmap_limit = 0" execsql "PRAGMA auto_vacuum = $AutoVacuumMode" do_test incrblob-2.$AutoVacuumMode.1 { @@ -149,6 +150,7 @@ foreach AutoVacuumMode [list 0 1] { # Open and close the db to make sure the page cache is empty. db close sqlite3 db test.db + execsql "PRAGMA mmap_limit = 0" # Read the last 20 bytes of the blob via a blob handle. set ::blob [db incrblob blobs v 1] @@ -171,6 +173,7 @@ foreach AutoVacuumMode [list 0 1] { # Open and close the db to make sure the page cache is empty. db close sqlite3 db test.db + execsql "PRAGMA mmap_limit = 0" # Write the second-to-last 20 bytes of the blob via a blob handle. # @@ -200,6 +203,7 @@ foreach AutoVacuumMode [list 0 1] { # Open and close the db to make sure the page cache is empty. db close sqlite3 db test.db + execsql { PRAGMA mmap_limit = 0 } execsql { SELECT i FROM blobs } } {45} diff --git a/test/malloc_common.tcl b/test/malloc_common.tcl index 5937b958f8..2ac619b1df 100644 --- a/test/malloc_common.tcl +++ b/test/malloc_common.tcl @@ -264,7 +264,7 @@ proc faultsim_test_result_int {args} { set t [list $testrc $testresult] set r $args if { ($testnfail==0 && $t != [lindex $r 0]) || [lsearch $r $t]<0 } { - error "nfail=$testnfail rc=$testrc result=$testresult" + error "nfail=$testnfail rc=$testrc result=$testresult list=$r" } } diff --git a/test/mmap1.test b/test/mmap1.test new file mode 100644 index 0000000000..805fb76525 --- /dev/null +++ b/test/mmap1.test @@ -0,0 +1,250 @@ +# 2013 March 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +set testprefix mmap1 + +proc nRead {db} { + set bt [btree_from_db $db] + db_enter $db + array set stats [btree_pager_stats $bt] + db_leave $db + # puts [array get stats] + return $stats(read) +} + +foreach {t mmap_limit nRead c2init} { + 1.1 { PRAGMA mmap_limit = 67108864 } 4 {} + 1.2 { PRAGMA mmap_limit = 53248 } 150 {} + 1.3 { PRAGMA mmap_limit = 0 } 344 {} + 1.4 { PRAGMA mmap_limit = 67108864 } 4 {PRAGMA mmap_limit = 67108864 } + 1.5 { PRAGMA mmap_limit = 53248 } 150 {PRAGMA mmap_limit = 67108864 } + 1.6 { PRAGMA mmap_limit = 0 } 344 {PRAGMA mmap_limit = 67108864 } +} { + do_multiclient_test tn { + sql1 {PRAGMA page_size=1024} + sql1 $mmap_limit + sql2 $c2init + + code2 { + set ::rcnt 0 + proc rblob {n} { + set ::rcnt [expr (($::rcnt << 3) + $::rcnt + 456) & 0xFFFFFFFF] + set str [format %.8x [expr $::rcnt ^ 0xbdf20da3]] + string range [string repeat $str [expr $n/4]] 1 $n + } + db2 func rblob rblob + } + + sql2 { + PRAGMA page_size=1024; + PRAGMA auto_vacuum = 1; + CREATE TABLE t1(a, b, UNIQUE(a, b)); + INSERT INTO t1 VALUES(rblob(500), rblob(500)); + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 2 + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 4 + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 8 + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 16 + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 32 + } + do_test $t.$tn.1 { + sql1 "SELECT count(*) FROM t1; PRAGMA integrity_check ; PRAGMA page_count" + } {32 ok 77} + + # Have connection 2 shrink the file. Check connection 1 can still read it. + sql2 { DELETE FROM t1 WHERE rowid%2; } + do_test $t.$tn.2 { + sql1 "SELECT count(*) FROM t1; PRAGMA integrity_check ; PRAGMA page_count" + } {16 ok 42} + + # Have connection 2 grow the file. Check connection 1 can still read it. + sql2 { INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1 } + do_test $t.$tn.3 { + sql1 "SELECT count(*) FROM t1; PRAGMA integrity_check ; PRAGMA page_count" + } {32 ok 79} + + # Have connection 2 grow the file again. Check connection 1 is still ok. + sql2 { INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1 } + do_test $t.$tn.4 { + sql1 "SELECT count(*) FROM t1; PRAGMA integrity_check ; PRAGMA page_count" + } {64 ok 149} + + # Check that the number of pages read by connection 1 indicates that the + # "PRAGMA mmap_limit" command worked. + do_test $t.$tn.5 { nRead db } $nRead + } +} + +set ::rcnt 0 +proc rblob {n} { + set ::rcnt [expr (($::rcnt << 3) + $::rcnt + 456) & 0xFFFFFFFF] + set str [format %.8x [expr $::rcnt ^ 0xbdf20da3]] + string range [string repeat $str [expr $n/4]] 1 $n +} + +reset_db +db func rblob rblob + +do_execsql_test 2.1 { + PRAGMA auto_vacuum = 1; + PRAGMA mmap_limit = 67108864; + PRAGMA journal_mode = wal; + CREATE TABLE t1(a, b, UNIQUE(a, b)); + INSERT INTO t1 VALUES(rblob(500), rblob(500)); + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 2 + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 4 + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 8 + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 16 + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 32 + PRAGMA wal_checkpoint; +} {67108864 wal 0 103 103} + +do_execsql_test 2.2 { + PRAGMA auto_vacuum; + SELECT count(*) FROM t1; +} {1 32} + +do_test 2.3 { + sqlite3 db2 test.db + db2 func rblob rblob + db2 eval { + DELETE FROM t1 WHERE (rowid%4); + PRAGMA wal_checkpoint; + } + db2 eval { + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 16 + SELECT count(*) FROM t1; + } +} {16} + +do_execsql_test 2.4 { + PRAGMA wal_checkpoint; +} {0 24 24} + +db2 close +reset_db +db func rblob rblob +do_execsql_test 3.1 { + PRAGMA auto_vacuum = 1; + + CREATE TABLE t1(a, b, UNIQUE(a, b)); + INSERT INTO t1 VALUES(rblob(500), rblob(500)); + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 2 + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 4 + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 8 + + CREATE TABLE t2(a, b, UNIQUE(a, b)); + INSERT INTO t2 SELECT * FROM t1; +} {} + +do_test 3.2 { + set nRow 0 + db eval {SELECT * FROM t2 ORDER BY a, b} { + if {$nRow==4} { db eval { DELETE FROM t1 } } + incr nRow + } + set nRow +} {8} + +#------------------------------------------------------------------------- +# Ensure that existing cursors using xFetch() pages see changes made +# to rows using the incrblob API. +# +reset_db +set aaa [string repeat a 400] +set bbb [string repeat b 400] +set ccc [string repeat c 400] +set ddd [string repeat d 400] +set eee [string repeat e 400] + +do_execsql_test 4.1 { + PRAGMA page_size = 1024; + CREATE TABLE t1(x); + INSERT INTO t1 VALUES($aaa); + INSERT INTO t1 VALUES($bbb); + INSERT INTO t1 VALUES($ccc); + INSERT INTO t1 VALUES($ddd); + SELECT * FROM t1; + BEGIN; +} [list $aaa $bbb $ccc $ddd] + +do_test 4.2 { + set ::STMT [sqlite3_prepare db "SELECT * FROM t1 ORDER BY rowid" -1 dummy] + sqlite3_step $::STMT + sqlite3_column_text $::STMT 0 +} $aaa + +do_test 4.3 { + foreach r {2 3 4} { + set fd [db incrblob t1 x $r] + puts -nonewline $fd $eee + close $fd + } + + set res [list] + while {"SQLITE_ROW" == [sqlite3_step $::STMT]} { + lappend res [sqlite3_column_text $::STMT 0] + } + set res +} [list $eee $eee $eee] + +do_test 4.4 { + sqlite3_finalize $::STMT +} SQLITE_OK + +do_execsql_test 4.5 { COMMIT } + +#------------------------------------------------------------------------- +# Ensure that existing cursors holding xFetch() references are not +# confused if those pages are moved to make way for the root page of a +# new table or index. +# +reset_db +do_execsql_test 5.1 { + PRAGMA auto_vacuum = 2; + PRAGMA page_size = 1024; + CREATE TABLE t1(x); + INSERT INTO t1 VALUES($aaa); + INSERT INTO t1 VALUES($bbb); + INSERT INTO t1 VALUES($ccc); + INSERT INTO t1 VALUES($ddd); + + PRAGMA auto_vacuum; + SELECT * FROM t1; +} [list 2 $aaa $bbb $ccc $ddd] + +do_test 5.2 { + set ::STMT [sqlite3_prepare db "SELECT * FROM t1 ORDER BY rowid" -1 dummy] + sqlite3_step $::STMT + sqlite3_column_text $::STMT 0 +} $aaa + +do_execsql_test 5.3 { + CREATE TABLE t2(x); + INSERT INTO t2 VALUES('tricked you!'); + INSERT INTO t2 VALUES('tricked you!'); +} + +do_test 5.4 { + sqlite3_step $::STMT + sqlite3_column_text $::STMT 0 +} $bbb + +do_test 5.5 { + sqlite3_finalize $::STMT +} SQLITE_OK + +finish_test + + diff --git a/test/mmap2.test b/test/mmap2.test new file mode 100644 index 0000000000..33830a6943 --- /dev/null +++ b/test/mmap2.test @@ -0,0 +1,82 @@ +# 2013 March 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file tests the effect of the mmap() or mremap() system calls +# returning an error on the library. +# +# If either mmap() or mremap() fails, SQLite should log an error +# message, then continue accessing the database using read() and +# write() exclusively. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix mmap2 + +if {$::tcl_platform(platform)!="unix" || [test_syscall defaultvfs] != "unix"} { + finish_test + return +} + +db close +sqlite3_shutdown +test_sqlite3_log xLog +proc xLog {error_code msg} { + if {[string match os_unix.c* $msg]} { + lappend ::log $msg + } +} + +foreach syscall {mmap mremap} { + test_syscall uninstall + if {[catch {test_syscall install $syscall}]} continue + + for {set i 1} {$i < 20} {incr i} { + reset_db + + test_syscall fault $i 1 + test_syscall errno $syscall ENOMEM + set ::log "" + + do_execsql_test 1.$syscall.$i.1 { + CREATE TABLE t1(a, b, UNIQUE(a, b)); + INSERT INTO t1 VALUES(randomblob(1000), randomblob(1000)); + INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1; + INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1; + INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1; + INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1; + INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1; + INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1; + } + + set nFail [test_syscall fault 0 0] + + do_execsql_test 1.$syscall.$i.2 { + SELECT count(*) FROM t1; + PRAGMA integrity_check; + } {64 ok} + + do_test 1.$syscall.$i.3 { + expr {$nFail==0 || $nFail==1} + } {1} + + do_test 1.$syscall.$i.4.nFail=$nFail { + regexp ".*${syscall}.*" $::log + } [expr $nFail>0] + } +} + +db close +test_syscall uninstall +sqlite3_shutdown +test_sqlite3_log +sqlite3_initialize +finish_test diff --git a/test/pagerfault.test b/test/pagerfault.test index e04e97e4fc..9ea1aa6328 100644 --- a/test/pagerfault.test +++ b/test/pagerfault.test @@ -1246,5 +1246,62 @@ do_faultsim_test pagerfault-27 -faults ioerr-persistent -prep { faultsim_integrity_check } +#------------------------------------------------------------------------- +# +do_test pagerfault-28-pre { + faultsim_delete_and_reopen + db func a_string a_string + execsql { + PRAGMA page_size = 512; + + PRAGMA journal_mode = wal; + PRAGMA wal_autocheckpoint = 0; + PRAGMA cache_size = 100000; + + BEGIN; + CREATE TABLE t2(a UNIQUE, b UNIQUE); + INSERT INTO t2 VALUES( a_string(800), a_string(800) ); + INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2; + INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2; + INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2; + INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2; + INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2; + INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2; + INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2; + INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2; + INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2; + INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2; + INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2; + COMMIT; + CREATE TABLE t1(a PRIMARY KEY, b); + } + expr {[file size test.db-shm] >= 96*1024} +} {1} +faultsim_save_and_close + +do_faultsim_test pagerfault-28 -faults oom* -prep { + faultsim_restore_and_reopen + execsql { PRAGMA mmap_limit=0 } + + sqlite3 db2 test.db + db2 eval { SELECT count(*) FROM t2 } + + db func a_string a_string + execsql { + BEGIN; + INSERT INTO t1 VALUES(a_string(2000), a_string(2000)); + INSERT INTO t1 VALUES(a_string(2000), a_string(2000)); + } + set ::STMT [sqlite3_prepare db "SELECT * FROM t1 ORDER BY a" -1 DUMMY] + sqlite3_step $::STMT +} -body { + execsql { ROLLBACK } +} -test { + db2 close + sqlite3_finalize $::STMT + catchsql { ROLLBACK } + faultsim_integrity_check +} + finish_test diff --git a/test/pageropt.test b/test/pageropt.test index 82311965a5..10d3d4917e 100644 --- a/test/pageropt.test +++ b/test/pageropt.test @@ -92,7 +92,7 @@ do_test pageropt-1.5 { pagercount_sql { SELECT hex(x) FROM t1 } -} [list 6 0 0 $blobcontent] +} [list [expr {[permutation]=="nommap" ? 6 : 1}] 0 0 $blobcontent] do_test pageropt-1.6 { pagercount_sql { SELECT hex(x) FROM t1 diff --git a/test/permutations.test b/test/permutations.test index 711d4e57d3..3aba9d9b38 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -138,6 +138,14 @@ test_suite "veryquick" -prefix "" -description { test_set $allquicktests -exclude *malloc* *ioerr* *fault* ] +test_suite "nommap" -prefix "nomm-" -description { + Similar to veryquick. Except with memory mapping disabled. +} -presql { + pragma mmap_limit = 0; +} -files [ + test_set $allquicktests -exclude *malloc* *ioerr* *fault* -include malloc.test +] + test_suite "valgrind" -prefix "" -description { Run the "veryquick" test suite with a couple of multi-process tests (that fail under valgrind) omitted. diff --git a/test/speed1p.test b/test/speed1p.test index 915f165354..6bf7b10e1f 100644 --- a/test/speed1p.test +++ b/test/speed1p.test @@ -24,6 +24,8 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl speed_trial_init speed1 +sqlite3_memdebug_vfs_oom_test 0 + # Set a uniform random seed expr srand(0) @@ -78,7 +80,6 @@ do_test speed1p-1.0 { } } {i2a i2b t1 t2} - # 50000 INSERTs on an unindexed table # set list {} diff --git a/test/syscall.test b/test/syscall.test index d841a9a0b2..5bf1225867 100644 --- a/test/syscall.test +++ b/test/syscall.test @@ -60,7 +60,7 @@ foreach s { open close access getcwd stat fstat ftruncate fcntl read pread write pwrite fchmod fallocate pread64 pwrite64 unlink openDirectory mkdir rmdir - statvfs fchown umask + statvfs fchown umask mmap munmap mremap } { if {[test_syscall exists $s]} {lappend syscall_list $s} } diff --git a/test/sysfault.test b/test/sysfault.test index 07d525ca9e..4ecaa1fabc 100644 --- a/test/sysfault.test +++ b/test/sysfault.test @@ -243,5 +243,35 @@ do_faultsim_test 3 -faults vfsfault-* -prep { faultsim_test_result {0 20000} } -finish_test +#------------------------------------------------------------------------- +# Test errors in mmap(). +# +proc vfsfault_install {} { + test_syscall reset + test_syscall install {mmap} +} +faultsim_delete_and_reopen +execsql { + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); +} +faultsim_save_and_close + +do_faultsim_test 4 -faults vfsfault-* -prep { + faultsim_restore_and_reopen + file_control_chunksize_test db main 8192 + execsql { + PRAGMA mmap_limit = 1000000; + } +} -body { + test_syscall errno mmap EACCES + + execsql { + SELECT * FROM t1; + } +} -test { + faultsim_test_result {0 {1 2}} {1 {disk I/O error}} +} + +finish_test diff --git a/test/wal.test b/test/wal.test index 24ce5f86a3..f833b6ab1a 100644 --- a/test/wal.test +++ b/test/wal.test @@ -727,6 +727,8 @@ do_test wal-11.9 { list [expr [file size test.db]/1024] [log_deleted test.db-wal] } {37 1} sqlite3_wal db test.db +set nWal 39 +if {[permutation]=="nommap"} {set nWal 37} do_test wal-11.10 { execsql { PRAGMA cache_size = 10; @@ -735,7 +737,7 @@ do_test wal-11.10 { SELECT count(*) FROM t1; } list [expr [file size test.db]/1024] [file size test.db-wal] -} [list 37 [wal_file_size 37 1024]] +} [list 37 [wal_file_size $nWal 1024]] do_test wal-11.11 { execsql { SELECT count(*) FROM t1; @@ -745,7 +747,7 @@ do_test wal-11.11 { } {32 16} do_test wal-11.12 { list [expr [file size test.db]/1024] [file size test.db-wal] -} [list 37 [wal_file_size 37 1024]] +} [list 37 [wal_file_size $nWal 1024]] do_test wal-11.13 { execsql { INSERT INTO t1 VALUES( blob(900) ); @@ -755,7 +757,7 @@ do_test wal-11.13 { } {17 ok} do_test wal-11.14 { list [expr [file size test.db]/1024] [file size test.db-wal] -} [list 37 [wal_file_size 37 1024]] +} [list 37 [wal_file_size $nWal 1024]] #------------------------------------------------------------------------- @@ -1567,6 +1569,26 @@ ifcapable autovacuum { } [wal_file_size 1 1024] } +reset_db +do_test 25 { + sqlite3 db test.db + + execsql { + CREATE TABLE t1(x, y); + PRAGMA journal_mode = WAL; + INSERT INTO t1 VALUES(1, 2); + } + + execsql { + BEGIN; + CREATE TABLE t2(a, b); + } + + hexio_write test.db-shm [expr 16*1024] [string repeat 0055 8192] + catchsql ROLLBACK +} {0 {}} + + db close sqlite3_shutdown test_sqlite3_log diff --git a/test/wal5.test b/test/wal5.test index 6eceed5e59..ebd1dbfc31 100644 --- a/test/wal5.test +++ b/test/wal5.test @@ -235,7 +235,15 @@ foreach {testprefix do_wal_checkpoint} { do_test 2.3.$tn.5 { sql1 { INSERT INTO t2 VALUES(3, 4) } } {} do_test 2.3.$tn.6 { file_page_counts } {1 4 1 4} do_test 2.3.$tn.7 { code1 { do_wal_checkpoint db -mode full } } {1 4 3} - do_test 2.3.$tn.8 { file_page_counts } {1 4 2 4} + + # The checkpoint above only writes page 1 of the db file. The other + # page (page 2) is locked by the read-transaction opened by the + # [sql2] commmand above. So normally, the db is 1 page in size here. + # However, in mmap() mode, the db is pre-allocated to 2 pages at the + # start of the checkpoint, even though page 2 cannot be written. + set nDb 2 + if {[permutation]=="no-mmap"} {set nDb 1} + do_test 2.3.$tn.8 { file_page_counts } [list $nDb 4 2 4] } # Check that checkpoints block on the correct locks. And respond correctly @@ -343,4 +351,3 @@ foreach {testprefix do_wal_checkpoint} { finish_test -