diff --git a/Makefile.in b/Makefile.in index 894b352d09..efe744d209 100644 --- a/Makefile.in +++ b/Makefile.in @@ -945,6 +945,7 @@ clean: rm -f sqlite3.dll sqlite3.lib sqlite3.exp sqlite3.def rm -f sqlite3.c rm -f sqlite3_analyzer$(TEXE) sqlite3_analyzer.c + rm -f sqlite-output.vsix distclean: clean rm -f config.log config.status libtool Makefile sqlite3.pc diff --git a/Makefile.msc b/Makefile.msc index 33cde181c5..7f1638a31a 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -30,6 +30,11 @@ USE_NATIVE_LIBPATHS = 0 # FOR_WINRT = 0 +# Set this non-0 to skip attempting to look for and/or link with the Tcl +# runtime library. +# +NO_TCL = 0 + # Set this to non-0 to create and use PDBs. # SYMBOLS = 1 @@ -75,7 +80,9 @@ LD = link.exe # "NCC=""%VCINSTALLDIR%\bin\cl.exe""" # USE_NATIVE_LIBPATHS=1 # -!IFNDEF NCC +!IFDEF NCC +NCC = $(NCC:\\=\) +!ELSE NCC = $(CC) !ENDIF @@ -87,6 +94,8 @@ NCC = $(CC) NCRTLIBPATH = $(VCINSTALLDIR)\lib !ENDIF +NCRTLIBPATH = $(NCRTLIBPATH:\\=\) + # Check for the Platform SDK library path macro. Othertise, this # value will default to the 'lib' directory underneath the Windows # SDK installation directory (the environment variable used appears @@ -97,6 +106,8 @@ NCRTLIBPATH = $(VCINSTALLDIR)\lib NSDKLIBPATH = $(WINDOWSSDKDIR)\lib !ENDIF +NSDKLIBPATH = $(NSDKLIBPATH:\\=\) + # C compiler and options for use in building executables that # will run on the platform that is doing the build. # @@ -123,6 +134,7 @@ TCC = $(CC) -W3 -DSQLITE_OS_WIN=1 -I. -I$(TOP)\src -fp:precise # !IF $(FOR_WINRT)!=0 TCC = $(TCC) -DSQLITE_OS_WINRT=1 +TCC = $(TCC) -DWINAPI_FAMILY=WINAPI_PARTITION_APP !ENDIF # Also, we need to dynamically link to the correct MSVC runtime @@ -344,8 +356,10 @@ LDFLAGS = /DEBUG !ENDIF # Start with the Tcl related linker options. +!IF $(NO_TCL)==0 LTLIBPATHS = /LIBPATH:$(TCLLIBDIR) LTLIBS = $(LIBTCL) +!ENDIF # If ICU support is enabled, add the linker options for it. !IF $(USE_ICU)!=0 @@ -1116,6 +1130,7 @@ clean: del /Q sqlite3.dll sqlite3.lib sqlite3.exp sqlite3.def del /Q sqlite3.c del /Q sqlite3_analyzer.exe sqlite3_analyzer.exp sqlite3_analyzer.c + del /Q sqlite-output.vsix # # Windows section diff --git a/configure b/configure index e1fd1686b8..af10fe3bd1 100755 --- a/configure +++ b/configure @@ -12778,14 +12778,16 @@ $as_echo "$as_me: error: ${with_tclconfig} directory doesn't contain tclConfig.s fi # Start autosearch by asking tclsh - if test x"$cross_compiling" = xno; then - for i in `echo 'puts stdout $auto_path' | ${TCLSH_CMD}` - do - if test -f "$i/tclConfig.sh" ; then - ac_cv_c_tclconfig="$i" - break - fi - done + if test x"${ac_cv_c_tclconfig}" = x ; then + if test x"$cross_compiling" = xno; then + for i in `echo 'puts stdout $auto_path' | ${TCLSH_CMD}` + do + if test -f "$i/tclConfig.sh" ; then + ac_cv_c_tclconfig="$i" + break + fi + done + fi fi # then check for a private Tcl installation diff --git a/configure.ac b/configure.ac index aac1d472f2..882c3ccf92 100644 --- a/configure.ac +++ b/configure.ac @@ -388,14 +388,16 @@ if test "${use_tcl}" = "yes" ; then fi # Start autosearch by asking tclsh - if test x"$cross_compiling" = xno; then - for i in `echo 'puts stdout $auto_path' | ${TCLSH_CMD}` - do - if test -f "$i/tclConfig.sh" ; then - ac_cv_c_tclconfig="$i" - break - fi - done + if test x"${ac_cv_c_tclconfig}" = x ; then + if test x"$cross_compiling" = xno; then + for i in `echo 'puts stdout $auto_path' | ${TCLSH_CMD}` + do + if test -f "$i/tclConfig.sh" ; then + ac_cv_c_tclconfig="$i" + break + fi + done + fi fi # then check for a private Tcl installation diff --git a/main.mk b/main.mk index 7a7be4b9db..a38208b663 100644 --- a/main.mk +++ b/main.mk @@ -617,3 +617,4 @@ clean: rm -f threadtest3 threadtest3.exe rm -f sqlite3.c fts?amal.c tclsqlite3.c rm -f sqlite3_analyzer sqlite3_analyzer.exe sqlite3_analyzer.c + rm -f sqlite-output.vsix diff --git a/manifest b/manifest index e4cf469b13..8609fc7751 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Add\san\sassert()\sto\shelp\sverify\sthe\sreturn\scode\sfrom\sthe\sWin32\sthread\swait\sfunction. -D 2012-07-23T06:47:30.102 +C Update\sthe\sthreads\sbranch\sto\sinclude\sall\sthe\slatest\strunk\schanges. +D 2012-08-16T11:24:22.980 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f -F Makefile.in 7a89f9692d1369faa4071310164ffba0504c324d +F Makefile.in adec39f15a9c7000f634b87a535b95279b0cbd09 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 -F Makefile.msc 332c166048570b65127db33450bca22c53b4f0a2 +F Makefile.msc abd0ffddd856f9251a563cfa5e430a046d73e359 F Makefile.vxworks 879f034a64062a364b21000266bbd5bc6e0c19b9 F README cd04a36fbc7ea56932a4052d7d0b7f09f27c33d6 F VERSION a71848df48082f1d6585d4b0819d530fc455485d @@ -15,8 +15,8 @@ F art/sqlite370.jpg d512473dae7e378a67e28ff96a34da7cb331def2 F config.guess 226d9a188c6196f3033ffc651cbc9dcee1a42977 F config.h.in 0921066a13130082764ab4ab6456f7b5bebe56de F config.sub 9ebe4c3b3dab6431ece34f16828b594fb420da55 -F configure 8f973617dbc3fca46d26104070b75c17ef70beb7 x -F configure.ac 9ee886c21c095b3272137b1553ae416c8b8c8557 +F configure e2d0e3b67d2b1b1049d389fd671275d79bb80457 x +F configure.ac 6e909664785b8184db2179013cd9d574f96ca3a3 F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad F doc/lemon.html 3091574143dd3415669b6745843ff8d011d33549 F doc/pager-invariants.txt 870107036470d7c419e93768676fae2f8749cf9e @@ -103,7 +103,7 @@ F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 -F main.mk dd63e2de89cdcb9be441e8f046a0acce3fbae2a8 +F main.mk a463acdf2898fed3ccd716ece9d3c5d28450c852 F mkdll.sh 7d09b23c05d56532e9d44a50868eb4b12ff4f74a F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f F mkextw.sh 4123480947681d9b434a5e7b1ee08135abe409ac @@ -117,30 +117,30 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 6be1ad09113570e1fc8dcaff84c9b0b337db5ffc F sqlite3.pc.in ae6f59a76e862f5c561eb32a380228a02afc3cad F src/alter.c 149cc80d9257971b0bff34e58fb2263e01998289 -F src/analyze.c 70c46504c0d2543ea5cdca01140b2cd3e1d886e7 +F src/analyze.c 7553068d21e32a57fc33ab6b2393fc8c1ba41410 F src/attach.c 577bf5675b0c50495fc28549f2fcbdb1bac71143 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 -F src/backup.c 5524df36810668b3b5a6de2d7e3910b98e721e33 -F src/bitvec.c af50f1c8c0ff54d6bdb7a80e2fceca5a93670bef +F src/backup.c 5b31b24d6814b11de763debf342c8cd0a15a4910 +F src/bitvec.c 26675fe8e431dc555e6f2d0e11e651d172234aa1 F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c f0b71054103cb77eb5e782088c16998ec4f06624 -F src/btree.h 48a013f8964f12d944d90e4700df47b72dd6d923 -F src/btreeInt.h 38a639c0542c29fe8331a221c4aed0cb8686249e -F src/build.c 47c4506afe4bcb4ed1f4b5357582d1cb3402f8ad +F src/btree.c 82b6fcbec3101ff951f47797f407d5eb5d06fa44 +F src/btree.h 4aee02e879211bfcfd3f551769578d2e940ab6c2 +F src/btreeInt.h 4e5c2bd0f9b36b2a815a6d84f771a61a65830621 +F src/build.c 0f6b40ad6211dcaba6159d0f9a297f0704f22142 F src/callback.c 0cb4228cdcd827dcc5def98fb099edcc9142dbcd F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 500d019da966631ad957c37705642be87524463b F src/date.c 067a81c9942c497aafd2c260e13add8a7d0c7dd4 F src/delete.c 4c20ea4f6213b3bc1c6a510586864b679946e05e -F src/expr.c 06a7733d19dc725dc46ba51afd9feadb4b85d991 +F src/expr.c e2927abf9c69ce4ff9a931bd201946961c34819a F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 657212460bf5cfd3ae607d12ea62092844c227b5 F src/func.c 18dfedfb857e100b05755a1b12e88b389f957879 F src/global.c 4cfdca5cb0edd33c4d021baec4ede958cb2c793b -F src/hash.c 458488dcc159c301b8e7686280ab209f1fb915af +F src/hash.c a4031441741932da9e7a65bee2b36b5d0e81c073 F src/hash.h 2894c932d84d9f892d4b4023a75e501f83050970 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 -F src/insert.c 0bbffe75c254c62a5686ab5e7f88e29235e16174 +F src/insert.c 770ed633830fb49d73d90c3fdf20b703973e1e84 F src/journal.c 552839e54d1bf76fb8f7abe51868b66acacf6a0e F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c 0ee69fca0be54cd93939df98d2aca4ca46f44416 @@ -160,27 +160,27 @@ F src/mutex_unix.c c3a4e00f96ba068a8dbef34084465979aaf369cc F src/mutex_w32.c 32a9b3841e2d757355f0012b860b1bc5e01eafa0 F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c e1acdc09ff3ac2412945cca9766e2dcf4675f31c -F src/os.h c2ebd26a68a4223fe170b003852b97d9e7211498 +F src/os.h 027491c77d2404c0a678bb3fb06286f331eb9b57 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c d7c96b5d140f550f07345870112fae5d7ef99757 -F src/os_win.c f9f2a4569f9a6d2415170261641e6e3ee6ed8121 +F src/os_win.c 904f85152f806084826f5e6d7aae05d3cf4dc5d1 F src/pager.c e381c118b77dc22021a1a59d3fec24815e91df78 F src/pager.h 8b8c9bc065a3c66769df8724dfdf492ee1aab3c5 F src/parse.y f29df90bd3adc64b33114ab1de9fb7768fcf2099 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h 1b5dcc3dc8103d03e625b177023ee67764fa6b7c -F src/pcache1.c 2234d84f9c003d800a57f00f8535c91667fa4f6c +F src/pcache1.c 9fd22671c270b35131ef480bbc00392b8b5f8ab9 F src/pragma.c 97f9357f0e7e5fb46a2519f14539550aa07db49f F src/prepare.c 33291b83cca285718048d219c67b8298501fa3a5 F src/printf.c 4a9f882f1c1787a8b494a2987765acf9d97ac21f F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 F src/resolve.c b3c70ab28cac60de33684c9aa9e5138dcf71d6dd F src/rowset.c f6a49f3e9579428024662f6e2931832511f831a1 -F src/select.c f6c4833c4d8e94714761d99013d74f381e084f1d +F src/select.c a365da6d7a6d7d8a10ad60ca71837ab5e9369466 F src/shell.c 076e1c90d594644f36027c8ecff9a392cf2d3a06 -F src/sqlite.h.in 310ae7e538883fa1619ab0638c775ce11ad43015 +F src/sqlite.h.in 3e8035bc406b1571a5cc8ea46bcc831201676f1a F src/sqlite3ext.h 6904f4aadf976f95241311fbffb00823075d9477 -F src/sqliteInt.h 4bb99c73090cde71ea5534d0f9c3ee299a1c10e7 +F src/sqliteInt.h 1ef5f8b64d204fe5deec83c58297493b2f5fbfa0 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 35939e7e03abf1b7577ce311f48f682c40de3208 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -221,13 +221,13 @@ F src/test_quota.h 8761e463b25e75ebc078bd67d70e39b9c817a0cb F src/test_rtree.c aba603c949766c4193f1068b91c787f57274e0d9 F src/test_schema.c 8c06ef9ddb240c7a0fcd31bc221a6a2aade58bf0 F src/test_server.c 2f99eb2837dfa06a4aacf24af24c6affdf66a84f -F src/test_spellfix.c 5ed989693d4040f4d343316c338e25c5a6a1f05d +F src/test_spellfix.c 3a260d237fabbf5884389aa8c0e516b4e61ab98a F src/test_stat.c d1569c7a4839f13e80187e2c26b2ab4da2d03935 F src/test_superlock.c 2b97936ca127d13962c3605dbc9a4ef269c424cd F src/test_syscall.c a992d8c80ea91fbf21fb2dd570db40e77dd7e6ae F src/test_tclvar.c f4dc67d5f780707210d6bb0eb6016a431c04c7fa F src/test_thread.c e286f2173563f2a1747c24bcda6b9d030bf4f4e4 -F src/test_vfs.c da6d0d982b11756c94c1760196355d33d03ff745 +F src/test_vfs.c c6260ef238c1142c8f8bd402db02216afd182ae3 F src/test_vfstrace.c 6b28adb2a0e8ecd0f2e3581482e1f658b11b4067 F src/test_wholenumber.c 3d2b9ed1505c40ad5c5ca2ad16ae7a289d6cc251 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 @@ -238,15 +238,15 @@ F src/update.c d3076782c887c10e882996550345da9c4c9f9dea F src/utf.c 890c67dcfcc7a74623c95baac7535aadfe265e84 F src/util.c 0af2e515dc0dabacec931bca39525f6c3f1c5455 F src/vacuum.c 587a52bb8833d7ac15af8916f25437e2575028bd -F src/vdbe.c f5ad3c06dc3fe647097065829c013f3f1b9eadca +F src/vdbe.c 75da79cdcd58481825a06f045bc2f5ea3966eeae F src/vdbe.h 18f581cac1f4339ec3299f3e0cc6e11aec654cdb -F src/vdbeInt.h 6ff4180a05683566a8835d12f7ec504b22932c82 +F src/vdbeInt.h 986b6b11a13c517337355009e5438703ba5b0a40 F src/vdbeapi.c 88ea823bbcb4320f5a6607f39cd7c2d3cc4c26b1 F src/vdbeaux.c dce80038c3c41f2680e5ab4dd0f7e0d8b7ff9071 F src/vdbeblob.c 32f2a4899d67f69634ea4dd93e3f651936d732cb F src/vdbemem.c cb55e84b8e2c15704968ee05f0fae25883299b74 -F src/vdbesort.c b25814d385895544ebc8118245c8311ded7f81c9 -F src/vdbetrace.c 79059ebd17b3c8545fab2a24253713e77e4ab392 +F src/vdbesort.c 0dc1b274dcb4d4c8e71b0b2b15261f286caba39b +F src/vdbetrace.c 8bd5da325fc90f28464335e4cc4ad1407fe30835 F src/vtab.c bb8ea3a26608bb1357538a5d2fc72beba6638998 F src/wal.c 9294df6f96aae5909ae1a9b733fd1e1b4736978b F src/wal.h 29c197540b19044e6cd73487017e5e47a1d3dac6 @@ -532,6 +532,7 @@ F test/index.test b5429732b3b983fa810e3ac867d7ca85dae35097 F test/index2.test ee83c6b5e3173a3d7137140d945d9a5d4fdfb9d6 F test/index3.test 423a25c789fc8cc51aaf2a4370bbdde2d9e9eed7 F test/index4.test 2983216eb8c86ee62d9ed7cb206b5cc3331c0026 +F test/index5.test edc8c64ca78bee140c21ce3836820fadf47906bb F test/indexedby.test be501e381b82b2f8ab406309ba7aac46e221f4ad F test/indexfault.test 31d4ab9a7d2f6e9616933eb079722362a883eb1d F test/init.test 15c823093fdabbf7b531fe22cf037134d09587a7 @@ -642,7 +643,7 @@ F test/pageropt.test 9191867ed19a2b3db6c42d1b36b6fbc657cd1ab0 F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16 F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 -F test/permutations.test 2af90e00cea9e7e7c0a6b16d34727cb5bbae14dd +F test/permutations.test d12fabf8abdb71b79eb6c3ef3be5e875fe790071 F test/pragma.test a62f73293b0f0d79b0c87f8dd32d46fe53b0bd17 F test/pragma2.test 3a55f82b954242c642f8342b17dffc8b47472947 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 @@ -676,6 +677,7 @@ F test/schema3.test 1bc1008e1f8cb5654b248c55f27249366eb7ed38 F test/schema4.test e6a66e20cc69f0e306667c08be7fda3d11707dc5 F test/schema5.test 0103e4c0313b3725b5ae5600bdca53006ab53db3 F test/securedel.test 87a2561151af1f1e349071a89fdd77059f50113c +F test/securedel2.test f13a916155f790a6b9de835049641b14ef312986 F test/select1.test deba017eed9daa5af33de868676c997e7eebb931 F test/select2.test 352480e0e9c66eda9c3044e412abdf5be0215b56 F test/select3.test 2ce595f8fb8e2ac10071d3b4e424cadd4634a054 @@ -717,7 +719,7 @@ F test/speed3.test d32043614c08c53eafdc80f33191d5bd9b920523 F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa F test/speed4p.test 0e51908951677de5a969b723e03a27a1c45db38b -F test/spellfix.test 936be6f7ba1c4d096adb280c68b32f4848af8d2e +F test/spellfix.test 4e339920585e7555660bd3b11cf338af82c656ae F test/sqllimits1.test b1aae27cc98eceb845e7f7adf918561256e31298 F test/stat.test 08e8185b3fd5b010c90d7ad82b9dd4ea1cbf14b0 F test/stmt.test 25d64e3dbf9a3ce89558667d7f39d966fe2a71b9 @@ -946,10 +948,10 @@ F test/walhook.test ed00a40ba7255da22d6b66433ab61fab16a63483 F test/walmode.test 4022fe03ae6e830583672caa101f046438a0473c F test/walnoshm.test 84ca10c544632a756467336b7c3b864d493ee496 F test/walpersist.test 8c6b7e3ec1ba91b5e4dc4e0921d6d3f87cd356a6 -F test/walro.test f183c0003060a64793f9bfafef4c479418ed7fa8 +F test/walro.test a31deb621033442a76c3a61e44929250d06f81b1 F test/walshared.test 6dda2293880c300baf5d791c307f653094585761 F test/walslow.test e7be6d9888f83aa5d3d3c7c08aa9b5c28b93609a -F test/walthread.test a2ed5270eb695284d4ad27d252517bdc3317ee2a +F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e F test/where.test 4c9f69987ed2aa0173fa930f2b41ab9879478cd8 F test/where2.test 43d4becaf5a5df854e6c21d624a1cb84c6904554 F test/where3.test 667e75642102c97a00bf9b23d3cb267db321d006 @@ -967,6 +969,7 @@ F test/wherelimit.test 5e9fd41e79bb2b2d588ed999d641d9c965619b31 F test/win32lock.test b2a539e85ae6b2d78475e016a9636b4451dc7fb9 F test/zeroblob.test caaecfb4f908f7bc086ed238668049f96774d688 F test/zerodamage.test 0de750389990b1078bab203c712dc3fefd1d8b82 +F tool/build-all-msvc.bat 1a18aa39983ae7354d834bc55a850a54fc007576 x F tool/build-shell.sh b64a481901fc9ffe5ca8812a2a9255b6cfb77381 F tool/diffdb.c 7524b1b5df217c20cd0431f6789851a4e0cb191b F tool/extract.c 054069d81b095fbdc189a6f5d4466e40380505e2 @@ -983,6 +986,7 @@ F tool/mksqlite3c-noext.tcl 752f1a9d3287f6c0ef5738b1c4add0b96fbe0854 F tool/mksqlite3c.tcl d4923e8e75b7710ddbe4eb37f83dda5eadef63d8 F tool/mksqlite3h.tcl 78013ad79a5e492e5f764f3c7a8ef834255061f8 F tool/mksqlite3internalh.tcl 3dca7bb5374cee003379b8cbac73714f610ef795 +F tool/mkvsix.tcl 19b2ab9ea16445953a76568a5bbe4cb864f92dfe F tool/offsets.c fe4262fdfa378e8f5499a42136d17bf3b98f6091 F tool/omittest.tcl 4665982e95a6e5c1bd806cf7bc3dea95be422d77 F tool/opcodeDoc.awk b3a2a3d5d3075b8bd90b7afe24283efdd586659c @@ -1006,7 +1010,8 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 049b04117353c3e163ffc87916cbe121403a2821 -R 8ad4cfcece99f2543c3fc25ff90ff8da -U mistachkin -Z c7ca4753dded1e68bd82c1792e2c5272 +F tool/win/sqlite.vsix 67d8a99aceb56384a81b3f30d6c71743146d2cc9 +P ed3dc7a89f3416622fcd741ae5fba437929d06d6 31c07db2560ee867723c41cdb634e2aa7993634d +R 8983882407637c60aee542aa1d4c9bca +U drh +Z 9b3c8e4e1baa5816be48b30862221038 diff --git a/manifest.uuid b/manifest.uuid index acb6e11d72..0a036ded25 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ed3dc7a89f3416622fcd741ae5fba437929d06d6 \ No newline at end of file +f4125771e21f1ca29d5442b5441dacfc06b8032b \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 4dfc331bef..632fdc1ac1 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -176,7 +176,7 @@ static void openStatTable( "CREATE TABLE %Q.%s(%s)", pDb->zName, zTab, aTable[i].zCols ); aRoot[i] = pParse->regRoot; - aCreateTbl[i] = 1; + aCreateTbl[i] = OPFLAG_P2ISREG; }else{ /* The table already exists. If zWhere is not NULL, delete all entries ** associated with the table zWhere. If zWhere is NULL, delete the @@ -256,12 +256,11 @@ static void stat3Init( nRow = (tRowcnt)sqlite3_value_int64(argv[0]); mxSample = sqlite3_value_int(argv[1]); n = sizeof(*p) + sizeof(p->a[0])*mxSample; - p = sqlite3_malloc( n ); + p = sqlite3MallocZero( n ); if( p==0 ){ sqlite3_result_error_nomem(context); return; } - memset(p, 0, n); p->a = (struct Stat3Sample*)&p[1]; p->nRow = nRow; p->mxSample = mxSample; diff --git a/src/backup.c b/src/backup.c index 527ecb5744..4881215e96 100644 --- a/src/backup.c +++ b/src/backup.c @@ -164,7 +164,7 @@ sqlite3_backup *sqlite3_backup_init( ** EVIDENCE-OF: R-64852-21591 The sqlite3_backup object is created by a ** call to sqlite3_backup_init() and is destroyed by a call to ** sqlite3_backup_finish(). */ - p = (sqlite3_backup *)sqlite3_malloc(sizeof(sqlite3_backup)); + p = (sqlite3_backup *)sqlite3MallocZero(sizeof(sqlite3_backup)); if( !p ){ sqlite3Error(pDestDb, SQLITE_NOMEM, 0); } @@ -172,7 +172,6 @@ sqlite3_backup *sqlite3_backup_init( /* If the allocation succeeded, populate the new object. */ if( p ){ - memset(p, 0, sizeof(sqlite3_backup)); p->pSrc = findBtree(pDestDb, pSrcDb, zSrcDb); p->pDest = findBtree(pDestDb, pDestDb, zDestDb); p->pDestDb = pDestDb; diff --git a/src/bitvec.c b/src/bitvec.c index 47d33ea840..8d805a6fe5 100644 --- a/src/bitvec.c +++ b/src/bitvec.c @@ -340,10 +340,9 @@ int sqlite3BitvecBuiltinTest(int sz, int *aOp){ /* Allocate the Bitvec to be tested and a linear array of ** bits to act as the reference */ pBitvec = sqlite3BitvecCreate( sz ); - pV = sqlite3_malloc( (sz+7)/8 + 1 ); + pV = sqlite3MallocZero( (sz+7)/8 + 1 ); pTmpSpace = sqlite3_malloc(BITVEC_SZ); if( pBitvec==0 || pV==0 || pTmpSpace==0 ) goto bitvec_end; - memset(pV, 0, (sz+7)/8 + 1); /* NULL pBitvec tests */ sqlite3BitvecSet(0, 1); diff --git a/src/btree.c b/src/btree.c index c16eca5f64..097f02323e 100644 --- a/src/btree.c +++ b/src/btree.c @@ -5926,7 +5926,8 @@ static int balance_nonroot( MemPage *pParent, /* Parent page of siblings being balanced */ int iParentIdx, /* Index of "the page" in pParent */ u8 *aOvflSpace, /* page-size bytes of space for parent ovfl */ - int isRoot /* True if pParent is a root-page */ + int isRoot, /* True if pParent is a root-page */ + int bBulk /* True if this call is part of a bulk load */ ){ BtShared *pBt; /* The whole database */ int nCell = 0; /* Number of cells in apCell[] */ @@ -5990,18 +5991,19 @@ static int balance_nonroot( i = pParent->nOverflow + pParent->nCell; if( i<2 ){ nxDiv = 0; - nOld = i+1; }else{ - nOld = 3; + assert( bBulk==0 || bBulk==1 ); if( iParentIdx==0 ){ nxDiv = 0; }else if( iParentIdx==i ){ - nxDiv = i-2; + nxDiv = i-2+bBulk; }else{ + assert( bBulk==0 ); nxDiv = iParentIdx-1; } - i = 2; + i = 2-bBulk; } + nOld = i+1; if( (i+nxDiv-pParent->nOverflow)==pParent->nCell ){ pRight = &pParent->aData[pParent->hdrOffset+8]; }else{ @@ -6210,7 +6212,9 @@ static int balance_nonroot( d = r + 1 - leafData; assert( d0 ); - rc = allocateBtreePage(pBt, &pNew, &pgno, pgno, 0); + rc = allocateBtreePage(pBt, &pNew, &pgno, (bBulk ? 1 : pgno), 0); if( rc ) goto balance_cleanup; apNew[i] = pNew; nNew++; @@ -6469,6 +6473,7 @@ static int balance_nonroot( ** sibling page j. If the siblings are not leaf pages of an ** intkey b-tree, then cell i was a divider cell. */ assert( j+1 < ArraySize(apCopy) ); + assert( j+1 < nOld ); pOld = apCopy[++j]; iNextOld = i + !leafData + pOld->nCell + pOld->nOverflow; if( pOld->nOverflow ){ @@ -6707,7 +6712,7 @@ static int balance(BtCursor *pCur){ ** pSpace buffer passed to the latter call to balance_nonroot(). */ u8 *pSpace = sqlite3PageMalloc(pCur->pBt->pageSize); - rc = balance_nonroot(pParent, iIdx, pSpace, iPage==1); + rc = balance_nonroot(pParent, iIdx, pSpace, iPage==1, pCur->hints); if( pFree ){ /* If pFree is not NULL, it points to the pSpace buffer used ** by a previous call to balance_nonroot(). Its contents are @@ -8294,3 +8299,12 @@ int sqlite3BtreeSetVersion(Btree *pBtree, int iVersion){ pBt->btsFlags &= ~BTS_NO_WAL; return rc; } + +/* +** set the mask of hint flags for cursor pCsr. Currently the only valid +** values are 0 and BTREE_BULKLOAD. +*/ +void sqlite3BtreeCursorHints(BtCursor *pCsr, unsigned int mask){ + assert( mask==BTREE_BULKLOAD || mask==0 ); + pCsr->hints = mask; +} diff --git a/src/btree.h b/src/btree.h index 9832001b7f..95897d5662 100644 --- a/src/btree.h +++ b/src/btree.h @@ -135,6 +135,12 @@ int sqlite3BtreeUpdateMeta(Btree*, int idx, u32 value); #define BTREE_USER_VERSION 6 #define BTREE_INCR_VACUUM 7 +/* +** Values that may be OR'd together to form the second argument of an +** sqlite3BtreeCursorHints() call. +*/ +#define BTREE_BULKLOAD 0x00000001 + int sqlite3BtreeCursor( Btree*, /* BTree containing table to open */ int iTable, /* Index of root page */ @@ -178,8 +184,8 @@ struct Pager *sqlite3BtreePager(Btree*); int sqlite3BtreePutData(BtCursor*, u32 offset, u32 amt, void*); void sqlite3BtreeCacheOverflow(BtCursor *); void sqlite3BtreeClearCursor(BtCursor *); - int sqlite3BtreeSetVersion(Btree *pBt, int iVersion); +void sqlite3BtreeCursorHints(BtCursor *, unsigned int mask); #ifndef NDEBUG int sqlite3BtreeCursorIsValid(BtCursor*); diff --git a/src/btreeInt.h b/src/btreeInt.h index 0d21497966..b157decec7 100644 --- a/src/btreeInt.h +++ b/src/btreeInt.h @@ -510,6 +510,7 @@ struct BtCursor { #ifndef SQLITE_OMIT_INCRBLOB u8 isIncrblobHandle; /* True if this cursor is an incr. io handle */ #endif + u8 hints; /* As configured by CursorSetHints() */ i16 iPage; /* Index of current page in apPage */ u16 aiIdx[BTCURSOR_MAX_DEPTH]; /* Current index in apPage[i] */ MemPage *apPage[BTCURSOR_MAX_DEPTH]; /* Pages from root to current page */ diff --git a/src/build.c b/src/build.c index 9f13b7b11c..776ffa4db3 100644 --- a/src/build.c +++ b/src/build.c @@ -1581,7 +1581,7 @@ void sqlite3EndTable( assert(pParse->nTab==1); sqlite3VdbeAddOp3(v, OP_OpenWrite, 1, pParse->regRoot, iDb); - sqlite3VdbeChangeP5(v, 1); + sqlite3VdbeChangeP5(v, OPFLAG_P2ISREG); pParse->nTab = 2; sqlite3SelectDestInit(&dest, SRT_Table, 1); sqlite3Select(pParse, pSelect, &dest); @@ -2397,9 +2397,7 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ pKey = sqlite3IndexKeyinfo(pParse, pIndex); sqlite3VdbeAddOp4(v, OP_OpenWrite, iIdx, tnum, iDb, (char *)pKey, P4_KEYINFO_HANDOFF); - if( memRootPage>=0 ){ - sqlite3VdbeChangeP5(v, 1); - } + sqlite3VdbeChangeP5(v, OPFLAG_BULKCSR|((memRootPage>=0)?OPFLAG_P2ISREG:0)); #ifndef SQLITE_OMIT_MERGE_SORT /* Open the sorter cursor if we are to use one. */ diff --git a/src/expr.c b/src/expr.c index 1e46596a2f..790aa45f3b 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1701,7 +1701,7 @@ int sqlite3CodeSubselect( assert( !isRowid ); sqlite3SelectDestInit(&dest, SRT_Set, pExpr->iTable); - dest.affinity = (u8)affinity; + dest.affSdst = (u8)affinity; assert( (pExpr->iTable&0x0000FFFF)==pExpr->iTable ); pExpr->x.pSelect->iLimit = 0; if( sqlite3Select(pParse, pExpr->x.pSelect, &dest) ){ @@ -1794,11 +1794,11 @@ int sqlite3CodeSubselect( sqlite3SelectDestInit(&dest, 0, ++pParse->nMem); if( pExpr->op==TK_SELECT ){ dest.eDest = SRT_Mem; - sqlite3VdbeAddOp2(v, OP_Null, 0, dest.iParm); + sqlite3VdbeAddOp2(v, OP_Null, 0, dest.iSDParm); VdbeComment((v, "Init subquery result")); }else{ dest.eDest = SRT_Exists; - sqlite3VdbeAddOp2(v, OP_Integer, 0, dest.iParm); + sqlite3VdbeAddOp2(v, OP_Integer, 0, dest.iSDParm); VdbeComment((v, "Init EXISTS result")); } sqlite3ExprDelete(pParse->db, pSel->pLimit); @@ -1808,7 +1808,7 @@ int sqlite3CodeSubselect( if( sqlite3Select(pParse, pSel, &dest) ){ return 0; } - rReg = dest.iParm; + rReg = dest.iSDParm; ExprSetIrreducible(pExpr); break; } diff --git a/src/hash.c b/src/hash.c index d4daf92a6f..d7625d3913 100644 --- a/src/hash.c +++ b/src/hash.c @@ -113,7 +113,11 @@ static int rehash(Hash *pH, unsigned int new_size){ /* The inability to allocates space for a larger hash table is ** a performance hit but it is not a fatal error. So mark the - ** allocation as a benign. + ** allocation as a benign. Use sqlite3Malloc()/memset(0) instead of + ** sqlite3MallocZero() to make the allocation, as sqlite3MallocZero() + ** only zeroes the requested number of bytes whereas this module will + ** use the actual amount of space allocated for the hash table (which + ** may be larger than the requested amount). */ sqlite3BeginBenignMalloc(); new_ht = (struct _ht *)sqlite3Malloc( new_size*sizeof(struct _ht) ); diff --git a/src/insert.c b/src/insert.c index a589c8aef6..faed6470c2 100644 --- a/src/insert.c +++ b/src/insert.c @@ -597,7 +597,7 @@ void sqlite3Insert( VdbeComment((v, "SELECT eof flag")); sqlite3SelectDestInit(&dest, SRT_Coroutine, ++pParse->nMem); addrSelect = sqlite3VdbeCurrentAddr(v)+2; - sqlite3VdbeAddOp2(v, OP_Integer, addrSelect-1, dest.iParm); + sqlite3VdbeAddOp2(v, OP_Integer, addrSelect-1, dest.iSDParm); j1 = sqlite3VdbeAddOp2(v, OP_Goto, 0, 0); VdbeComment((v, "Jump over SELECT coroutine")); @@ -608,15 +608,15 @@ void sqlite3Insert( goto insert_cleanup; } sqlite3VdbeAddOp2(v, OP_Integer, 1, regEof); /* EOF <- 1 */ - sqlite3VdbeAddOp1(v, OP_Yield, dest.iParm); /* yield X */ + sqlite3VdbeAddOp1(v, OP_Yield, dest.iSDParm); /* yield X */ sqlite3VdbeAddOp2(v, OP_Halt, SQLITE_INTERNAL, OE_Abort); VdbeComment((v, "End of SELECT coroutine")); sqlite3VdbeJumpHere(v, j1); /* label B: */ - regFromSelect = dest.iMem; + regFromSelect = dest.iSdst; assert( pSelect->pEList ); nColumn = pSelect->pEList->nExpr; - assert( dest.nMem==nColumn ); + assert( dest.nSdst==nColumn ); /* Set useTempTable to TRUE if the result of the SELECT statement ** should be written into a temporary table (template 4). Set to @@ -652,7 +652,7 @@ void sqlite3Insert( regRec = sqlite3GetTempReg(pParse); regTempRowid = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp2(v, OP_OpenEphemeral, srcTab, nColumn); - addrTop = sqlite3VdbeAddOp1(v, OP_Yield, dest.iParm); + addrTop = sqlite3VdbeAddOp1(v, OP_Yield, dest.iSDParm); addrIf = sqlite3VdbeAddOp1(v, OP_If, regEof); sqlite3VdbeAddOp3(v, OP_MakeRecord, regFromSelect, nColumn, regRec); sqlite3VdbeAddOp2(v, OP_NewRowid, srcTab, regTempRowid); @@ -789,7 +789,7 @@ void sqlite3Insert( ** goto C ** D: ... */ - addrCont = sqlite3VdbeAddOp1(v, OP_Yield, dest.iParm); + addrCont = sqlite3VdbeAddOp1(v, OP_Yield, dest.iSDParm); addrInsTop = sqlite3VdbeAddOp1(v, OP_If, regEof); } diff --git a/src/os.h b/src/os.h index 37d08fc254..1ec7d4ba11 100644 --- a/src/os.h +++ b/src/os.h @@ -92,8 +92,8 @@ #endif /* -** Determine if we are dealing with WindowsRT (Metro) as this has a different and -** incompatible API from win32. +** Determine if we are dealing with WinRT, which provides only a subset of +** the full Win32 API. */ #if !defined(SQLITE_OS_WINRT) # define SQLITE_OS_WINRT 0 diff --git a/src/os_win.c b/src/os_win.c index 75d0edcddb..9fd6f8ab46 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -288,7 +288,8 @@ static struct win_syscall { #define osCreateFileW ((HANDLE(WINAPI*)(LPCWSTR,DWORD,DWORD, \ LPSECURITY_ATTRIBUTES,DWORD,DWORD,HANDLE))aSyscall[5].pCurrent) -#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) +#if SQLITE_OS_WINCE || (!SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) && \ + !defined(SQLITE_OMIT_WAL)) { "CreateFileMappingW", (SYSCALL)CreateFileMappingW, 0 }, #else { "CreateFileMappingW", (SYSCALL)0, 0 }, @@ -600,7 +601,7 @@ static struct win_syscall { LPOVERLAPPED))aSyscall[45].pCurrent) #endif -#if !SQLITE_OS_WINRT +#if SQLITE_OS_WINCE || (!SQLITE_OS_WINRT && !defined(SQLITE_OMIT_WAL)) { "MapViewOfFile", (SYSCALL)MapViewOfFile, 0 }, #else { "MapViewOfFile", (SYSCALL)0, 0 }, @@ -670,7 +671,11 @@ static struct win_syscall { #define osUnlockFileEx ((BOOL(WINAPI*)(HANDLE,DWORD,DWORD,DWORD, \ LPOVERLAPPED))aSyscall[55].pCurrent) +#if SQLITE_OS_WINCE || !defined(SQLITE_OMIT_WAL) { "UnmapViewOfFile", (SYSCALL)UnmapViewOfFile, 0 }, +#else + { "UnmapViewOfFile", (SYSCALL)0, 0 }, +#endif #define osUnmapViewOfFile ((BOOL(WINAPI*)(LPCVOID))aSyscall[56].pCurrent) @@ -702,7 +707,7 @@ static struct win_syscall { #define osWaitForSingleObject ((DWORD(WINAPI*)(HANDLE, \ DWORD))aSyscall[60].pCurrent) -#if !SQLITE_OS_WINCE +#if SQLITE_OS_WINRT { "WaitForSingleObjectEx", (SYSCALL)WaitForSingleObjectEx, 0 }, #else { "WaitForSingleObjectEx", (SYSCALL)0, 0 }, @@ -711,7 +716,7 @@ static struct win_syscall { #define osWaitForSingleObjectEx ((DWORD(WINAPI*)(HANDLE,DWORD, \ BOOL))aSyscall[61].pCurrent) -#if !SQLITE_OS_WINCE +#if SQLITE_OS_WINRT { "SetFilePointerEx", (SYSCALL)SetFilePointerEx, 0 }, #else { "SetFilePointerEx", (SYSCALL)0, 0 }, @@ -729,7 +734,7 @@ static struct win_syscall { #define osGetFileInformationByHandleEx ((BOOL(WINAPI*)(HANDLE, \ FILE_INFO_BY_HANDLE_CLASS,LPVOID,DWORD))aSyscall[63].pCurrent) -#if SQLITE_OS_WINRT +#if SQLITE_OS_WINRT && !defined(SQLITE_OMIT_WAL) { "MapViewOfFileFromApp", (SYSCALL)MapViewOfFileFromApp, 0 }, #else { "MapViewOfFileFromApp", (SYSCALL)0, 0 }, @@ -793,7 +798,7 @@ static struct win_syscall { #define osGetProcessHeap ((HANDLE(WINAPI*)(VOID))aSyscall[71].pCurrent) -#if SQLITE_OS_WINRT +#if SQLITE_OS_WINRT && !defined(SQLITE_OMIT_WAL) { "CreateFileMappingFromApp", (SYSCALL)CreateFileMappingFromApp, 0 }, #else { "CreateFileMappingFromApp", (SYSCALL)0, 0 }, diff --git a/src/pcache1.c b/src/pcache1.c index c41b49e6c8..4147d2eff5 100644 --- a/src/pcache1.c +++ b/src/pcache1.c @@ -396,11 +396,10 @@ static int pcache1ResizeHash(PCache1 *p){ pcache1LeaveMutex(p->pGroup); if( p->nHash ){ sqlite3BeginBenignMalloc(); } - apNew = (PgHdr1 **)sqlite3_malloc(sizeof(PgHdr1 *)*nNew); + apNew = (PgHdr1 **)sqlite3MallocZero(sizeof(PgHdr1 *)*nNew); if( p->nHash ){ sqlite3EndBenignMalloc(); } pcache1EnterMutex(p->pGroup); if( apNew ){ - memset(apNew, 0, sizeof(PgHdr1 *)*nNew); for(i=0; inHash; i++){ PgHdr1 *pPage; PgHdr1 *pNext = p->apHash[i]; @@ -584,9 +583,8 @@ static sqlite3_pcache *pcache1Create(int szPage, int szExtra, int bPurgeable){ assert( szExtra < 300 ); sz = sizeof(PCache1) + sizeof(PGroup)*separateCache; - pCache = (PCache1 *)sqlite3_malloc(sz); + pCache = (PCache1 *)sqlite3MallocZero(sz); if( pCache ){ - memset(pCache, 0, sz); if( separateCache ){ pGroup = (PGroup*)&pCache[1]; pGroup->mxPinned = 10; diff --git a/src/select.c b/src/select.c index d79a6112f7..491356acbb 100644 --- a/src/select.c +++ b/src/select.c @@ -36,10 +36,10 @@ static void clearSelect(sqlite3 *db, Select *p){ */ void sqlite3SelectDestInit(SelectDest *pDest, int eDest, int iParm){ pDest->eDest = (u8)eDest; - pDest->iParm = iParm; - pDest->affinity = 0; - pDest->iMem = 0; - pDest->nMem = 0; + pDest->iSDParm = iParm; + pDest->affSdst = 0; + pDest->iSdst = 0; + pDest->nSdst = 0; } @@ -551,7 +551,7 @@ static void selectInnerLoop( int hasDistinct; /* True if the DISTINCT keyword is present */ int regResult; /* Start of memory holding result set */ int eDest = pDest->eDest; /* How to dispose of results */ - int iParm = pDest->iParm; /* First argument to disposal method */ + int iParm = pDest->iSDParm; /* First argument to disposal method */ int nResultCol; /* Number of result columns */ assert( v ); @@ -569,14 +569,14 @@ static void selectInnerLoop( }else{ nResultCol = pEList->nExpr; } - if( pDest->iMem==0 ){ - pDest->iMem = pParse->nMem+1; - pDest->nMem = nResultCol; + if( pDest->iSdst==0 ){ + pDest->iSdst = pParse->nMem+1; + pDest->nSdst = nResultCol; pParse->nMem += nResultCol; }else{ - assert( pDest->nMem==nResultCol ); + assert( pDest->nSdst==nResultCol ); } - regResult = pDest->iMem; + regResult = pDest->iSdst; if( nColumn>0 ){ for(i=0; iaffinity = sqlite3CompareAffinity(pEList->a[0].pExpr, pDest->affinity); + p->affinity = sqlite3CompareAffinity(pEList->a[0].pExpr, pDest->affSdst); if( pOrderBy ){ /* At first glance you would think we could optimize out the ** ORDER BY in this case since the order of entries in the set @@ -710,7 +710,7 @@ static void selectInnerLoop( pushOntoSorter(pParse, pOrderBy, p, r1); sqlite3ReleaseTempReg(pParse, r1); }else if( eDest==SRT_Coroutine ){ - sqlite3VdbeAddOp1(v, OP_Yield, pDest->iParm); + sqlite3VdbeAddOp1(v, OP_Yield, pDest->iSDParm); }else{ sqlite3VdbeAddOp2(v, OP_ResultRow, regResult, nColumn); sqlite3ExprCacheAffinityChange(pParse, regResult, nColumn); @@ -890,7 +890,7 @@ static void generateSortTail( ExprList *pOrderBy = p->pOrderBy; int eDest = pDest->eDest; - int iParm = pDest->iParm; + int iParm = pDest->iSDParm; int regRow; int regRowid; @@ -949,17 +949,17 @@ static void generateSortTail( testcase( eDest==SRT_Output ); testcase( eDest==SRT_Coroutine ); for(i=0; iiMem+i ); - sqlite3VdbeAddOp3(v, OP_Column, pseudoTab, i, pDest->iMem+i); + assert( regRow!=pDest->iSdst+i ); + sqlite3VdbeAddOp3(v, OP_Column, pseudoTab, i, pDest->iSdst+i); if( i==0 ){ sqlite3VdbeChangeP5(v, OPFLAG_CLEARCACHE); } } if( eDest==SRT_Output ){ - sqlite3VdbeAddOp2(v, OP_ResultRow, pDest->iMem, nColumn); - sqlite3ExprCacheAffinityChange(pParse, pDest->iMem, nColumn); + sqlite3VdbeAddOp2(v, OP_ResultRow, pDest->iSdst, nColumn); + sqlite3ExprCacheAffinityChange(pParse, pDest->iSdst, nColumn); }else{ - sqlite3VdbeAddOp1(v, OP_Yield, pDest->iParm); + sqlite3VdbeAddOp1(v, OP_Yield, pDest->iSDParm); } break; } @@ -1610,7 +1610,7 @@ static int multiSelect( */ if( dest.eDest==SRT_EphemTab ){ assert( p->pEList ); - sqlite3VdbeAddOp2(v, OP_OpenEphemeral, dest.iParm, p->pEList->nExpr); + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, dest.iSDParm, p->pEList->nExpr); sqlite3VdbeChangeP5(v, BTREE_UNORDERED); dest.eDest = SRT_Table; } @@ -1696,7 +1696,7 @@ static int multiSelect( ** of a 3-way or more compound */ assert( p->pLimit==0 ); /* Not allowed on leftward elements */ assert( p->pOffset==0 ); /* Not allowed on leftward elements */ - unionTab = dest.iParm; + unionTab = dest.iSDParm; }else{ /* We will need to create our own temporary table to hold the ** intermediate results. @@ -1753,7 +1753,7 @@ static int multiSelect( /* Convert the data in the temporary table into whatever form ** it is that we currently need. */ - assert( unionTab==dest.iParm || dest.eDest!=priorOp ); + assert( unionTab==dest.iSDParm || dest.eDest!=priorOp ); if( dest.eDest!=priorOp ){ int iCont, iBreak, iStart; assert( p->pEList ); @@ -1817,7 +1817,7 @@ static int multiSelect( p->pLimit = 0; pOffset = p->pOffset; p->pOffset = 0; - intersectdest.iParm = tab2; + intersectdest.iSDParm = tab2; explainSetInteger(iSub2, pParse->iNextSelectId); rc = sqlite3Select(pParse, p, &intersectdest); testcase( rc!=SQLITE_OK ); @@ -1911,8 +1911,8 @@ static int multiSelect( } multi_select_end: - pDest->iMem = dest.iMem; - pDest->nMem = dest.nMem; + pDest->iSdst = dest.iSdst; + pDest->nSdst = dest.nSdst; sqlite3SelectDelete(db, pDelete); return rc; } @@ -1922,8 +1922,8 @@ multi_select_end: ** Code an output subroutine for a coroutine implementation of a ** SELECT statment. ** -** The data to be output is contained in pIn->iMem. There are -** pIn->nMem columns to be output. pDest is where the output should +** The data to be output is contained in pIn->iSdst. There are +** pIn->nSdst columns to be output. pDest is where the output should ** be sent. ** ** regReturn is the number of the register holding the subroutine @@ -1961,11 +1961,11 @@ static int generateOutputSubroutine( if( regPrev ){ int j1, j2; j1 = sqlite3VdbeAddOp1(v, OP_IfNot, regPrev); - j2 = sqlite3VdbeAddOp4(v, OP_Compare, pIn->iMem, regPrev+1, pIn->nMem, + j2 = sqlite3VdbeAddOp4(v, OP_Compare, pIn->iSdst, regPrev+1, pIn->nSdst, (char*)pKeyInfo, p4type); sqlite3VdbeAddOp3(v, OP_Jump, j2+2, iContinue, j2+2); sqlite3VdbeJumpHere(v, j1); - sqlite3ExprCodeCopy(pParse, pIn->iMem, regPrev+1, pIn->nMem); + sqlite3ExprCodeCopy(pParse, pIn->iSdst, regPrev+1, pIn->nSdst); sqlite3VdbeAddOp2(v, OP_Integer, 1, regPrev); } if( pParse->db->mallocFailed ) return 0; @@ -1983,9 +1983,9 @@ static int generateOutputSubroutine( int r2 = sqlite3GetTempReg(pParse); testcase( pDest->eDest==SRT_Table ); testcase( pDest->eDest==SRT_EphemTab ); - sqlite3VdbeAddOp3(v, OP_MakeRecord, pIn->iMem, pIn->nMem, r1); - sqlite3VdbeAddOp2(v, OP_NewRowid, pDest->iParm, r2); - sqlite3VdbeAddOp3(v, OP_Insert, pDest->iParm, r1, r2); + sqlite3VdbeAddOp3(v, OP_MakeRecord, pIn->iSdst, pIn->nSdst, r1); + sqlite3VdbeAddOp2(v, OP_NewRowid, pDest->iSDParm, r2); + sqlite3VdbeAddOp3(v, OP_Insert, pDest->iSDParm, r1, r2); sqlite3VdbeChangeP5(v, OPFLAG_APPEND); sqlite3ReleaseTempReg(pParse, r2); sqlite3ReleaseTempReg(pParse, r1); @@ -1999,13 +1999,13 @@ static int generateOutputSubroutine( */ case SRT_Set: { int r1; - assert( pIn->nMem==1 ); + assert( pIn->nSdst==1 ); p->affinity = - sqlite3CompareAffinity(p->pEList->a[0].pExpr, pDest->affinity); + sqlite3CompareAffinity(p->pEList->a[0].pExpr, pDest->affSdst); r1 = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp4(v, OP_MakeRecord, pIn->iMem, 1, r1, &p->affinity, 1); - sqlite3ExprCacheAffinityChange(pParse, pIn->iMem, 1); - sqlite3VdbeAddOp2(v, OP_IdxInsert, pDest->iParm, r1); + sqlite3VdbeAddOp4(v, OP_MakeRecord, pIn->iSdst, 1, r1, &p->affinity, 1); + sqlite3ExprCacheAffinityChange(pParse, pIn->iSdst, 1); + sqlite3VdbeAddOp2(v, OP_IdxInsert, pDest->iSDParm, r1); sqlite3ReleaseTempReg(pParse, r1); break; } @@ -2014,7 +2014,7 @@ static int generateOutputSubroutine( /* If any row exist in the result set, record that fact and abort. */ case SRT_Exists: { - sqlite3VdbeAddOp2(v, OP_Integer, 1, pDest->iParm); + sqlite3VdbeAddOp2(v, OP_Integer, 1, pDest->iSDParm); /* The LIMIT clause will terminate the loop for us */ break; } @@ -2025,23 +2025,23 @@ static int generateOutputSubroutine( ** of the scan loop. */ case SRT_Mem: { - assert( pIn->nMem==1 ); - sqlite3ExprCodeMove(pParse, pIn->iMem, pDest->iParm, 1); + assert( pIn->nSdst==1 ); + sqlite3ExprCodeMove(pParse, pIn->iSdst, pDest->iSDParm, 1); /* The LIMIT clause will jump out of the loop for us */ break; } #endif /* #ifndef SQLITE_OMIT_SUBQUERY */ /* The results are stored in a sequence of registers - ** starting at pDest->iMem. Then the co-routine yields. + ** starting at pDest->iSdst. Then the co-routine yields. */ case SRT_Coroutine: { - if( pDest->iMem==0 ){ - pDest->iMem = sqlite3GetTempRange(pParse, pIn->nMem); - pDest->nMem = pIn->nMem; + if( pDest->iSdst==0 ){ + pDest->iSdst = sqlite3GetTempRange(pParse, pIn->nSdst); + pDest->nSdst = pIn->nSdst; } - sqlite3ExprCodeMove(pParse, pIn->iMem, pDest->iMem, pDest->nMem); - sqlite3VdbeAddOp1(v, OP_Yield, pDest->iParm); + sqlite3ExprCodeMove(pParse, pIn->iSdst, pDest->iSdst, pDest->nSdst); + sqlite3VdbeAddOp1(v, OP_Yield, pDest->iSDParm); break; } @@ -2055,8 +2055,8 @@ static int generateOutputSubroutine( */ default: { assert( pDest->eDest==SRT_Output ); - sqlite3VdbeAddOp2(v, OP_ResultRow, pIn->iMem, pIn->nMem); - sqlite3ExprCacheAffinityChange(pParse, pIn->iMem, pIn->nMem); + sqlite3VdbeAddOp2(v, OP_ResultRow, pIn->iSdst, pIn->nSdst); + sqlite3ExprCacheAffinityChange(pParse, pIn->iSdst, pIn->nSdst); break; } } @@ -2475,7 +2475,7 @@ static int multiSelectOrderBy( */ sqlite3VdbeResolveLabel(v, labelCmpr); sqlite3VdbeAddOp4(v, OP_Permutation, 0, 0, 0, (char*)aPermute, P4_INTARRAY); - sqlite3VdbeAddOp4(v, OP_Compare, destA.iMem, destB.iMem, nOrderBy, + sqlite3VdbeAddOp4(v, OP_Compare, destA.iSdst, destB.iSdst, nOrderBy, (char*)pKeyMerge, P4_KEYINFO_HANDOFF); sqlite3VdbeAddOp3(v, OP_Jump, addrAltB, addrAeqB, addrAgtB); @@ -3721,23 +3721,24 @@ static void explainSimpleCount( ** ** SRT_Mem Only valid if the result is a single column. ** Store the first column of the first result row -** in register pDest->iParm then abandon the rest +** in register pDest->iSDParm then abandon the rest ** of the query. This destination implies "LIMIT 1". ** ** SRT_Set The result must be a single column. Store each -** row of result as the key in table pDest->iParm. -** Apply the affinity pDest->affinity before storing +** row of result as the key in table pDest->iSDParm. +** Apply the affinity pDest->affSdst before storing ** results. Used to implement "IN (SELECT ...)". ** -** SRT_Union Store results as a key in a temporary table pDest->iParm. +** SRT_Union Store results as a key in a temporary table +** identified by pDest->iSDParm. ** -** SRT_Except Remove results from the temporary table pDest->iParm. +** SRT_Except Remove results from the temporary table pDest->iSDParm. ** -** SRT_Table Store results in temporary table pDest->iParm. +** SRT_Table Store results in temporary table pDest->iSDParm. ** This is like SRT_EphemTab except that the table ** is assumed to already be open. ** -** SRT_EphemTab Create an temporary table pDest->iParm and store +** SRT_EphemTab Create an temporary table pDest->iSDParm and store ** the result there. The cursor is left open after ** returning. This is like SRT_Table except that ** this destination uses OP_OpenEphemeral to create @@ -3745,9 +3746,9 @@ static void explainSimpleCount( ** ** SRT_Coroutine Generate a co-routine that returns a new row of ** results each time it is invoked. The entry point -** of the co-routine is stored in register pDest->iParm. +** of the co-routine is stored in register pDest->iSDParm. ** -** SRT_Exists Store a 1 in memory cell pDest->iParm if the result +** SRT_Exists Store a 1 in memory cell pDest->iSDParm if the result ** set is not empty. ** ** SRT_Discard Throw the results away. This is used by SELECT @@ -3991,7 +3992,7 @@ int sqlite3Select( /* If the output is destined for a temporary table, open that table. */ if( pDest->eDest==SRT_EphemTab ){ - sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pDest->iParm, pEList->nExpr); + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pDest->iSDParm, pEList->nExpr); } /* Set the limiter. diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 69ea6d15d7..aa60f80b22 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -4152,11 +4152,11 @@ typedef void (*sqlite3_destructor_type)(void*); ** the error code is SQLITE_ERROR. ^A subsequent call to sqlite3_result_error() ** or sqlite3_result_error16() resets the error code to SQLITE_ERROR. ** -** ^The sqlite3_result_toobig() interface causes SQLite to throw an error -** indicating that a string or BLOB is too long to represent. +** ^The sqlite3_result_error_toobig() interface causes SQLite to throw an +** error indicating that a string or BLOB is too long to represent. ** -** ^The sqlite3_result_nomem() interface causes SQLite to throw an error -** indicating that a memory allocation failed. +** ^The sqlite3_result_error_nomem() interface causes SQLite to throw an +** error indicating that a memory allocation failed. ** ** ^The sqlite3_result_int() interface sets the return value ** of the application-defined function to be the 32-bit signed integer diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 559a552de0..fc2c5593a1 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2119,10 +2119,10 @@ struct Select { typedef struct SelectDest SelectDest; struct SelectDest { u8 eDest; /* How to dispose of the results */ - u8 affinity; /* Affinity used when eDest==SRT_Set */ - int iParm; /* A parameter used by the eDest disposal method */ - int iMem; /* Base register where results are written */ - int nMem; /* Number of registers allocated */ + u8 affSdst; /* Affinity used when eDest==SRT_Set */ + int iSDParm; /* A parameter used by the eDest disposal method */ + int iSdst; /* Base register where results are written */ + int nSdst; /* Number of registers allocated */ }; /* @@ -2318,6 +2318,8 @@ struct AuthContext { #define OPFLAG_CLEARCACHE 0x20 /* Clear pseudo-table cache in OP_Column */ #define OPFLAG_LENGTHARG 0x40 /* OP_Column only used for length() */ #define OPFLAG_TYPEOFARG 0x80 /* OP_Column only used for typeof() */ +#define OPFLAG_BULKCSR 0x01 /* OP_Open** used to open bulk cursor */ +#define OPFLAG_P2ISREG 0x02 /* P2 to OP_Open** is a register number */ /* * Each trigger present in the database schema is stored as an instance of diff --git a/src/test_spellfix.c b/src/test_spellfix.c index cc3531268b..92755cd1d6 100644 --- a/src/test_spellfix.c +++ b/src/test_spellfix.c @@ -10,276 +10,9 @@ ** ************************************************************************* ** -** This module implements a VIRTUAL TABLE that can be used to search -** a large vocabulary for close matches. For example, this virtual -** table can be used to suggest corrections to misspelled words. Or, -** it could be used with FTS4 to do full-text search using potentially -** misspelled words. -** -** Create an instance of the virtual table this way: -** -** CREATE VIRTUAL TABLE demo USING spellfix1; -** -** The "spellfix1" term is the name of this module. The "demo" is the -** name of the virtual table you will be creating. The table is initially -** empty. You have to populate it with your vocabulary. Suppose you -** have a list of words in a table named "big_vocabulary". Then do this: -** -** INSERT INTO demo(word) SELECT word FROM big_vocabulary; -** -** If you intend to use this virtual table in cooperation with an FTS4 -** table (for spelling correctly of search terms) then you can extract -** the vocabulary using an fts3aux table: -** -** INSERT INTO demo(word) SELECT term FROM search_aux WHERE col='*'; -** -** You can also provide the virtual table with a "rank" for each word. -** The "rank" is an estimate of how common the word is. Larger numbers -** mean the word is more common. If you omit the rank when populating -** the table, then a rank of 1 is assumed. But if you have rank -** information, you can supply it and the virtual table will show a -** slight preference for selecting more commonly used terms. To -** populate the rank from an fts4aux table "search_aux" do something -** like this: -** -** INSERT INTO demo(word,rank) -** SELECT term, documents FROM search_aux WHERE col='*'; -** -** To query the virtual table, include a MATCH operator in the WHERE -** clause. For example: -** -** SELECT word FROM demo WHERE word MATCH 'kennasaw'; -** -** Using a dataset of American place names (derived from -** http://geonames.usgs.gov/domestic/download_data.htm) the query above -** returns 20 results beginning with: -** -** kennesaw -** kenosha -** kenesaw -** kenaga -** keanak -** -** If you append the character '*' to the end of the pattern, then -** a prefix search is performed. For example: -** -** SELECT word FROM demo WHERE word MATCH 'kennes*'; -** -** Yields 20 results beginning with: -** -** kennesaw -** kennestone -** kenneson -** kenneys -** keanes -** keenes -** -** The virtual table actually has a unique rowid with five columns plus three -** extra hidden columns. The columns are as follows: -** -** rowid A unique integer number associated with each -** vocabulary item in the table. This can be used -** as a foreign key on other tables in the database. -** -** word The text of the word that matches the pattern. -** Both word and pattern can contains unicode characters -** and can be mixed case. -** -** rank This is the rank of the word, as specified in the -** original INSERT statement. -** -** distance This is an edit distance or Levensthein distance going -** from the pattern to the word. -** -** langid This is the language-id of the word. All queries are -** against a single language-id, which defaults to 0. -** For any given query this value is the same on all rows. -** -** score The score is a combination of rank and distance. The -** idea is that a lower score is better. The virtual table -** attempts to find words with the lowest score and -** by default (unless overridden by ORDER BY) returns -** results in order of increasing score. -** -** matchlen For prefix queries, the number of characters in the prefix -** of the returned value (word) that matched the query term. -** For non-prefix queries, the number of characters in the -** returned value. -** -** top (HIDDEN) For any query, this value is the same on all -** rows. It is an integer which is the maximum number of -** rows that will be output. The actually number of rows -** output might be less than this number, but it will never -** be greater. The default value for top is 20, but that -** can be changed for each query by including a term of -** the form "top=N" in the WHERE clause of the query. -** -** scope (HIDDEN) For any query, this value is the same on all -** rows. The scope is a measure of how widely the virtual -** table looks for matching words. Smaller values of -** scope cause a broader search. The scope is normally -** choosen automatically and is capped at 4. Applications -** can change the scope by including a term of the form -** "scope=N" in the WHERE clause of the query. Increasing -** the scope will make the query run faster, but will reduce -** the possible corrections. -** -** srchcnt (HIDDEN) For any query, this value is the same on all -** rows. This value is an integer which is the number of -** of words examined using the edit-distance algorithm to -** find the top matches that are ultimately displayed. This -** value is for diagnostic use only. -** -** soundslike (HIDDEN) When inserting vocabulary entries, this field -** can be set to an spelling that matches what the word -** sounds like. See the DEALING WITH UNUSUAL AND DIFFICULT -** SPELLINGS section below for details. -** -** When inserting into or updating the virtual table, only the rowid, word, -** rank, and langid may be changes. Any attempt to set or modify the values -** of distance, score, top, scope, or srchcnt is silently ignored. -** -** ALGORITHM -** -** A shadow table named "%_vocab" (where the % is replaced by the name of -** the virtual table; Ex: "demo_vocab" for the "demo" virtual table) is -** constructed with these columns: -** -** id The unique id (INTEGER PRIMARY KEY) -** -** rank The rank of word. -** -** langid The language id for this entry. -** -** word The original UTF8 text of the vocabulary word -** -** k1 The word transliterated into lower-case ASCII. -** There is a standard table of mappings from non-ASCII -** characters into ASCII. Examples: "æ" -> "ae", -** "þ" -> "th", "ß" -> "ss", "á" -> "a", ... The -** accessory function spellfix1_translit(X) will do -** the non-ASCII to ASCII mapping. The built-in lower(X) -** function will convert to lower-case. Thus: -** k1 = lower(spellfix1_translit(word)). -** -** k2 This field holds a phonetic code derived from k1. Letters -** that have similar sounds are mapped into the same symbol. -** For example, all vowels and vowel clusters become the -** single symbol "A". And the letters "p", "b", "f", and -** "v" all become "B". All nasal sounds are represented -** as "N". And so forth. The mapping is base on -** ideas found in Soundex, Metaphone, and other -** long-standing phonetic matching systems. This key can -** be generated by the function spellfix1_phonehash(X). -** Hence: k2 = spellfix1_phonehash(k1) -** -** There is also a function for computing the Wagner edit distance or the -** Levenshtein distance between a pattern and a word. This function -** is exposed as spellfix1_editdist(X,Y). The edit distance function -** returns the "cost" of converting X into Y. Some transformations -** cost more than others. Changing one vowel into a different vowel, -** for example is relatively cheap, as is doubling a constant, or -** omitting the second character of a double-constant. Other transformations -** or more expensive. The idea is that the edit distance function returns -** a low cost of words that are similar and a higher cost for words -** that are futher apart. In this implementation, the maximum cost -** of any single-character edit (delete, insert, or substitute) is 100, -** with lower costs for some edits (such as transforming vowels). -** -** The "score" for a comparison is the edit distance between the pattern -** and the word, adjusted down by the base-2 logorithm of the word rank. -** For example, a match with distance 100 but rank 1000 would have a -** score of 122 (= 100 - log2(1000) + 32) where as a match with distance -** 100 with a rank of 1 would have a score of 131 (100 - log2(1) + 32). -** (NB: The constant 32 is added to each score to keep it from going -** negative in case the edit distance is zero.) In this way, frequently -** used words get a slightly lower cost which tends to move them toward -** the top of the list of alternative spellings. -** -** A straightforward implementation of a spelling corrector would be -** to compare the search term against every word in the vocabulary -** and select the 20 with the lowest scores. However, there will -** typically be hundreds of thousands or millions of words in the -** vocabulary, and so this approach is not fast enough. -** -** Suppose the term that is being spell-corrected is X. To limit -** the search space, X is converted to a k2-like key using the -** equivalent of: -** -** key = spellfix1_phonehash(lower(spellfix1_translit(X))) -** -** This key is then limited to "scope" characters. The default scope -** value is 4, but an alternative scope can be specified using the -** "scope=N" term in the WHERE clause. After the key has been truncated, -** the edit distance is run against every term in the vocabulary that -** has a k2 value that begins with the abbreviated key. -** -** For example, suppose the input word is "Paskagula". The phonetic -** key is "BACACALA" which is then truncated to 4 characters "BACA". -** The edit distance is then run on the 4980 entries (out of -** 272,597 entries total) of the vocabulary whose k2 values begin with -** BACA, yielding "Pascagoula" as the best match. -** -** Only terms of the vocabulary with a matching langid are searched. -** Hence, the same table can contain entries from multiple languages -** and only the requested language will be used. The default langid -** is 0. -** -** DEALING WITH UNUSUAL AND DIFFICULT SPELLINGS -** -** The algorithm above works quite well for most cases, but there are -** exceptions. These exceptions can be dealt with by making additional -** entries in the virtual table using the "soundslike" column. -** -** For example, many words of Greek origin begin with letters "ps" where -** the "p" is silent. Ex: psalm, pseudonym, psoriasis, psyche. In -** another example, many Scottish surnames can be spelled with an -** initial "Mac" or "Mc". Thus, "MacKay" and "McKay" are both pronounced -** the same. -** -** Accommodation can be made for words that are not spelled as they -** sound by making additional entries into the virtual table for the -** same word, but adding an alternative spelling in the "soundslike" -** column. For example, the canonical entry for "psalm" would be this: -** -** INSERT INTO demo(word) VALUES('psalm'); -** -** To enhance the ability to correct the spelling of "salm" into -** "psalm", make an addition entry like this: -** -** INSERT INTO demo(word,soundslike) VALUES('psalm','salm'); -** -** It is ok to make multiple entries for the same word as long as -** each entry has a different soundslike value. Note that if no -** soundslike value is specified, the soundslike defaults to the word -** itself. -** -** Listed below are some cases where it might make sense to add additional -** soundslike entries. The specific entries will depend on the application -** and the target language. -** -** * Silent "p" in words beginning with "ps": psalm, psyche -** -** * Silent "p" in words beginning with "pn": pneumonia, pneumatic -** -** * Silent "p" in words beginning with "pt": pterodactyl, ptolemaic -** -** * Silent "d" in words beginning with "dj": djinn, Djikarta -** -** * Silent "k" in words beginning with "kn": knight, Knuthson -** -** * Silent "g" in words beginning with "gn": gnarly, gnome, gnat -** -** * "Mac" versus "Mc" beginning Scottish surnames -** -** * "Tch" sounds in Slavic words: Tchaikovsky vs. Chaykovsky -** -** * The letter "j" pronounced like "h" in Spanish: LaJolla -** -** * Words beginning with "wr" versus "r": write vs. rite -** -** * Miscellanous problem words such as "debt", "tsetse", -** "Nguyen", "Van Nuyes". +** This module implements the spellfix1 VIRTUAL TABLE that can be used +** to search a large vocabulary for close matches. See separate +** documentation files (spellfix1.wiki and editdist3.wiki) for details. */ #if SQLITE_CORE # include "sqliteInt.h" @@ -306,7 +39,7 @@ ** 8 'M' Nasals: M N ** 9 'W' Letter W at the beginning of a word ** 10 'Y' Letter Y at the beginning of a word. -** 11 '9' A digit: 0 1 2 3 4 5 6 7 8 9 +** 11 '9' Digits: 0 1 2 3 4 5 6 7 8 9 ** 12 ' ' White space ** 13 '?' Other. */ @@ -466,7 +199,6 @@ static unsigned char *phoneticHash(const unsigned char *zIn, int nIn){ } } } - if( zIn[0]=='k' && zIn[1]=='n' ){ zIn++, nIn--; } for(i=0; i=CCLASS_B && classFrom<=CCLASS_Y && classTo>=CCLASS_B && classTo<=CCLASS_Y ){ @@ -616,7 +348,7 @@ static int substituteCost(char cPrev, char cFrom, char cTo){ ** If zA does end in a '*', then it is the number of bytes in the prefix ** of zB that was deemed to match zA. */ -static int editdist1(const char *zA, const char *zB, int iLangId, int *pnMatch){ +static int editdist1(const char *zA, const char *zB, int *pnMatch){ int nA, nB; /* Number of characters in zA[] and zB[] */ int xA, xB; /* Loop counters for zA[] and zB[] */ char cA, cB; /* Current character of zA and zB */ @@ -645,10 +377,10 @@ static int editdist1(const char *zA, const char *zB, int iLangId, int *pnMatch){ /* Verify input strings and measure their lengths */ for(nA=0; zA[nA]; nA++){ - if( zA[nA]>127 ) return -2; + if( zA[nA]&0x80 ) return -2; } for(nB=0; zB[nB]; nB++){ - if( zB[nB]>127 ) return -2; + if( zB[nB]&0x80 ) return -2; } /* Special processing if either string is empty */ @@ -756,7 +488,9 @@ static int editdist1(const char *zA, const char *zB, int iLangId, int *pnMatch){ } }else{ res = m[nB]; - if( pnMatch ) *pnMatch = -1; + /* In the current implementation, pnMatch is always NULL if zA does + ** not end in "*" */ + assert( pnMatch==0 ); } sqlite3_free(toFree); return res; @@ -764,7 +498,6 @@ static int editdist1(const char *zA, const char *zB, int iLangId, int *pnMatch){ /* ** Function: editdist(A,B) -** editdist(A,B,langid) ** ** Return the cost of transforming string A into string B. Both strings ** must be pure ASCII text. If A ends with '*' then it is assumed to be @@ -776,11 +509,10 @@ static void editdistSqlFunc( int argc, sqlite3_value **argv ){ - int langid = argc==2 ? 0 : sqlite3_value_int(argv[2]); int res = editdist1( (const char*)sqlite3_value_text(argv[0]), (const char*)sqlite3_value_text(argv[1]), - langid, 0); + 0); if( res<0 ){ if( res==(-3) ){ sqlite3_result_error_nomem(context); @@ -924,7 +656,7 @@ static int editDist3ConfigLoad( const char *zTable /* Name of the table from which to load */ ){ sqlite3_stmt *pStmt; - int rc; + int rc, rc2; char *zSql; int iLangPrev = -9999; EditDist3Lang *pLang; @@ -939,24 +671,26 @@ static int editDist3ConfigLoad( while( sqlite3_step(pStmt)==SQLITE_ROW ){ int iLang = sqlite3_column_int(pStmt, 0); const char *zFrom = (const char*)sqlite3_column_text(pStmt, 1); - int nFrom = sqlite3_column_bytes(pStmt, 1); + int nFrom = zFrom ? sqlite3_column_bytes(pStmt, 1) : 0; const char *zTo = (const char*)sqlite3_column_text(pStmt, 2); - int nTo = sqlite3_column_bytes(pStmt, 2); + int nTo = zTo ? sqlite3_column_bytes(pStmt, 2) : 0; int iCost = sqlite3_column_int(pStmt, 3); - if( nFrom>100 || nFrom<0 || nTo>100 || nTo<0 ) continue; + assert( zFrom!=0 || nFrom==0 ); + assert( zTo!=0 || nTo==0 ); + if( nFrom>100 || nTo>100 ) continue; if( iCost<0 ) continue; if( iLang!=iLangPrev ){ EditDist3Lang *pNew; - p->nLang++; - pNew = sqlite3_realloc(p->a, p->nLang*sizeof(p->a[0])); + pNew = sqlite3_realloc(p->a, (p->nLang+1)*sizeof(p->a[0])); if( pNew==0 ){ rc = SQLITE_NOMEM; break; } p->a = pNew; - pLang = &p->a[p->nLang-1]; + pLang = &p->a[p->nLang]; + p->nLang++; pLang->iLang = iLang; pLang->iInsCost = 100; pLang->iDelCost = 100; - pLang->iSubCost = 200; + pLang->iSubCost = 150; pLang->pCost = 0; iLangPrev = iLang; } @@ -981,7 +715,8 @@ static int editDist3ConfigLoad( pLang->pCost = pCost; } } - sqlite3_finalize(pStmt); + rc2 = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ) rc = rc2; return rc; } @@ -1019,7 +754,7 @@ static int matchTo(EditDist3Cost *p, const char *z, int n){ ** the given string. */ static int matchFrom(EditDist3Cost *p, const char *z, int n){ - if( p->nFrom>n ) return 0; + assert( p->nFrom<=n ); if( memcmp(p->a, z, p->nFrom)!=0 ) return 0; return 1; } @@ -1066,10 +801,12 @@ static EditDist3FromString *editDist3FromStringNew( EditDist3Cost *p; int i; + if( z==0 ) return 0; if( n<0 ) n = (int)strlen(z); pStr = sqlite3_malloc( sizeof(*pStr) + sizeof(pStr->a[0])*n + n + 1 ); if( pStr==0 ) return 0; pStr->a = (EditDist3From*)&pStr[1]; + memset(pStr->a, 0, sizeof(pStr->a[0])*n); pStr->n = n; pStr->z = (char*)&pStr->a[n]; memcpy(pStr->z, z, n+1); @@ -1113,30 +850,6 @@ static EditDist3FromString *editDist3FromStringNew( return pStr; } -#if 0 /* No longer used */ -/* -** Return the number of bytes in the common prefix of two UTF8 strings. -** Only complete characters are considered. -*/ -static int editDist3PrefixLen(const char *z1, const char *z2){ - int n = 0; - while( z1[n] && z1[n]==z2[n] ){ n++; } - while( n && (z1[n]&0xc0)==0x80 ){ n--; } - return n; -} - -/* -** Return the number of bytes in the common suffix of two UTF8 strings. -** Only complete characters are considered. -*/ -static int editDist3SuffixLen(const char *z1, int n1, const char *z2, int n2){ - int origN1 = n1; - while( n1>0 && n2>0 && z1[n1-1]==z2[n2-1] ){ n1--; n2--; } - while( n10. */ + if( NEVER(n==0) ){ c = i = 0; }else{ c = z[0]; @@ -1880,10 +1583,10 @@ static const struct { */ static unsigned char *transliterate(const unsigned char *zIn, int nIn){ unsigned char *zOut = sqlite3_malloc( nIn*4 + 1 ); - int i, c, sz, nOut; + int c, sz, nOut; if( zOut==0 ) return 0; - i = nOut = 0; - while( i0 ){ c = utf8Read(zIn, nIn, &sz); zIn += sz; nIn -= sz; @@ -2035,128 +1738,6 @@ static void scriptCodeSqlFunc( /* End transliterate ****************************************************************************** ****************************************************************************** -** Begin Polloc & Zamora SPEEDCOP style keying functions. -*/ -/* -** The Pollock & Zamora skeleton function. Move all consonants to the -** front and all vowels to the end, removing duplicates. Except if the -** first letter is a vowel then it remains as the first letter. -*/ -static void pollockSkeletonKey(const char *zIn, char *zOut){ - int i, j; - unsigned char c; - char seen[26]; - static const unsigned char isVowel[] = { 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, - 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0 }; - memset(seen, 0, sizeof(seen)); - for(i=j=0; (c = (unsigned char)zIn[i])!=0; i++){ - if( c<'a' || c>'z' ) continue; - if( j>0 || isVowel[c-'a'] ) continue; - if( seen[c-'a'] ) continue; - seen[c-'a'] = 1; - zOut[j++] = c; - } - for(i=0; (c = (unsigned char)zIn[i])!=0; i++){ - if( c<'a' || c>'z' ) continue; - if( seen[c-'a'] ) continue; - if( !isVowel[c-'a'] ) continue; - seen[c-'a'] = 1; - zOut[j++] = c; - } - zOut[j] = 0; -} - -/* -** Function: pollock_skeleton(X) -** -** Return the Pollock and Zamora skeleton key for a string X of all -** lower-case letters. -*/ -static void pollockSkeletonSqlFunc( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - const char *zIn = (const char*)sqlite3_value_text(argv[0]); - int nIn = sqlite3_value_bytes(argv[0]); - char *zOut; - if( zIn ){ - zOut = sqlite3_malloc( nIn + 1 ); - if( zOut==0 ){ - sqlite3_result_error_nomem(context); - }else{ - pollockSkeletonKey(zIn, zOut); - sqlite3_result_text(context, (char*)zOut, -1, sqlite3_free); - } - } -} - -/* -** The Pollock & Zamora omission key. -** -** The key consists of unique consonants in the following order: -** -** jkqxzvwybfmgpdhclntsr -** -** These are followed by unique vowels in input order. -*/ -static void pollockOmissionKey(const char *zIn, char *zOut){ - int i, j; - unsigned char c; - char seen[26]; - static const unsigned char isVowel[] = { 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, - 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0 }; - static const unsigned char constOrder[] = "jkqxzvwybfmgpdhclntsr"; - - memset(seen, 0, sizeof(seen)); - for(i=j=0; (c = (unsigned char)zIn[i])!=0; i++){ - if( c<'a' || c>'z' ) continue; - if( isVowel[c-'a'] ) continue; - if( seen[c-'a'] ) continue; - seen[c-'a'] = 1; - } - for(i=0; (c = constOrder[i])!=0; i++){ - if( seen[c-'a'] ) zOut[j++] = c; - } - for(i=0; (c = (unsigned char)zIn[i])!=0; i++){ - if( c<'a' || c>'z' ) continue; - if( seen[c-'a'] ) continue; - if( !isVowel[c-'a'] ) continue; - seen[c-'a'] = 1; - zOut[j++] = c; - } - zOut[j] = 0; -} - -/* -** Function: pollock_omission(X) -** -** Return the Pollock and Zamora omission key for a string X of all -** lower-case letters. -*/ -static void pollockOmissionSqlFunc( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - const char *zIn = (const char*)sqlite3_value_text(argv[0]); - int nIn = sqlite3_value_bytes(argv[0]); - char *zOut; - if( zIn ){ - zOut = sqlite3_malloc( nIn + 1 ); - if( zOut==0 ){ - sqlite3_result_error_nomem(context); - }else{ - pollockOmissionKey(zIn, zOut); - sqlite3_result_text(context, (char*)zOut, -1, sqlite3_free); - } - } -} - - -/* End SPEEDCOP keying functions -****************************************************************************** -****************************************************************************** ** Begin spellfix1 virtual table. */ @@ -2164,7 +1745,7 @@ static void pollockOmissionSqlFunc( #define SPELLFIX_MX_HASH 8 /* Maximum number of hash strings to examine per query */ -#define SPELLFIX_MX_RUN 8 +#define SPELLFIX_MX_RUN 1 typedef struct spellfix1_vtab spellfix1_vtab; typedef struct spellfix1_cursor spellfix1_cursor; @@ -2187,10 +1768,11 @@ struct spellfix1_cursor { int nRow; /* Number of rows of content */ int nAlloc; /* Number of allocated rows */ int iRow; /* Current row of content */ - int iLang; /* Value of the lang= constraint */ + int iLang; /* Value of the langid= constraint */ int iTop; /* Value of the top= constraint */ int iScope; /* Value of the scope= constraint */ int nSearch; /* Number of vocabulary items checked */ + sqlite3_stmt *pFullScan; /* Shadow query for a full table scan */ struct spellfix1_row { /* For each row of content */ sqlite3_int64 iRowid; /* Rowid for this row */ char *zWord; /* Text for this row */ @@ -2267,11 +1849,13 @@ static char *spellfix1Dequote(const char *zIn){ zOut = sqlite3_mprintf("%s", zIn); if( zOut==0 ) return 0; i = (int)strlen(zOut); +#if 0 /* The parser will never leave spaces at the end */ while( i>0 && isspace(zOut[i-1]) ){ i--; } +#endif zOut[i] = 0; c = zOut[0]; if( c=='\'' || c=='"' ){ - for(i=1, j=0; zOut[i]; i++){ + for(i=1, j=0; ALWAYS(zOut[i]); i++){ zOut[j++] = zOut[i]; if( zOut[i]==c ){ if( zOut[i+1]==c ){ @@ -2311,31 +1895,25 @@ static int spellfix1Init( int rc = SQLITE_OK; int i; - if( argc<3 ){ - *pzErr = sqlite3_mprintf( - "%s: wrong number of CREATE VIRTUAL TABLE arguments", argv[0] - ); - rc = SQLITE_ERROR; + nDbName = strlen(zDbName); + pNew = sqlite3_malloc( sizeof(*pNew) + nDbName + 1); + if( pNew==0 ){ + rc = SQLITE_NOMEM; }else{ - nDbName = strlen(zDbName); - pNew = sqlite3_malloc( sizeof(*pNew) + nDbName + 1); - if( pNew==0 ){ + memset(pNew, 0, sizeof(*pNew)); + pNew->zDbName = (char*)&pNew[1]; + memcpy(pNew->zDbName, zDbName, nDbName+1); + pNew->zTableName = sqlite3_mprintf("%s", zTableName); + pNew->db = db; + if( pNew->zTableName==0 ){ rc = SQLITE_NOMEM; }else{ - memset(pNew, 0, sizeof(*pNew)); - pNew->zDbName = (char*)&pNew[1]; - memcpy(pNew->zDbName, zDbName, nDbName+1); - pNew->zTableName = sqlite3_mprintf("%s", zTableName); - pNew->db = db; - if( pNew->zTableName==0 ){ - rc = SQLITE_NOMEM; - }else{ - rc = sqlite3_declare_vtab(db, - "CREATE TABLE x(word,rank,distance,langid, " - "score, matchlen, phonehash, " - "top HIDDEN, scope HIDDEN, srchcnt HIDDEN, " - "soundslike HIDDEN, command HIDDEN)" - ); + rc = sqlite3_declare_vtab(db, + "CREATE TABLE x(word,rank,distance,langid, " + "score, matchlen, phonehash HIDDEN, " + "top HIDDEN, scope HIDDEN, srchcnt HIDDEN, " + "soundslike HIDDEN, command HIDDEN)" + ); #define SPELLFIX_COL_WORD 0 #define SPELLFIX_COL_RANK 1 #define SPELLFIX_COL_DISTANCE 2 @@ -2348,39 +1926,44 @@ static int spellfix1Init( #define SPELLFIX_COL_SRCHCNT 9 #define SPELLFIX_COL_SOUNDSLIKE 10 #define SPELLFIX_COL_COMMAND 11 + } + if( rc==SQLITE_OK && isCreate ){ + sqlite3_uint64 r; + spellfix1DbExec(&rc, db, + "CREATE TABLE IF NOT EXISTS \"%w\".\"%w_vocab\"(\n" + " id INTEGER PRIMARY KEY,\n" + " rank INT,\n" + " langid INT,\n" + " word TEXT,\n" + " k1 TEXT,\n" + " k2 TEXT\n" + ");\n", + zDbName, zTableName + ); + sqlite3_randomness(sizeof(r), &r); + spellfix1DbExec(&rc, db, + "CREATE INDEX IF NOT EXISTS \"%w\".\"%w_index_%llx\" " + "ON \"%w_vocab\"(langid,k2);", + zDbName, zModule, r, zTableName + ); + } + for(i=3; rc==SQLITE_OK && ibase); + }else{ + *ppVTab = (sqlite3_vtab *)pNew; + } return rc; } @@ -2417,6 +2000,10 @@ static void spellfix1ResetCursor(spellfix1_cursor *pCur){ pCur->nRow = 0; pCur->iRow = 0; pCur->nSearch = 0; + if( pCur->pFullScan ){ + sqlite3_finalize(pCur->pFullScan); + pCur->pFullScan = 0; + } } /* @@ -2535,7 +2122,7 @@ static int spellfix1BestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ int idx = 2; pIdxInfo->idxNum = iPlan; if( pIdxInfo->nOrderBy==1 - && pIdxInfo->aOrderBy[0].iColumn==4 + && pIdxInfo->aOrderBy[0].iColumn==SPELLFIX_COL_SCORE && pIdxInfo->aOrderBy[0].desc==0 ){ pIdxInfo->orderByConsumed = 1; /* Default order by iScore */ @@ -2640,9 +2227,9 @@ static void spellfix1RunQuery(MatchQuery *p, const char *zQuery, int nQuery){ char zHash2[SPELLFIX_MX_HASH]; char *zClass; int nClass; + int rc; if( pCur->a==0 || p->rc ) return; /* Prior memory allocation failure */ - if( p->nRun>=SPELLFIX_MX_RUN ) return; zClass = (char*)phoneticHash((unsigned char*)zQuery, nQuery); if( zClass==0 ){ p->rc = SQLITE_NOMEM; @@ -2666,18 +2253,27 @@ static void spellfix1RunQuery(MatchQuery *p, const char *zQuery, int nQuery){ memcpy(zHash2, zHash1, iScope); zHash2[iScope] = 'Z'; zHash2[iScope+1] = 0; +#if SPELLFIX_MX_RUN>1 for(i=0; inRun; i++){ if( strcmp(p->azPrior[i], zHash1)==0 ) return; } +#endif + assert( p->nRunazPrior[p->nRun++], zHash1, iScope+1); - sqlite3_bind_text(pStmt, 1, zHash1, -1, SQLITE_STATIC); - sqlite3_bind_text(pStmt, 2, zHash2, -1, SQLITE_STATIC); + if( sqlite3_bind_text(pStmt, 1, zHash1, -1, SQLITE_STATIC)==SQLITE_NOMEM + || sqlite3_bind_text(pStmt, 2, zHash2, -1, SQLITE_STATIC)==SQLITE_NOMEM + ){ + p->rc = SQLITE_NOMEM; + return; + } +#if SPELLFIX_MX_RUN>1 for(i=0; inRow; i++){ if( pCur->a[i].iScore>iWorst ){ iWorst = pCur->a[i].iScore; idxWorst = i; } } +#endif while( sqlite3_step(pStmt)==SQLITE_ROW ){ int iMatchlen = -1; iRank = sqlite3_column_int(pStmt, 2); @@ -2688,7 +2284,11 @@ static void spellfix1RunQuery(MatchQuery *p, const char *zQuery, int nQuery){ }else{ zK1 = (const char*)sqlite3_column_text(pStmt, 3); if( zK1==0 ) continue; - iDist = editdist1(p->zPattern, zK1, pCur->iLang, 0); + iDist = editdist1(p->zPattern, zK1, 0); + } + if( iDist<0 ){ + p->rc = SQLITE_NOMEM; + break; } pCur->nSearch++; iScore = spellfix1Score(iDist,iRank); @@ -2708,6 +2308,10 @@ static void spellfix1RunQuery(MatchQuery *p, const char *zQuery, int nQuery){ continue; } pCur->a[idx].zWord = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 1)); + if( pCur->a[idx].zWord==0 ){ + p->rc = SQLITE_NOMEM; + break; + } pCur->a[idx].iRowid = sqlite3_column_int64(pStmt, 0); pCur->a[idx].iRank = iRank; pCur->a[idx].iDistance = iDist; @@ -2727,7 +2331,8 @@ static void spellfix1RunQuery(MatchQuery *p, const char *zQuery, int nQuery){ } } } - sqlite3_reset(pStmt); + rc = sqlite3_reset(pStmt); + if( rc ) p->rc = rc; } /* @@ -2748,7 +2353,7 @@ static int spellfix1FilterForMatch( int iScope = 3; /* Use this many characters of zClass */ int iLang = 0; /* Language code */ char *zSql; /* SQL of shadow table query */ - sqlite3_stmt *pStmt; /* Shadow table query */ + sqlite3_stmt *pStmt = 0; /* Shadow table query */ int rc; /* Result code */ int idx = 1; /* Next available filter parameter */ spellfix1_vtab *p = pCur->pVTab; /* The virtual table that owns pCur */ @@ -2790,13 +2395,20 @@ static int spellfix1FilterForMatch( if( p->pConfig3 ){ x.pLang = editDist3FindLang(p->pConfig3, iLang); pMatchStr3 = editDist3FromStringNew(x.pLang, (const char*)zMatchThis, -1); + if( pMatchStr3==0 ){ + x.rc = SQLITE_NOMEM; + goto filter_exit; + } }else{ x.pLang = 0; } zPattern = (char*)transliterate(zMatchThis, sqlite3_value_bytes(argv[0])); sqlite3_free(pCur->zPattern); pCur->zPattern = zPattern; - if( zPattern==0 ) return SQLITE_NOMEM; + if( zPattern==0 ){ + x.rc = SQLITE_NOMEM; + goto filter_exit; + } nPattern = strlen(zPattern); if( zPattern[nPattern-1]=='*' ) nPattern--; zSql = sqlite3_mprintf( @@ -2805,6 +2417,11 @@ static int spellfix1FilterForMatch( " WHERE langid=%d AND k2>=?1 AND k2zDbName, p->zTableName, iLang ); + if( zSql==0 ){ + x.rc = SQLITE_NOMEM; + pStmt = 0; + goto filter_exit; + } rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); sqlite3_free(zSql); pCur->iLang = iLang; @@ -2820,34 +2437,18 @@ static int spellfix1FilterForMatch( spellfix1RunQuery(&x, zPattern, nPattern); } -#if 0 - /* Convert "ght" to "t" in the original pattern and try again */ - if( x.rc==SQLITE_OK ){ - int i, j; /* Loop counters */ - char zQuery[50]; /* Space for alternative query string */ - for(i=j=0; ia ){ qsort(pCur->a, pCur->nRow, sizeof(pCur->a[0]), spellfix1RowCompare); pCur->iTop = iLimit; pCur->iScope = iScope; + }else{ + x.rc = SQLITE_NOMEM; } + +filter_exit: sqlite3_finalize(pStmt); editDist3FromStringDelete(pMatchStr3); - return pCur->a ? x.rc : SQLITE_NOMEM; + return x.rc; } /* @@ -2859,9 +2460,25 @@ static int spellfix1FilterForFullScan( int argc, sqlite3_value **argv ){ + int rc; + char *zSql; + spellfix1_vtab *pVTab = pCur->pVTab; spellfix1ResetCursor(pCur); - spellfix1ResizeCursor(pCur, 0); - return SQLITE_OK; + zSql = sqlite3_mprintf( + "SELECT word, rank, NULL, langid, id FROM \"%w\".\"%w_vocab\"", + pVTab->zDbName, pVTab->zTableName); + if( zSql==0 ) return SQLITE_NOMEM; + rc = sqlite3_prepare_v2(pVTab->db, zSql, -1, &pCur->pFullScan, 0); + sqlite3_free(zSql); + pCur->nRow = pCur->iRow = 0; + if( rc==SQLITE_OK ){ + rc = sqlite3_step(pCur->pFullScan); + if( rc==SQLITE_ROW ){ pCur->iRow = -1; rc = SQLITE_OK; } + if( rc==SQLITE_DONE ){ rc = SQLITE_OK; } + }else{ + pCur->iRow = 0; + } + return rc; } @@ -2891,7 +2508,14 @@ static int spellfix1Filter( */ static int spellfix1Next(sqlite3_vtab_cursor *cur){ spellfix1_cursor *pCur = (spellfix1_cursor *)cur; - if( pCur->iRow < pCur->nRow ) pCur->iRow++; + if( pCur->iRow < pCur->nRow ){ + if( pCur->pFullScan ){ + int rc = sqlite3_step(pCur->pFullScan); + if( rc!=SQLITE_ROW ) pCur->iRow = pCur->nRow; + }else{ + pCur->iRow++; + } + } return SQLITE_OK; } @@ -2906,8 +2530,20 @@ static int spellfix1Eof(sqlite3_vtab_cursor *cur){ /* ** Return columns from the current row. */ -static int spellfix1Column(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){ +static int spellfix1Column( + sqlite3_vtab_cursor *cur, + sqlite3_context *ctx, + int i +){ spellfix1_cursor *pCur = (spellfix1_cursor*)cur; + if( pCur->pFullScan ){ + if( i<=SPELLFIX_COL_LANGID ){ + sqlite3_result_value(ctx, sqlite3_column_value(pCur->pFullScan, i)); + }else{ + sqlite3_result_null(ctx); + } + return SQLITE_OK; + } switch( i ){ case SPELLFIX_COL_WORD: { sqlite3_result_text(ctx, pCur->a[pCur->iRow].zWord, -1, SQLITE_STATIC); @@ -2941,7 +2577,7 @@ static int spellfix1Column(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i int res; zTranslit = (char *)transliterate((unsigned char *)zWord, nWord); if( !zTranslit ) return SQLITE_NOMEM; - res = editdist1(pCur->zPattern, zTranslit, pCur->iLang, &iMatchlen); + res = editdist1(pCur->zPattern, zTranslit, &iMatchlen); sqlite3_free(zTranslit); if( res<0 ) return SQLITE_NOMEM; iMatchlen = translen_to_charlen(zWord, nWord, iMatchlen); @@ -2982,7 +2618,11 @@ static int spellfix1Column(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i */ static int spellfix1Rowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ spellfix1_cursor *pCur = (spellfix1_cursor*)cur; - *pRowid = pCur->a[pCur->iRow].iRowid; + if( pCur->pFullScan ){ + *pRowid = sqlite3_column_int64(pCur->pFullScan, 4); + }else{ + *pRowid = pCur->a[pCur->iRow].iRowid; + } return SQLITE_OK; } @@ -3066,8 +2706,8 @@ static int spellfix1Update( rowid = sqlite3_value_int64(argv[0]); newRowid = *pRowid = sqlite3_value_int64(argv[1]); spellfix1DbExec(&rc, db, - "UPDATE \"%w\".\"%w_vocab\" SET id=%lld, rank=%d, lang=%d," - " word=%Q, rank=%d, k1=%Q, k2=%Q WHERE id=%lld", + "UPDATE \"%w\".\"%w_vocab\" SET id=%lld, rank=%d, langid=%d," + " word=%Q, k1=%Q, k2=%Q WHERE id=%lld", p->zDbName, p->zTableName, newRowid, iRank, iLang, zWord, zK1, zK2, rowid ); @@ -3096,6 +2736,8 @@ static int spellfix1Rename(sqlite3_vtab *pVTab, const char *zNew){ if( rc==SQLITE_OK ){ sqlite3_free(p->zTableName); p->zTableName = zNewName; + }else{ + sqlite3_free(zNewName); } return rc; } @@ -3137,16 +2779,10 @@ static int spellfix1Register(sqlite3 *db){ transliterateSqlFunc, 0, 0); nErr += sqlite3_create_function(db, "spellfix1_editdist", 2, SQLITE_UTF8, 0, editdistSqlFunc, 0, 0); - nErr += sqlite3_create_function(db, "spellfix1_editdist", 3, SQLITE_UTF8, 0, - editdistSqlFunc, 0, 0); nErr += sqlite3_create_function(db, "spellfix1_phonehash", 1, SQLITE_UTF8, 0, phoneticHashSqlFunc, 0, 0); nErr += sqlite3_create_function(db, "spellfix1_scriptcode", 1, SQLITE_UTF8, 0, scriptCodeSqlFunc, 0, 0); - nErr += sqlite3_create_function(db, "pollock_skeleton", 1, SQLITE_UTF8, 0, - pollockSkeletonSqlFunc, 0, 0); - nErr += sqlite3_create_function(db, "pollock_omission", 1, SQLITE_UTF8, 0, - pollockOmissionSqlFunc, 0, 0); nErr += sqlite3_create_module(db, "spellfix1", &spellfix1Module, 0); nErr += editDist3Install(db); diff --git a/src/test_vfs.c b/src/test_vfs.c index fd2aa9fb07..93c556b56e 100644 --- a/src/test_vfs.c +++ b/src/test_vfs.c @@ -361,7 +361,8 @@ static int tvfsWrite( if( p->pScript && p->mask&TESTVFS_WRITE_MASK ){ tvfsExecTcl(p, "xWrite", - Tcl_NewStringObj(pFd->zFilename, -1), pFd->pShmId, 0 + Tcl_NewStringObj(pFd->zFilename, -1), pFd->pShmId, + Tcl_NewWideIntObj(iOfst) ); tvfsResultCode(p, &rc); } diff --git a/src/vdbe.c b/src/vdbe.c index 19c0255b8f..12e7325143 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -3120,6 +3120,9 @@ case OP_OpenWrite: { VdbeCursor *pCur; Db *pDb; + assert( (pOp->p5&(OPFLAG_P2ISREG|OPFLAG_BULKCSR))==pOp->p5 ); + assert( pOp->opcode==OP_OpenWrite || pOp->p5==0 ); + if( p->expired ){ rc = SQLITE_ABORT; break; @@ -3143,7 +3146,7 @@ case OP_OpenWrite: { }else{ wrFlag = 0; } - if( pOp->p5 ){ + if( pOp->p5 & OPFLAG_P2ISREG ){ assert( p2>0 ); assert( p2<=p->nMem ); pIn2 = &aMem[p2]; @@ -3174,6 +3177,8 @@ case OP_OpenWrite: { pCur->isOrdered = 1; rc = sqlite3BtreeCursor(pX, p2, wrFlag, pKeyInfo, pCur->pCursor); pCur->pKeyInfo = pKeyInfo; + assert( OPFLAG_BULKCSR==BTREE_BULKLOAD ); + sqlite3BtreeCursorHints(pCur->pCursor, (pOp->p5 & OPFLAG_BULKCSR)); /* Since it performs no memory allocation or IO, the only value that ** sqlite3BtreeCursor() may return is SQLITE_OK. */ diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 9c1af35c33..1f5694a595 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -431,11 +431,11 @@ int sqlite3VdbeTransferError(Vdbe *p); #else int sqlite3VdbeSorterInit(sqlite3 *, VdbeCursor *); void sqlite3VdbeSorterClose(sqlite3 *, VdbeCursor *); -int sqlite3VdbeSorterRowkey(VdbeCursor *, Mem *); -int sqlite3VdbeSorterNext(sqlite3 *, VdbeCursor *, int *); -int sqlite3VdbeSorterRewind(sqlite3 *, VdbeCursor *, int *); -int sqlite3VdbeSorterWrite(sqlite3 *, VdbeCursor *, Mem *); -int sqlite3VdbeSorterCompare(VdbeCursor *, Mem *, int *); +int sqlite3VdbeSorterRowkey(const VdbeCursor *, Mem *); +int sqlite3VdbeSorterNext(sqlite3 *, const VdbeCursor *, int *); +int sqlite3VdbeSorterRewind(sqlite3 *, const VdbeCursor *, int *); +int sqlite3VdbeSorterWrite(sqlite3 *, const VdbeCursor *, Mem *); +int sqlite3VdbeSorterCompare(const VdbeCursor *, Mem *, int *); #endif #if !defined(SQLITE_OMIT_SHARED_CACHE) && SQLITE_THREADSAFE>0 diff --git a/src/vdbesort.c b/src/vdbesort.c index afea1f510a..ba1e9f0f23 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -22,6 +22,7 @@ typedef struct VdbeSorterIter VdbeSorterIter; typedef struct SorterRecord SorterRecord; +typedef struct FileWriter FileWriter; /* ** NOTES ON DATA STRUCTURE USED FOR N-WAY MERGES: @@ -119,6 +120,24 @@ struct VdbeSorterIter { sqlite3_file *pFile; /* File iterator is reading from */ u8 *aAlloc; /* Allocated space */ u8 *aKey; /* Pointer to current key */ + u8 *aBuffer; /* Current read buffer */ + int nBuffer; /* Size of read buffer in bytes */ +}; + +/* +** An instance of this structure is used to organize the stream of records +** being written to files by the merge-sort code into aligned, page-sized +** blocks. Doing all I/O in aligned page-sized blocks helps I/O to go +** faster on many operating systems. +*/ +struct FileWriter { + int eFWErr; /* Non-zero if in an error state */ + u8 *aBuffer; /* Pointer to write buffer */ + int nBuffer; /* Size of write buffer in bytes */ + int iBufStart; /* First byte of buffer to write */ + int iBufEnd; /* Last byte of buffer to write */ + i64 iWriteOff; /* Offset of start of buffer in file */ + sqlite3_file *pFile; /* File to write to */ }; /* @@ -144,9 +163,123 @@ struct SorterRecord { */ static void vdbeSorterIterZero(sqlite3 *db, VdbeSorterIter *pIter){ sqlite3DbFree(db, pIter->aAlloc); + sqlite3DbFree(db, pIter->aBuffer); memset(pIter, 0, sizeof(VdbeSorterIter)); } +/* +** Read nByte bytes of data from the stream of data iterated by object p. +** If successful, set *ppOut to point to a buffer containing the data +** and return SQLITE_OK. Otherwise, if an error occurs, return an SQLite +** error code. +** +** The buffer indicated by *ppOut may only be considered valid until the +** next call to this function. +*/ +static int vdbeSorterIterRead( + sqlite3 *db, /* Database handle (for malloc) */ + VdbeSorterIter *p, /* Iterator */ + int nByte, /* Bytes of data to read */ + u8 **ppOut /* OUT: Pointer to buffer containing data */ +){ + int iBuf; /* Offset within buffer to read from */ + int nAvail; /* Bytes of data available in buffer */ + assert( p->aBuffer ); + + /* If there is no more data to be read from the buffer, read the next + ** p->nBuffer bytes of data from the file into it. Or, if there are less + ** than p->nBuffer bytes remaining in the PMA, read all remaining data. */ + iBuf = p->iReadOff % p->nBuffer; + if( iBuf==0 ){ + int nRead; /* Bytes to read from disk */ + int rc; /* sqlite3OsRead() return code */ + + /* Determine how many bytes of data to read. */ + nRead = (int)(p->iEof - p->iReadOff); + if( nRead>p->nBuffer ) nRead = p->nBuffer; + assert( nRead>0 ); + + /* Read data from the file. Return early if an error occurs. */ + rc = sqlite3OsRead(p->pFile, p->aBuffer, nRead, p->iReadOff); + assert( rc!=SQLITE_IOERR_SHORT_READ ); + if( rc!=SQLITE_OK ) return rc; + } + nAvail = p->nBuffer - iBuf; + + if( nByte<=nAvail ){ + /* The requested data is available in the in-memory buffer. In this + ** case there is no need to make a copy of the data, just return a + ** pointer into the buffer to the caller. */ + *ppOut = &p->aBuffer[iBuf]; + p->iReadOff += nByte; + }else{ + /* The requested data is not all available in the in-memory buffer. + ** In this case, allocate space at p->aAlloc[] to copy the requested + ** range into. Then return a copy of pointer p->aAlloc to the caller. */ + int nRem; /* Bytes remaining to copy */ + + /* Extend the p->aAlloc[] allocation if required. */ + if( p->nAllocnAlloc*2; + while( nByte>nNew ) nNew = nNew*2; + p->aAlloc = sqlite3DbReallocOrFree(db, p->aAlloc, nNew); + if( !p->aAlloc ) return SQLITE_NOMEM; + p->nAlloc = nNew; + } + + /* Copy as much data as is available in the buffer into the start of + ** p->aAlloc[]. */ + memcpy(p->aAlloc, &p->aBuffer[iBuf], nAvail); + p->iReadOff += nAvail; + nRem = nByte - nAvail; + + /* The following loop copies up to p->nBuffer bytes per iteration into + ** the p->aAlloc[] buffer. */ + while( nRem>0 ){ + int rc; /* vdbeSorterIterRead() return code */ + int nCopy; /* Number of bytes to copy */ + u8 *aNext; /* Pointer to buffer to copy data from */ + + nCopy = nRem; + if( nRem>p->nBuffer ) nCopy = p->nBuffer; + rc = vdbeSorterIterRead(db, p, nCopy, &aNext); + if( rc!=SQLITE_OK ) return rc; + assert( aNext!=p->aAlloc ); + memcpy(&p->aAlloc[nByte - nRem], aNext, nCopy); + nRem -= nCopy; + } + + *ppOut = p->aAlloc; + } + + return SQLITE_OK; +} + +/* +** Read a varint from the stream of data accessed by p. Set *pnOut to +** the value read. +*/ +static int vdbeSorterIterVarint(sqlite3 *db, VdbeSorterIter *p, u64 *pnOut){ + int iBuf; + + iBuf = p->iReadOff % p->nBuffer; + if( iBuf && (p->nBuffer-iBuf)>=9 ){ + p->iReadOff += sqlite3GetVarint(&p->aBuffer[iBuf], pnOut); + }else{ + u8 aVarint[16], *a; + int i = 0, rc; + do{ + rc = vdbeSorterIterRead(db, p, 1, &a); + if( rc ) return rc; + aVarint[(i++)&0xf] = a[0]; + }while( (a[0]&0x80)!=0 ); + sqlite3GetVarint(aVarint, pnOut); + } + + return SQLITE_OK; +} + + /* ** Advance iterator pIter to the next key in its PMA. Return SQLITE_OK if ** no error occurs, or an SQLite error code if one does. @@ -156,96 +289,18 @@ static int vdbeSorterIterNext( VdbeSorterIter *pIter /* Iterator to advance */ ){ int rc; /* Return Code */ - int nRead; /* Number of bytes read */ - int nRec = 0; /* Size of record in bytes */ - int iOff = 0; /* Size of serialized size varint in bytes */ + u64 nRec = 0; /* Size of record in bytes */ - assert( pIter->iEof>=pIter->iReadOff ); - if( pIter->iEof-pIter->iReadOff>5 ){ - nRead = 5; - }else{ - nRead = (int)(pIter->iEof - pIter->iReadOff); - } - if( nRead<=0 ){ + if( pIter->iReadOff>=pIter->iEof ){ /* This is an EOF condition */ vdbeSorterIterZero(db, pIter); return SQLITE_OK; } - rc = sqlite3OsRead(pIter->pFile, pIter->aAlloc, nRead, pIter->iReadOff); + rc = vdbeSorterIterVarint(db, pIter, &nRec); if( rc==SQLITE_OK ){ - iOff = getVarint32(pIter->aAlloc, nRec); - if( (iOff+nRec)>nRead ){ - int nRead2; /* Number of extra bytes to read */ - if( (iOff+nRec)>pIter->nAlloc ){ - int nNew = pIter->nAlloc*2; - while( (iOff+nRec)>nNew ) nNew = nNew*2; - pIter->aAlloc = sqlite3DbReallocOrFree(db, pIter->aAlloc, nNew); - if( !pIter->aAlloc ) return SQLITE_NOMEM; - pIter->nAlloc = nNew; - } - - nRead2 = iOff + nRec - nRead; - rc = sqlite3OsRead( - pIter->pFile, &pIter->aAlloc[nRead], nRead2, pIter->iReadOff+nRead - ); - } - } - - assert( rc!=SQLITE_OK || nRec>0 ); - pIter->iReadOff += iOff+nRec; - pIter->nKey = nRec; - pIter->aKey = &pIter->aAlloc[iOff]; - return rc; -} - -/* -** Write a single varint, value iVal, to file-descriptor pFile. Return -** SQLITE_OK if successful, or an SQLite error code if some error occurs. -** -** The value of *piOffset when this function is called is used as the byte -** offset in file pFile to write to. Before returning, *piOffset is -** incremented by the number of bytes written. -*/ -static int vdbeSorterWriteVarint( - sqlite3_file *pFile, /* File to write to */ - i64 iVal, /* Value to write as a varint */ - i64 *piOffset /* IN/OUT: Write offset in file pFile */ -){ - u8 aVarint[9]; /* Buffer large enough for a varint */ - int nVarint; /* Number of used bytes in varint */ - int rc; /* Result of write() call */ - - nVarint = sqlite3PutVarint(aVarint, iVal); - rc = sqlite3OsWrite(pFile, aVarint, nVarint, *piOffset); - *piOffset += nVarint; - - return rc; -} - -/* -** Read a single varint from file-descriptor pFile. Return SQLITE_OK if -** successful, or an SQLite error code if some error occurs. -** -** The value of *piOffset when this function is called is used as the -** byte offset in file pFile from whence to read the varint. If successful -** (i.e. if no IO error occurs), then *piOffset is set to the offset of -** the first byte past the end of the varint before returning. *piVal is -** set to the integer value read. If an error occurs, the final values of -** both *piOffset and *piVal are undefined. -*/ -static int vdbeSorterReadVarint( - sqlite3_file *pFile, /* File to read from */ - i64 *piOffset, /* IN/OUT: Read offset in pFile */ - i64 *piVal /* OUT: Value read from file */ -){ - u8 aVarint[9]; /* Buffer large enough for a varint */ - i64 iOff = *piOffset; /* Offset in file to read from */ - int rc; /* Return code */ - - rc = sqlite3OsRead(pFile, aVarint, 9, iOff); - if( rc==SQLITE_OK ){ - *piOffset += getVarint(aVarint, (u64 *)piVal); + pIter->nKey = (int)nRec; + rc = vdbeSorterIterRead(db, pIter, (int)nRec, &pIter->aKey); } return rc; @@ -259,27 +314,52 @@ static int vdbeSorterReadVarint( */ static int vdbeSorterIterInit( sqlite3 *db, /* Database handle */ - VdbeSorter *pSorter, /* Sorter object */ + const VdbeSorter *pSorter, /* Sorter object */ i64 iStart, /* Start offset in pFile */ VdbeSorterIter *pIter, /* Iterator to populate */ i64 *pnByte /* IN/OUT: Increment this value by PMA size */ ){ - int rc; + int rc = SQLITE_OK; + int nBuf; + + nBuf = sqlite3BtreeGetPageSize(db->aDb[0].pBt); assert( pSorter->iWriteOff>iStart ); assert( pIter->aAlloc==0 ); + assert( pIter->aBuffer==0 ); pIter->pFile = pSorter->pTemp1; pIter->iReadOff = iStart; pIter->nAlloc = 128; pIter->aAlloc = (u8 *)sqlite3DbMallocRaw(db, pIter->nAlloc); - if( !pIter->aAlloc ){ + pIter->nBuffer = nBuf; + pIter->aBuffer = (u8 *)sqlite3DbMallocRaw(db, nBuf); + + if( !pIter->aBuffer ){ rc = SQLITE_NOMEM; }else{ - i64 nByte; /* Total size of PMA in bytes */ - rc = vdbeSorterReadVarint(pSorter->pTemp1, &pIter->iReadOff, &nByte); - *pnByte += nByte; - pIter->iEof = pIter->iReadOff + nByte; + int iBuf; + + iBuf = iStart % nBuf; + if( iBuf ){ + int nRead = nBuf - iBuf; + if( (iStart + nRead) > pSorter->iWriteOff ){ + nRead = (int)(pSorter->iWriteOff - iStart); + } + rc = sqlite3OsRead( + pSorter->pTemp1, &pIter->aBuffer[iBuf], nRead, iStart + ); + assert( rc!=SQLITE_IOERR_SHORT_READ ); + } + + if( rc==SQLITE_OK ){ + u64 nByte; /* Size of PMA in bytes */ + pIter->iEof = pSorter->iWriteOff; + rc = vdbeSorterIterVarint(db, pIter, &nByte); + pIter->iEof = pIter->iReadOff + nByte; + *pnByte += nByte; + } } + if( rc==SQLITE_OK ){ rc = vdbeSorterIterNext(db, pIter); } @@ -303,10 +383,10 @@ static int vdbeSorterIterInit( ** has been allocated and contains an unpacked record that is used as key2. */ static void vdbeSorterCompare( - VdbeCursor *pCsr, /* Cursor object (for pKeyInfo) */ + const VdbeCursor *pCsr, /* Cursor object (for pKeyInfo) */ int bOmitRowid, /* Ignore rowid field at end of keys */ - void *pKey1, int nKey1, /* Left side of comparison */ - void *pKey2, int nKey2, /* Right side of comparison */ + const void *pKey1, int nKey1, /* Left side of comparison */ + const void *pKey2, int nKey2, /* Right side of comparison */ int *pRes /* OUT: Result of comparison */ ){ KeyInfo *pKeyInfo = pCsr->pKeyInfo; @@ -338,7 +418,7 @@ static void vdbeSorterCompare( ** multiple b-tree segments. Parameter iOut is the index of the aTree[] ** value to recalculate. */ -static int vdbeSorterDoCompare(VdbeCursor *pCsr, int iOut){ +static int vdbeSorterDoCompare(const VdbeCursor *pCsr, int iOut){ VdbeSorter *pSorter = pCsr->pSorter; int i1; int i2; @@ -464,7 +544,7 @@ static int vdbeSorterOpenTempFile(sqlite3 *db, sqlite3_file **ppFile){ ** Set *ppOut to the head of the new list. */ static void vdbeSorterMerge( - VdbeCursor *pCsr, /* For pKeyInfo */ + const VdbeCursor *pCsr, /* For pKeyInfo */ SorterRecord *p1, /* First list to merge */ SorterRecord *p2, /* Second list to merge */ SorterRecord **ppOut /* OUT: Head of merged list */ @@ -498,7 +578,7 @@ static void vdbeSorterMerge( ** if successful, or an SQLite error code (i.e. SQLITE_NOMEM) if an error ** occurs. */ -static int vdbeSorterSort(VdbeCursor *pCsr){ +static int vdbeSorterSort(const VdbeCursor *pCsr){ int i; SorterRecord **aSlot; SorterRecord *p; @@ -531,6 +611,91 @@ static int vdbeSorterSort(VdbeCursor *pCsr){ return SQLITE_OK; } +/* +** Initialize a file-writer object. +*/ +static void fileWriterInit( + sqlite3 *db, /* Database (for malloc) */ + sqlite3_file *pFile, /* File to write to */ + FileWriter *p, /* Object to populate */ + i64 iStart /* Offset of pFile to begin writing at */ +){ + int nBuf = sqlite3BtreeGetPageSize(db->aDb[0].pBt); + + memset(p, 0, sizeof(FileWriter)); + p->aBuffer = (u8 *)sqlite3DbMallocRaw(db, nBuf); + if( !p->aBuffer ){ + p->eFWErr = SQLITE_NOMEM; + }else{ + p->iBufEnd = p->iBufStart = (iStart % nBuf); + p->iWriteOff = iStart - p->iBufStart; + p->nBuffer = nBuf; + p->pFile = pFile; + } +} + +/* +** Write nData bytes of data to the file-write object. Return SQLITE_OK +** if successful, or an SQLite error code if an error occurs. +*/ +static void fileWriterWrite(FileWriter *p, u8 *pData, int nData){ + int nRem = nData; + while( nRem>0 && p->eFWErr==0 ){ + int nCopy = nRem; + if( nCopy>(p->nBuffer - p->iBufEnd) ){ + nCopy = p->nBuffer - p->iBufEnd; + } + + memcpy(&p->aBuffer[p->iBufEnd], &pData[nData-nRem], nCopy); + p->iBufEnd += nCopy; + if( p->iBufEnd==p->nBuffer ){ + p->eFWErr = sqlite3OsWrite(p->pFile, + &p->aBuffer[p->iBufStart], p->iBufEnd - p->iBufStart, + p->iWriteOff + p->iBufStart + ); + p->iBufStart = p->iBufEnd = 0; + p->iWriteOff += p->nBuffer; + } + assert( p->iBufEndnBuffer ); + + nRem -= nCopy; + } +} + +/* +** Flush any buffered data to disk and clean up the file-writer object. +** The results of using the file-writer after this call are undefined. +** Return SQLITE_OK if flushing the buffered data succeeds or is not +** required. Otherwise, return an SQLite error code. +** +** Before returning, set *piEof to the offset immediately following the +** last byte written to the file. +*/ +static int fileWriterFinish(sqlite3 *db, FileWriter *p, i64 *piEof){ + int rc; + if( p->eFWErr==0 && ALWAYS(p->aBuffer) && p->iBufEnd>p->iBufStart ){ + p->eFWErr = sqlite3OsWrite(p->pFile, + &p->aBuffer[p->iBufStart], p->iBufEnd - p->iBufStart, + p->iWriteOff + p->iBufStart + ); + } + *piEof = (p->iWriteOff + p->iBufEnd); + sqlite3DbFree(db, p->aBuffer); + rc = p->eFWErr; + memset(p, 0, sizeof(FileWriter)); + return rc; +} + +/* +** Write value iVal encoded as a varint to the file-write object. Return +** SQLITE_OK if successful, or an SQLite error code if an error occurs. +*/ +static void fileWriterWriteVarint(FileWriter *p, u64 iVal){ + int nByte; + u8 aByte[10]; + nByte = sqlite3PutVarint(aByte, iVal); + fileWriterWrite(p, aByte, nByte); +} /* ** Write the current contents of the in-memory linked-list to a PMA. Return @@ -545,9 +710,12 @@ static int vdbeSorterSort(VdbeCursor *pCsr){ ** Each record consists of a varint followed by a blob of data (the ** key). The varint is the number of bytes in the blob of data. */ -static int vdbeSorterListToPMA(sqlite3 *db, VdbeCursor *pCsr){ +static int vdbeSorterListToPMA(sqlite3 *db, const VdbeCursor *pCsr){ int rc = SQLITE_OK; /* Return code */ VdbeSorter *pSorter = pCsr->pSorter; + FileWriter writer; + + memset(&writer, 0, sizeof(FileWriter)); if( pSorter->nInMemory==0 ){ assert( pSorter->pRecord==0 ); @@ -565,39 +733,20 @@ static int vdbeSorterListToPMA(sqlite3 *db, VdbeCursor *pCsr){ } if( rc==SQLITE_OK ){ - i64 iOff = pSorter->iWriteOff; SorterRecord *p; SorterRecord *pNext = 0; - static const char eightZeros[8] = { 0, 0, 0, 0, 0, 0, 0, 0 }; + fileWriterInit(db, pSorter->pTemp1, &writer, pSorter->iWriteOff); pSorter->nPMA++; - rc = vdbeSorterWriteVarint(pSorter->pTemp1, pSorter->nInMemory, &iOff); - for(p=pSorter->pRecord; rc==SQLITE_OK && p; p=pNext){ + fileWriterWriteVarint(&writer, pSorter->nInMemory); + for(p=pSorter->pRecord; p; p=pNext){ pNext = p->pNext; - rc = vdbeSorterWriteVarint(pSorter->pTemp1, p->nVal, &iOff); - - if( rc==SQLITE_OK ){ - rc = sqlite3OsWrite(pSorter->pTemp1, p->pVal, p->nVal, iOff); - iOff += p->nVal; - } - + fileWriterWriteVarint(&writer, p->nVal); + fileWriterWrite(&writer, p->pVal, p->nVal); sqlite3DbFree(db, p); } - - /* This assert verifies that unless an error has occurred, the size of - ** the PMA on disk is the same as the expected size stored in - ** pSorter->nInMemory. */ - assert( rc!=SQLITE_OK || pSorter->nInMemory==( - iOff-pSorter->iWriteOff-sqlite3VarintLen(pSorter->nInMemory) - )); - - pSorter->iWriteOff = iOff; - if( rc==SQLITE_OK ){ - /* Terminate each file with 8 extra bytes so that from any offset - ** in the file we can always read 9 bytes without a SHORT_READ error */ - rc = sqlite3OsWrite(pSorter->pTemp1, eightZeros, 8, iOff); - } pSorter->pRecord = p; + rc = fileWriterFinish(db, &writer, &pSorter->iWriteOff); } return rc; @@ -608,7 +757,7 @@ static int vdbeSorterListToPMA(sqlite3 *db, VdbeCursor *pCsr){ */ int sqlite3VdbeSorterWrite( sqlite3 *db, /* Database handle */ - VdbeCursor *pCsr, /* Sorter cursor */ + const VdbeCursor *pCsr, /* Sorter cursor */ Mem *pVal /* Memory cell containing record */ ){ VdbeSorter *pSorter = pCsr->pSorter; @@ -642,8 +791,14 @@ int sqlite3VdbeSorterWrite( (pSorter->nInMemory>pSorter->mxPmaSize) || (pSorter->nInMemory>pSorter->mnPmaSize && sqlite3HeapNearlyFull()) )){ +#ifdef SQLITE_DEBUG + i64 nExpect = pSorter->iWriteOff + + sqlite3VarintLen(pSorter->nInMemory) + + pSorter->nInMemory; +#endif rc = vdbeSorterListToPMA(db, pCsr); pSorter->nInMemory = 0; + assert( rc!=SQLITE_OK || (nExpect==pSorter->iWriteOff) ); } return rc; @@ -654,7 +809,7 @@ int sqlite3VdbeSorterWrite( */ static int vdbeSorterInitMerge( sqlite3 *db, /* Database handle */ - VdbeCursor *pCsr, /* Cursor handle for this sorter */ + const VdbeCursor *pCsr, /* Cursor handle for this sorter */ i64 *pnByte /* Sum of bytes in all opened PMAs */ ){ VdbeSorter *pSorter = pCsr->pSorter; @@ -684,7 +839,7 @@ static int vdbeSorterInitMerge( ** Once the sorter has been populated, this function is called to prepare ** for iterating through its contents in sorted order. */ -int sqlite3VdbeSorterRewind(sqlite3 *db, VdbeCursor *pCsr, int *pbEof){ +int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ VdbeSorter *pSorter = pCsr->pSorter; int rc; /* Return code */ sqlite3_file *pTemp2 = 0; /* Second temp file to use */ @@ -704,7 +859,7 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, VdbeCursor *pCsr, int *pbEof){ return vdbeSorterSort(pCsr); } - /* Write the current b-tree to a PMA. Close the b-tree cursor. */ + /* Write the current in-memory list to a PMA. */ rc = vdbeSorterListToPMA(db, pCsr); if( rc!=SQLITE_OK ) return rc; @@ -726,8 +881,12 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, VdbeCursor *pCsr, int *pbEof){ rc==SQLITE_OK && iNew*SORTER_MAX_MERGE_COUNTnPMA; iNew++ ){ + int rc2; /* Return code from fileWriterFinish() */ + FileWriter writer; /* Object used to write to disk */ i64 nWrite; /* Number of bytes in new PMA */ + memset(&writer, 0, sizeof(FileWriter)); + /* If there are SORTER_MAX_MERGE_COUNT or less PMAs in file pTemp1, ** initialize an iterator for each of them and break out of the loop. ** These iterators will be incrementally merged as the VDBE layer calls @@ -749,23 +908,20 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, VdbeCursor *pCsr, int *pbEof){ rc = vdbeSorterOpenTempFile(db, &pTemp2); } - if( rc==SQLITE_OK ){ - rc = vdbeSorterWriteVarint(pTemp2, nWrite, &iWrite2); - } - if( rc==SQLITE_OK ){ int bEof = 0; + fileWriterInit(db, pTemp2, &writer, iWrite2); + fileWriterWriteVarint(&writer, nWrite); while( rc==SQLITE_OK && bEof==0 ){ - int nToWrite; VdbeSorterIter *pIter = &pSorter->aIter[ pSorter->aTree[1] ]; assert( pIter->pFile ); - nToWrite = pIter->nKey + sqlite3VarintLen(pIter->nKey); - rc = sqlite3OsWrite(pTemp2, pIter->aAlloc, nToWrite, iWrite2); - iWrite2 += nToWrite; - if( rc==SQLITE_OK ){ - rc = sqlite3VdbeSorterNext(db, pCsr, &bEof); - } + + fileWriterWriteVarint(&writer, pIter->nKey); + fileWriterWrite(&writer, pIter->aKey, pIter->nKey); + rc = sqlite3VdbeSorterNext(db, pCsr, &bEof); } + rc2 = fileWriterFinish(db, &writer, &iWrite2); + if( rc==SQLITE_OK ) rc = rc2; } } @@ -792,7 +948,7 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, VdbeCursor *pCsr, int *pbEof){ /* ** Advance to the next element in the sorter. */ -int sqlite3VdbeSorterNext(sqlite3 *db, VdbeCursor *pCsr, int *pbEof){ +int sqlite3VdbeSorterNext(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ VdbeSorter *pSorter = pCsr->pSorter; int rc; /* Return code */ @@ -822,7 +978,7 @@ int sqlite3VdbeSorterNext(sqlite3 *db, VdbeCursor *pCsr, int *pbEof){ ** current key. */ static void *vdbeSorterRowkey( - VdbeSorter *pSorter, /* Sorter object */ + const VdbeSorter *pSorter, /* Sorter object */ int *pnKey /* OUT: Size of current key in bytes */ ){ void *pKey; @@ -841,7 +997,7 @@ static void *vdbeSorterRowkey( /* ** Copy the current sorter key into the memory cell pOut. */ -int sqlite3VdbeSorterRowkey(VdbeCursor *pCsr, Mem *pOut){ +int sqlite3VdbeSorterRowkey(const VdbeCursor *pCsr, Mem *pOut){ VdbeSorter *pSorter = pCsr->pSorter; void *pKey; int nKey; /* Sorter key to copy into pOut */ @@ -867,7 +1023,7 @@ int sqlite3VdbeSorterRowkey(VdbeCursor *pCsr, Mem *pOut){ ** key. */ int sqlite3VdbeSorterCompare( - VdbeCursor *pCsr, /* Sorter cursor */ + const VdbeCursor *pCsr, /* Sorter cursor */ Mem *pVal, /* Value to compare to current sorter key */ int *pRes /* OUT: Result of comparison */ ){ diff --git a/src/vdbetrace.c b/src/vdbetrace.c index c71a7c41a4..35825c8736 100644 --- a/src/vdbetrace.c +++ b/src/vdbetrace.c @@ -169,9 +169,8 @@ void sqlite3ExplainBegin(Vdbe *pVdbe){ if( pVdbe ){ Explain *p; sqlite3BeginBenignMalloc(); - p = sqlite3_malloc( sizeof(Explain) ); + p = (Explain *)sqlite3MallocZero( sizeof(Explain) ); if( p ){ - memset(p, 0, sizeof(*p)); p->pVdbe = pVdbe; sqlite3_free(pVdbe->pExplain); pVdbe->pExplain = p; diff --git a/test/index5.test b/test/index5.test new file mode 100644 index 0000000000..c8e94b3985 --- /dev/null +++ b/test/index5.test @@ -0,0 +1,75 @@ +# 2012 August 6 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set ::testprefix index5 + +do_test 1.1 { + execsql { + PRAGMA page_size = 1024; + CREATE TABLE t1(x); + BEGIN; + } + for {set i 0} {$i < 100000} {incr i} { + execsql { INSERT INTO t1 VALUES(randstr(100,100)) } + } + execsql COMMIT + execsql { + CREATE INDEX i1 ON t1(x); + DROP INDEX I1; + PRAGMA main.page_size; + } +} {1024} + +db close +testvfs tvfs +tvfs filter xWrite +tvfs script write_cb +proc write_cb {xCall file handle iOfst} { + if {[file tail $file]=="test.db"} { + lappend ::write_list [expr $iOfst/1024] + } + puts "$xCall $file $args" +} + +do_test 1.2 { + sqlite3 db test.db -vfs tvfs + set ::write_list [list] + execsql { CREATE INDEX i1 ON t1(x) } +} {} + +do_test 1.3 { + set nForward 0 + set nBackward 0 + set nNoncont 0 + set iPrev [lindex $::write_list 0] + for {set i 1} {$i < [llength $::write_list]} {incr i} { + set iNext [lindex $::write_list $i] + if {$iNext==($iPrev+1)} { + incr nForward + } elseif {$iNext==($iPrev-1)} { + incr nBackward + } else { + incr nNoncont + } + set iPrev $iNext + } + + expr {$nForward > $nBackward} +} {1} +db close +tvfs delete + +finish_test + diff --git a/test/permutations.test b/test/permutations.test index 4370d8691d..c41c447a53 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -111,7 +111,7 @@ set allquicktests [test_set $alltests -exclude { thread003.test thread004.test thread005.test trans2.test vacuum3.test incrvacuum_ioerr.test autovacuum_crash.test btree8.test shared_err.test vtab_err.test walslow.test walcrash.test walcrash3.test - walthread.test rtree3.test indexfault.test + walthread.test rtree3.test indexfault.test securedel2.test }] if {[info exists ::env(QUICKTEST_INCLUDE)]} { set allquicktests [concat $allquicktests $::env(QUICKTEST_INCLUDE)] diff --git a/test/securedel2.test b/test/securedel2.test new file mode 100644 index 0000000000..b20f4f9212 --- /dev/null +++ b/test/securedel2.test @@ -0,0 +1,95 @@ +# 2012 August 7 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# +# Tests for the secure_delete pragma. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set ::testprefix securedel2 + +# Generate 1000 pseudo-random 64-bit blobs. +# +for {set i 1} {$i <= 1000} {incr i} { + set aBlob($i) [string range [db one {SELECT quote(randomblob(8))}] 2 end-1] +} + +proc detect_blob_prepare {zFile} { + set nByte [file size $zFile] + set ::detect_blob_data [hexio_read $zFile 0 $nByte] +} + +proc detect_blob {zFile iBlob} { + if {$zFile != ""} { detect_blob_prepare $zFile } + string match "*$::aBlob($iBlob)*" $::detect_blob_data +} + +do_test 1.1 { + execsql { PRAGMA secure_delete = 1 } + execsql { PRAGMA auto_vacuum = 0 } + execsql { CREATE TABLE t1(x, y) } + for {set i 1} {$i <= 1000} {incr i} { + set x "X'[string repeat $aBlob($i) 1]'" + set y "X'[string repeat $aBlob($i) 500]'" + execsql "INSERT INTO t1 VALUES($x, $y)" + } +} {} + +do_test 1.2 { detect_blob test.db 1 } {1} + +forcecopy test.db test.db.bak +do_execsql_test 1.3.1 { PRAGMA secure_delete = 0 } {0} +do_execsql_test 1.3.2 { DELETE FROM t1 WHERE rowid = 1 } +do_test 1.3.3 { detect_blob test.db 1 } {1} + +db close +forcecopy test.db.bak test.db +sqlite3 db test.db +do_execsql_test 1.4.1 { PRAGMA secure_delete = 1 } {1} +do_execsql_test 1.4.2 { DELETE FROM t1 WHERE rowid = 1 } +do_test 1.4.3 { detect_blob test.db 1 } {0} + +do_execsql_test 1.5.1 { DELETE FROM t1 WHERE rowid>850 } {} +do_test 1.5.2 { + set n 0 + detect_blob_prepare test.db + for {set i 851} {$i <= 1000} {incr i 5} { + incr n [detect_blob {} $i] + } + set n +} {0} + +db close +sqlite3 db test.db +do_test 1.6.1 { + execsql { + PRAGMA cache_size = 200; + PRAGMA secure_delete = 1; + CREATE TABLE t2(x); + SELECT * FROM t1; + } + for {set i 100} {$i < 5000} {incr i} { + execsql { INSERT INTO t2 VALUES(randomblob($i)) } + } + execsql { DELETE FROM t1 } +} {} + +do_test 1.6.2 { + set n 0 + detect_blob_prepare test.db + for {set i 2} {$i <= 850} {incr i 5} { + incr n [detect_blob {} $i] + } + set n +} {0} + +finish_test + diff --git a/test/spellfix.test b/test/spellfix.test index 7459ca2cd6..245fee24e1 100644 --- a/test/spellfix.test +++ b/test/spellfix.test @@ -14,6 +14,8 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl set testprefix spellfix +ifcapable !vtab { finish_test ; return } + register_spellfix_module db set vocab { @@ -67,17 +69,18 @@ do_test 1.1 { foreach {tn word res} { 1 raxpi* {rasping 5 rasped 5 raspberry 6 rasp 4 rasps 4} - 2 ril* {rail 4 railway 4 railing 4 rails 4 railways 4} - 3 rilis* {realist 6 realistic 6 realistically 6 realists 6 realism 6} - 4 reail* {realities 3 reality 3 real 3 realest 3 realist 3} - 5 ras* {rasp 3 rash 3 rasped 3 rasping 3 rasps 3} + 2 ril* {rail 4 railed 4 railer 4 railers 4 railing 4} + 3 rilis* {realism 6 realist 6 realistic 6 realistically 6 realists 6} + 4 reail* {real 3 realest 3 realign 3 realigned 3 realigning 3} + 5 ras* {rascal 3 rascally 3 rascals 3 rash 3 rasher 3} 6 realistss* {realists 8 realigns 8 realistic 9 realistically 9 realest 7} 7 realistss {realists 8 realist 7 realigns 8 realistic 9 realest 7} 8 rllation* {realities 9 reality 7 rallied 7 railed 4} - 9 renstom* {rainstorm 8 ransomer 6 ransom 6 ransoming 6 ransoms 6} + 9 renstom* {rainstorm 8 ransom 6 ransomer 6 ransoming 6 ransoms 6} } { do_execsql_test 1.2.$tn { - SELECT word, matchlen FROM t1 WHERE word MATCH $word LIMIT 5 + SELECT word, matchlen FROM t1 WHERE word MATCH $word + ORDER BY score, word LIMIT 5 } $res } @@ -135,12 +138,13 @@ do_test 3.2 { breakpoint foreach {tn word res} { - 1 kos* {kosher 3 kiosk 4 kudo 2 kappa 1 keypad 1} - 2 kellj* {killjoy 5 killed 4 killingly 4 kill 4 killer 4} + 1 kos* {kosher 3 kiosk 4 kudo 2 kiss 3 kissed 3} + 2 kellj* {killjoy 5 kill 4 killed 4 killer 4 killers 4} 3 kellj {kill 4 kills 5 killjoy 7 keel 4 killed 6} } { do_execsql_test 1.2.$tn { - SELECT word, matchlen FROM t3 WHERE word MATCH $word LIMIT 5 + SELECT word, matchlen FROM t3 WHERE word MATCH $word + ORDER BY score, word LIMIT 5 } $res } diff --git a/test/walro.test b/test/walro.test index 81e3e62ee9..465ce838c2 100644 --- a/test/walro.test +++ b/test/walro.test @@ -56,6 +56,7 @@ do_multiclient_test tn { do_test 1.1.1 { code2 { sqlite3 db2 test.db } sql2 { + PRAGMA auto_vacuum = 0; PRAGMA journal_mode = WAL; CREATE TABLE t1(x, y); INSERT INTO t1 VALUES('a', 'b'); @@ -268,6 +269,7 @@ do_multiclient_test tn { tv filter {} code1 { sqlite3 db test.db } csql1 { + PRAGMA auto_vacuum = 0; PRAGMA journal_mode = WAL; BEGIN; CREATE TABLE t2(x, y); diff --git a/test/walthread.test b/test/walthread.test index cbd6371857..6249ce11af 100644 --- a/test/walthread.test +++ b/test/walthread.test @@ -277,8 +277,8 @@ do_thread_test2 walthread-1 -seconds $seconds(walthread-1) -init { proc write_transaction {} { db eval { BEGIN; - INSERT INTO t1 VALUES(randomblob(100)); - INSERT INTO t1 VALUES(randomblob(100)); + INSERT INTO t1 VALUES(randomblob(101 + $::E(pid))); + INSERT INTO t1 VALUES(randomblob(101 + $::E(pid))); INSERT INTO t1 SELECT md5sum(x) FROM t1; COMMIT; } diff --git a/tool/build-all-msvc.bat b/tool/build-all-msvc.bat new file mode 100755 index 0000000000..a2d7dae3a3 --- /dev/null +++ b/tool/build-all-msvc.bat @@ -0,0 +1,394 @@ +@ECHO OFF + +:: +:: build-all-msvc.bat -- +:: +:: Multi-Platform Build Tool for MSVC +:: + +SETLOCAL + +REM SET __ECHO=ECHO +REM SET __ECHO2=ECHO +IF NOT DEFINED _AECHO (SET _AECHO=REM) +IF NOT DEFINED _CECHO (SET _CECHO=REM) +IF NOT DEFINED _VECHO (SET _VECHO=REM) + +%_AECHO% Running %0 %* + +REM SET DFLAGS=/L + +%_VECHO% DFlags = '%DFLAGS%' + +SET FFLAGS=/V /F /G /H /I /R /Y /Z + +%_VECHO% FFlags = '%FFLAGS%' + +SET ROOT=%~dp0\.. +SET ROOT=%ROOT:\\=\% + +%_VECHO% Root = '%ROOT%' + +REM +REM NOTE: The first and only argument to this batch file should be the output +REM directory where the platform-specific binary directories should be +REM created. +REM +SET BINARYDIRECTORY=%1 + +IF NOT DEFINED BINARYDIRECTORY ( + GOTO usage +) + +%_VECHO% BinaryDirectory = '%BINARYDIRECTORY%' + +SET DUMMY=%2 + +IF DEFINED DUMMY ( + GOTO usage +) + +REM +REM NOTE: From this point, we need a clean error level. Reset it now. +REM +CALL :fn_ResetErrorLevel + +REM +REM NOTE: Change the current directory to the root of the source tree, saving +REM the current directory on the directory stack. +REM +%__ECHO2% PUSHD "%ROOT%" + +IF ERRORLEVEL 1 ( + ECHO Could not change directory to "%ROOT%". + GOTO errors +) + +REM +REM NOTE: This batch file requires the ComSpec environment variable to be set, +REM typically to something like "C:\Windows\System32\cmd.exe". +REM +IF NOT DEFINED ComSpec ( + ECHO The ComSpec environment variable must be defined. + GOTO errors +) + +REM +REM NOTE: This batch file requires the VcInstallDir environment variable to be +REM set. Tyipcally, this means this batch file needs to be run from an +REM MSVC command prompt. +REM +IF NOT DEFINED VCINSTALLDIR ( + ECHO The VCINSTALLDIR environment variable must be defined. + GOTO errors +) + +REM +REM NOTE: If the list of platforms is not already set, use the default list. +REM +IF NOT DEFINED PLATFORMS ( + SET PLATFORMS=x86 x86_amd64 x86_arm +) + +%_VECHO% Platforms = '%PLATFORMS%' + +REM +REM NOTE: Setup environment variables to translate between the MSVC platform +REM names and the names to be used for the platform-specific binary +REM directories. +REM +SET x86_NAME=x86 +SET x86_amd64_NAME=x64 +SET x86_arm_NAME=ARM + +%_VECHO% x86_Name = '%x86_NAME%' +%_VECHO% x86_amd64_Name = '%x86_amd64_NAME%' +%_VECHO% x86_arm_Name = '%x86_arm_NAME%' + +REM +REM NOTE: Check for the external tools needed during the build process ^(i.e. +REM those that do not get compiled as part of the build process itself^) +REM along the PATH. +REM +FOR %%T IN (gawk.exe tclsh85.exe) DO ( + SET %%T_PATH=%%~dp$PATH:T +) + +REM +REM NOTE: Set the TOOLPATH variable to contain all the directories where the +REM external tools were found in the search above. +REM +SET TOOLPATH=%gawk.exe_PATH%;%tclsh85.exe_PATH% + +%_VECHO% ToolPath = '%TOOLPATH%' + +REM +REM NOTE: Check for MSVC 2012 because the Windows SDK directory handling is +REM slightly different for that version. +REM +IF "%VisualStudioVersion%" == "11.0" ( + SET SET_NSDKLIBPATH=1 +) ELSE ( + CALL :fn_UnsetVariable SET_NSDKLIBPATH +) + +REM +REM NOTE: This is the outer loop. There should be exactly one iteration per +REM platform. +REM +FOR %%P IN (%PLATFORMS%) DO ( + REM + REM NOTE: Using the MSVC platform name, lookup the simpler platform name to + REM be used for the name of the platform-specific binary directory via + REM the environment variables setup earlier. + REM + CALL :fn_SetVariable %%P_NAME PLATFORMNAME + + REM + REM NOTE: This is the inner loop. There should be exactly one iteration. + REM This loop is necessary because the PlatformName environment + REM variable was set above and that value is needed by some of the + REM commands contained in the inner loop. If these commands were + REM directly contained in the outer loop, the PlatformName environment + REM variable would be stuck with its initial empty value instead. + REM + FOR /F "tokens=2* delims==" %%D IN ('SET PLATFORMNAME') DO ( + REM + REM NOTE: Attempt to clean the environment of all variables used by MSVC + REM and/or Visual Studio. This block may need to be updated in the + REM future to account for additional environment variables. + REM + CALL :fn_UnsetVariable DevEnvDir + CALL :fn_UnsetVariable ExtensionSdkDir + CALL :fn_UnsetVariable Framework35Version + CALL :fn_UnsetVariable FrameworkDir + CALL :fn_UnsetVariable FrameworkDir32 + CALL :fn_UnsetVariable FrameworkVersion + CALL :fn_UnsetVariable FrameworkVersion32 + CALL :fn_UnsetVariable FSHARPINSTALLDIR + CALL :fn_UnsetVariable INCLUDE + CALL :fn_UnsetVariable LIB + CALL :fn_UnsetVariable LIBPATH + CALL :fn_UnsetVariable Platform + REM CALL :fn_UnsetVariable VCINSTALLDIR + CALL :fn_UnsetVariable VSINSTALLDIR + CALL :fn_UnsetVariable WindowsSdkDir + CALL :fn_UnsetVariable WindowsSdkDir_35 + CALL :fn_UnsetVariable WindowsSdkDir_old + + REM + REM NOTE: Reset the PATH here to the absolute bare minimum required. + REM + SET PATH=%TOOLPATH%;%SystemRoot%\System32;%SystemRoot% + + REM + REM NOTE: Launch a nested command shell to perform the following steps: + REM + REM 1. Setup the MSVC environment for this platform using the + REM official batch file. + REM + REM 2. Make sure that no stale build output files are present. + REM + REM 3. Build the "sqlite3.dll" and "sqlite3.lib" binaries for this + REM platform. + REM + REM 4. Copy the "sqlite3.dll" and "sqlite3.lib" binaries for this + REM platform to the platform-specific directory beneath the + REM binary directory. + REM + "%ComSpec%" /C ( + REM + REM NOTE: Attempt to setup the MSVC environment for this platform. + REM + %__ECHO% CALL "%VCINSTALLDIR%\vcvarsall.bat" %%P + + IF ERRORLEVEL 1 ( + ECHO Failed to call "%VCINSTALLDIR%\vcvarsall.bat" for platform %%P. + GOTO errors + ) + + REM + REM NOTE: If this batch file is not running in "what-if" mode, check to + REM be sure we were actually able to setup the MSVC environment as + REM current versions of their official batch file do not set the + REM exit code upon failure. + REM + IF NOT DEFINED __ECHO ( + IF NOT DEFINED WindowsSdkDir ( + ECHO Cannot build, Windows SDK not found for platform %%P. + GOTO errors + ) + ) + + REM + REM NOTE: When using MSVC 2012, the native SDK path cannot simply use + REM the "lib" sub-directory beneath the location specified in the + REM WindowsSdkDir environment variable because that location does + REM not actually contain the necessary library files for x86. + REM This must be done for each iteration because it relies upon + REM the WindowsSdkDir environment variable being set by the batch + REM file used to setup the MSVC environment. + REM + IF DEFINED SET_NSDKLIBPATH ( + CALL :fn_SetVariable WindowsSdkDir NSDKLIBPATH + CALL :fn_AppendVariable NSDKLIBPATH \lib\win8\um\x86 + ) + + REM + REM NOTE: Unless prevented from doing so, invoke NMAKE with the MSVC + REM makefile to clean any stale build output from previous + REM iterations of this loop and/or previous runs of this batch + REM file, etc. + REM + IF NOT DEFINED NOCLEAN ( + %__ECHO% nmake -f Makefile.msc clean + + IF ERRORLEVEL 1 ( + ECHO Failed to clean for platform %%P. + GOTO errors + ) + ) ELSE ( + REM + REM NOTE: Even when the cleaning step has been disabled, we still need + REM to remove the build output for the files we are specifically + REM wanting to build for each platform. + REM + %__ECHO% DEL /Q sqlite3.dll sqlite3.lib sqlite3.pdb + ) + + REM + REM NOTE: Invoke NMAKE with the MSVC makefile to build the "sqlite3.dll" + REM binary. The x86 compiler will be used to compile the native + REM command line tools needed during the build process itself. + REM Also, disable looking for and/or linking to the native Tcl + REM runtime library. + REM + %__ECHO% nmake -f Makefile.msc sqlite3.dll "NCC=""%VCINSTALLDIR%\bin\cl.exe""" USE_NATIVE_LIBPATHS=1 NO_TCL=1 %NMAKE_ARGS% + + IF ERRORLEVEL 1 ( + ECHO Failed to build "sqlite3.dll" for platform %%P. + GOTO errors + ) + + REM + REM NOTE: Copy the "sqlite3.dll" file to the platform-specific directory + REM beneath the binary directory. + REM + %__ECHO% XCOPY sqlite3.dll "%BINARYDIRECTORY%\%%D\" %FFLAGS% %DFLAGS% + + IF ERRORLEVEL 1 ( + ECHO Failed to copy "sqlite3.dll" to "%BINARYDIRECTORY%\%%D\". + GOTO errors + ) + + REM + REM NOTE: Copy the "sqlite3.lib" file to the platform-specific directory + REM beneath the binary directory. + REM + %__ECHO% XCOPY sqlite3.lib "%BINARYDIRECTORY%\%%D\" %FFLAGS% %DFLAGS% + + IF ERRORLEVEL 1 ( + ECHO Failed to copy "sqlite3.lib" to "%BINARYDIRECTORY%\%%D\". + GOTO errors + ) + + REM + REM NOTE: Copy the "sqlite3.pdb" file to the platform-specific directory + REM beneath the binary directory unless we are prevented from doing + REM so. + REM + IF NOT DEFINED NOSYMBOLS ( + %__ECHO% XCOPY sqlite3.pdb "%BINARYDIRECTORY%\%%D\" %FFLAGS% %DFLAGS% + + IF ERRORLEVEL 1 ( + ECHO Failed to copy "sqlite3.pdb" to "%BINARYDIRECTORY%\%%D\". + GOTO errors + ) + ) + ) + ) + + REM + REM NOTE: Handle any errors generated during the nested command shell. + REM + IF ERRORLEVEL 1 ( + GOTO errors + ) +) + +REM +REM NOTE: Restore the saved current directory from the directory stack. +REM +%__ECHO2% POPD + +IF ERRORLEVEL 1 ( + ECHO Could not restore directory. + GOTO errors +) + +REM +REM NOTE: If we get to this point, we have succeeded. +REM +GOTO no_errors + +:fn_ResetErrorLevel + VERIFY > NUL + GOTO :EOF + +:fn_SetErrorLevel + VERIFY MAYBE 2> NUL + GOTO :EOF + +:fn_SetVariable + SETLOCAL + IF NOT DEFINED %1 GOTO :EOF + IF "%2" == "" GOTO :EOF + SET __ECHO_CMD=ECHO %%%1%% + FOR /F "delims=" %%V IN ('%__ECHO_CMD%') DO ( + SET VALUE=%%V + ) + ENDLOCAL && SET %2=%VALUE% + GOTO :EOF + +:fn_UnsetVariable + IF NOT "%1" == "" ( + SET %1= + CALL :fn_ResetErrorLevel + ) + GOTO :EOF + +:fn_AppendVariable + SET __ECHO_CMD=ECHO %%%1%% + IF DEFINED %1 ( + FOR /F "delims=" %%V IN ('%__ECHO_CMD%') DO ( + SET %1=%%V%~2 + ) + ) ELSE ( + SET %1=%~2 + ) + SET __ECHO_CMD= + CALL :fn_ResetErrorLevel + GOTO :EOF + +:usage + ECHO. + ECHO Usage: %~nx0 ^ + ECHO. + GOTO errors + +:errors + CALL :fn_SetErrorLevel + ENDLOCAL + ECHO. + ECHO Failure, errors were encountered. + GOTO end_of_file + +:no_errors + CALL :fn_ResetErrorLevel + ENDLOCAL + ECHO. + ECHO Success, no errors were encountered. + GOTO end_of_file + +:end_of_file +%__ECHO% EXIT /B %ERRORLEVEL% diff --git a/tool/mkvsix.tcl b/tool/mkvsix.tcl new file mode 100644 index 0000000000..a7517786c5 --- /dev/null +++ b/tool/mkvsix.tcl @@ -0,0 +1,419 @@ +#!/usr/bin/tclsh +# +# This script is used to generate a VSIX (Visual Studio Extension) file for +# SQLite usable by Visual Studio. + +proc fail { {error ""} {usage false} } { + if {[string length $error] > 0} then { + puts stdout $error + if {!$usage} then {exit 1} + } + + puts stdout "usage:\ +[file tail [info nameofexecutable]]\ +[file tail [info script]] \[sourceDirectory\]" + + exit 1 +} + +proc getEnvironmentVariable { name } { + # + # NOTE: Returns the value of the specified environment variable or an empty + # string for environment variables that do not exist in the current + # process environment. + # + return [expr {[info exists ::env($name)] ? $::env($name) : ""}] +} + +proc getTemporaryPath {} { + # + # NOTE: Returns the normalized path to the first temporary directory found + # in the typical set of environment variables used for that purpose + # or an empty string to signal a failure to locate such a directory. + # + set names [list] + + foreach name [list TEMP TMP] { + lappend names [string toupper $name] [string tolower $name] \ + [string totitle $name] + } + + foreach name $names { + set value [getEnvironmentVariable $name] + + if {[string length $value] > 0} then { + return [file normalize $value] + } + } + + return "" +} + +proc appendArgs { args } { + # + # NOTE: Returns all passed arguments joined together as a single string with + # no intervening spaces between arguments. + # + eval append result $args +} + +proc readFile { fileName } { + # + # NOTE: Reads and returns the entire contents of the specified file, which + # may contain binary data. + # + set file_id [open $fileName RDONLY] + fconfigure $file_id -encoding binary -translation binary + set result [read $file_id] + close $file_id + return $result +} + +proc writeFile { fileName data } { + # + # NOTE: Writes the entire contents of the specified file, which may contain + # binary data. + # + set file_id [open $fileName {WRONLY CREAT TRUNC}] + fconfigure $file_id -encoding binary -translation binary + puts -nonewline $file_id $data + close $file_id + return "" +} + +proc substFile { fileName } { + # + # NOTE: Performs all Tcl command, variable, and backslash substitutions in + # the specified file and then re-writes the contents of that same file + # with the substituted data. + # + return [writeFile $fileName [uplevel 1 [list subst [readFile $fileName]]]] +} + +proc replacePlatform { fileName platformName } { + # + # NOTE: Returns the specified file name containing the platform name instead + # of platform placeholder tokens. + # + return [string map [list $platformName] $fileName] +} + +set script [file normalize [info script]] + +if {[string length $script] == 0} then { + fail "script file currently being evaluated is unknown" true +} + +set path [file dirname $script] +set rootName [file rootname [file tail $script]] + +############################################################################### + +# +# NOTE: Process and verify all the command line arguments. +# +set argc [llength $argv] +if {$argc != 1 && $argc != 2} then {fail} + +set binaryDirectory [lindex $argv 0] + +if {[string length $binaryDirectory] == 0} then { + fail "invalid binary directory" +} + +if {![file exists $binaryDirectory] || \ + ![file isdirectory $binaryDirectory]} then { + fail "binary directory does not exist" +} + +if {$argc == 2} then { + set sourceDirectory [lindex $argv 1] +} else { + # + # NOTE: Assume that the source directory is the parent directory of the one + # that contains this script file. + # + set sourceDirectory [file dirname $path] +} + +if {[string length $sourceDirectory] == 0} then { + fail "invalid source directory" +} + +if {![file exists $sourceDirectory] || \ + ![file isdirectory $sourceDirectory]} then { + fail "source directory does not exist" +} + +############################################################################### + +# +# NOTE: Evaluate the user-specific customizations file, if it exists. +# +set userFile [file join $path [appendArgs \ + $rootName . $tcl_platform(user) .tcl]] + +if {[file exists $userFile] && \ + [file isfile $userFile]} then { + source $userFile +} + +############################################################################### + +set templateFile [file join $path win sqlite.vsix] + +if {![file exists $templateFile] || \ + ![file isfile $templateFile]} then { + fail [appendArgs "template file \"" $templateFile "\" does not exist"] +} + +set currentDirectory [pwd] +set outputFile [file join $currentDirectory sqlite-output.vsix] + +if {[file exists $outputFile]} then { + fail [appendArgs "output file \"" $outputFile "\" already exists"] +} + +############################################################################### + +# +# NOTE: Make sure that a valid temporary directory exists. +# +set temporaryDirectory [getTemporaryPath] + +if {[string length $temporaryDirectory] == 0 || \ + ![file exists $temporaryDirectory] || \ + ![file isdirectory $temporaryDirectory]} then { + fail "cannot locate a usable temporary directory" +} + +# +# NOTE: Setup the staging directory to have a unique name inside of the +# configured temporary directory. +# +set stagingDirectory [file normalize [file join $temporaryDirectory \ + [appendArgs $rootName . [pid]]]] + +############################################################################### + +# +# NOTE: Configure the external zipping tool. First, see if it has already +# been pre-configured. If not, try to query it from the environment. +# Finally, fallback on the default of simply "zip", which will then +# be assumed to exist somewhere along the PATH. +# +if {![info exists zip]} then { + if {[info exists env(ZipTool)]} then { + set zip $env(ZipTool) + } + if {![info exists zip] || ![file exists $zip]} then { + set zip zip + } +} + +# +# NOTE: Configure the external unzipping tool. First, see if it has already +# been pre-configured. If not, try to query it from the environment. +# Finally, fallback on the default of simply "unzip", which will then +# be assumed to exist somewhere along the PATH. +# +if {![info exists unzip]} then { + if {[info exists env(UnZipTool)]} then { + set unzip $env(UnZipTool) + } + if {![info exists unzip] || ![file exists $unzip]} then { + set unzip unzip + } +} + +############################################################################### + +# +# NOTE: Attempt to extract the SQLite version from the "sqlite3.h" header file +# in the source directory. This script assumes that the header file has +# already been generated by the build process. +# +set pattern {^#define\s+SQLITE_VERSION\s+"(.*)"$} +set data [readFile [file join $sourceDirectory sqlite3.h]] + +if {![regexp -line -- $pattern $data dummy version]} then { + fail [appendArgs "cannot locate SQLITE_VERSION value in \"" \ + [file join $sourceDirectory sqlite3.h] \"] +} + +############################################################################### + +# +# NOTE: Setup the master file list data, including the necessary flags. +# +if {![info exists fileNames(source)]} then { + set fileNames(source) [list "" "" "" \ + [file join $sourceDirectory sqlite3.h] \ + [file join $binaryDirectory sqlite3.lib] \ + [file join $binaryDirectory sqlite3.dll]] + + if {![info exists no(symbols)]} then { + lappend fileNames(source) \ + [file join $binaryDirectory sqlite3.pdb] + } +} + +if {![info exists fileNames(destination)]} then { + set fileNames(destination) [list \ + [file join $stagingDirectory extension.vsixmanifest] \ + [file join $stagingDirectory SDKManifest.xml] \ + [file join $stagingDirectory DesignTime CommonConfiguration \ + SQLite.WinRT.props] \ + [file join $stagingDirectory DesignTime CommonConfiguration \ + sqlite3.h] \ + [file join $stagingDirectory DesignTime CommonConfiguration \ + sqlite3.lib] \ + [file join $stagingDirectory Redist CommonConfiguration \ + sqlite3.dll]] + + if {![info exists no(symbols)]} then { + lappend fileNames(destination) \ + [file join $stagingDirectory Redist Debug \ + sqlite3.pdb] + } +} + +if {![info exists fileNames(neutral)]} then { + set fileNames(neutral) [list 1 1 1 1 0 0] + + if {![info exists no(symbols)]} then { + lappend fileNames(neutral) 0 + } +} + +if {![info exists fileNames(subst)]} then { + set fileNames(subst) [list 1 1 1 0 0 0] + + if {![info exists no(symbols)]} then { + lappend fileNames(subst) 0 + } +} + +############################################################################### + +# +# NOTE: Setup the list of platforms supported by this script. +# +if {![info exists platformNames]} then { + set platformNames [list x86 x64 ARM] +} + +############################################################################### + +# +# NOTE: Make sure the staging directory exists, creating it if necessary. +# +file mkdir $stagingDirectory + +# +# NOTE: Build the Tcl command used to extract the template package to the +# staging directory. +# +set extractCommand [list exec -- $unzip $templateFile -d $stagingDirectory] + +# +# NOTE: Extract the template package to the staging directory. +# +eval $extractCommand + +############################################################################### + +# +# NOTE: Process each file in the master file list. There are actually four +# parallel lists that contain the source file names, destination file +# names, the platform-neutral flags, and the use-subst flags. When the +# platform-neutral flag is non-zero, the file is not platform-specific. +# When the use-subst flag is non-zero, the file is considered to be a +# text file that may contain Tcl variable and/or command replacements, +# to be dynamically replaced during processing. If the source file name +# is an empty string, then the destination file name will be assumed to +# already exist in the staging directory and will not be copied; however, +# dynamic replacements may still be performed on the destination file +# prior to the package being re-zipped. +# +foreach sourceFileName $fileNames(source) \ + destinationFileName $fileNames(destination) \ + isNeutral $fileNames(neutral) useSubst $fileNames(subst) { + # + # NOTE: If the current file is platform-neutral, then only one platform will + # be processed for it, namely "neutral"; otherwise, each supported + # platform will be processed for it individually. + # + foreach platformName [expr {$isNeutral ? [list neutral] : $platformNames}] { + # + # NOTE: Use the actual platform name in the destination file name. + # + set newDestinationFileName [replacePlatform $destinationFileName \ + $platformName] + + # + # NOTE: Does the source file need to be copied to the destination file? + # + if {[string length $sourceFileName] > 0} then { + # + # NOTE: First, make sure the destination directory exists. + # + file mkdir [file dirname $newDestinationFileName] + + # + # NOTE: Then, copy the source file to the destination file verbatim. + # + file copy [replacePlatform $sourceFileName $platformName] \ + $newDestinationFileName + } + + # + # NOTE: Does the destination file contain dynamic replacements that must + # be processed now? + # + if {$useSubst} then { + # + # NOTE: Perform any dynamic replacements contained in the destination + # file and then re-write it in-place. + # + substFile $newDestinationFileName + } + } +} + +############################################################################### + +# +# NOTE: Change the current directory to the staging directory so that the +# external archive building tool can pickup the necessary files using +# relative paths. +# +cd $stagingDirectory + +# +# NOTE: Build the Tcl command used to archive the final package in the +# output directory. +# +set archiveCommand [list exec -- $zip -r $outputFile *] + +# +# NOTE: Build the final package archive in the output directory. +# +eval $archiveCommand + +# +# NOTE: Change back to the previously saved current directory. +# +cd $currentDirectory + +# +# NOTE: Cleanup the temporary staging directory. +# +file delete -force $stagingDirectory + +############################################################################### + +# +# NOTE: Success, emit the fully qualified path of the generated VSIX file. +# +puts stdout $outputFile diff --git a/tool/win/sqlite.vsix b/tool/win/sqlite.vsix new file mode 100644 index 0000000000..4bdfda5c7d Binary files /dev/null and b/tool/win/sqlite.vsix differ