From 248f2be91aefce1e595dd1916fd104f8d3b4d680 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 23 Apr 2013 20:10:13 +0000 Subject: [PATCH 01/32] Make "test_regexp.c" into a loadable extension and move it over to ext/misc/regexp.c. Add the "load_static_extension" command for testing purposes. FossilOrigin-Name: 860fc393bff27045e0593c3c51bf5577accd0b79 --- src/test_regexp.c => ext/misc/regexp.c | 73 +++++++++----------------- main.mk | 2 +- manifest | 23 ++++---- manifest.uuid | 2 +- src/tclsqlite.c | 2 - src/test1.c | 48 +++++++++++++++++ test/regexp1.test | 2 +- 7 files changed, 89 insertions(+), 63 deletions(-) rename src/test_regexp.c => ext/misc/regexp.c (94%) diff --git a/src/test_regexp.c b/ext/misc/regexp.c similarity index 94% rename from src/test_regexp.c rename to ext/misc/regexp.c index 829d22ad25..cb39956da5 100644 --- a/src/test_regexp.c +++ b/ext/misc/regexp.c @@ -49,7 +49,17 @@ */ #include #include -#include "sqlite3.h" +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 + +/* +** The following #defines change the names of some functions implemented in +** this file to prevent name collisions with C-library functions of the +** same name. +*/ +#define re_match sqlite3re_match +#define re_compile sqlite3re_compile +#define re_free sqlite3re_free /* The end-of-input character */ #define RE_EOF 0 /* End of input */ @@ -175,7 +185,7 @@ static int re_space_char(int c){ /* Run a compiled regular expression on the zero-terminated input ** string zIn[]. Return true on a match and false if there is no match. */ -int re_match(ReCompiled *pRe, const unsigned char *zIn, int nIn){ +static int re_match(ReCompiled *pRe, const unsigned char *zIn, int nIn){ ReStateSet aStateSet[2], *pThis, *pNext; ReStateNumber aSpace[100]; ReStateNumber *pToFree; @@ -718,53 +728,20 @@ static void re_sql_func( } /* -** Invoke this routine in order to install the REGEXP function in an +** Invoke this routine to register the regexp() function with the ** SQLite database connection. -** -** Use: -** -** sqlite3_auto_extension(sqlite3_add_regexp_func); -** -** to cause this extension to be automatically loaded into each new -** database connection. */ -int sqlite3_add_regexp_func(sqlite3 *db){ - return sqlite3_create_function(db, "regexp", 2, SQLITE_UTF8, 0, - re_sql_func, 0, 0); -} - - -/***************************** Test Code ***********************************/ -#ifdef SQLITE_TEST -#include -extern int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite3 **ppDb); - -/* Implementation of the TCL command: -** -** sqlite3_add_regexp_func $DB -*/ -static int tclSqlite3AddRegexpFunc( - void * clientData, - Tcl_Interp *interp, - int objc, - Tcl_Obj *CONST objv[] +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_regexp_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi ){ - sqlite3 *db; - if( objc!=2 ){ - Tcl_WrongNumArgs(interp, 1, objv, "DB"); - return TCL_ERROR; - } - if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; - sqlite3_add_regexp_func(db); - return TCL_OK; + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + rc = sqlite3_create_function(db, "regexp", 2, SQLITE_UTF8, 0, + re_sql_func, 0, 0); + return rc; } - -/* Register the sqlite3_add_regexp_func TCL command with the TCL interpreter. -*/ -int Sqlitetestregexp_Init(Tcl_Interp *interp){ - Tcl_CreateObjCommand(interp, "sqlite3_add_regexp_func", - tclSqlite3AddRegexpFunc, 0, 0); - return TCL_OK; -} -#endif /* SQLITE_TEST */ -/**************************** End Of Test Code *******************************/ diff --git a/main.mk b/main.mk index 36a4da0294..550b8f10f0 100644 --- a/main.mk +++ b/main.mk @@ -256,7 +256,7 @@ TESTSRC = \ $(TOP)/src/test_osinst.c \ $(TOP)/src/test_pcache.c \ $(TOP)/src/test_quota.c \ - $(TOP)/src/test_regexp.c \ + $(TOP)/ext/misc/regexp.c \ $(TOP)/src/test_rtree.c \ $(TOP)/src/test_schema.c \ $(TOP)/src/test_server.c \ diff --git a/manifest b/manifest index f702eb82f2..bf91bd1649 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarnings. -D 2013-04-22T23:38:50.007 +C Make\s"test_regexp.c"\sinto\sa\sloadable\sextension\sand\smove\sit\sover\sto\next/misc/regexp.c.\s\sAdd\sthe\s"load_static_extension"\scommand\sfor\stesting\npurposes. +D 2013-04-23T20:10:13.518 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 87591ea5bf7d6ed521ad42d5bc69c124debe11a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -83,6 +83,7 @@ F ext/fts3/unicode/mkunicode.tcl 7a9bc018e2962abb79563c5a39fe581fcbf2f675 F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43 F ext/icu/icu.c eb9ae1d79046bd7871aa97ee6da51eb770134b5a F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37 +F ext/misc/regexp.c c0fdb8af86981ff9890d776cfb97fe66297cc3b2 w src/test_regexp.c F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 757abea591d4ff67c0ff4e8f9776aeda86b18c14 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e @@ -104,7 +105,7 @@ F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 -F main.mk e97fbfe4c69060560574e1b59c43f4f88a38a0ec +F main.mk 8f410dfbb97943889298a2da779be0f8f3dcb750 F mkdll.sh 7d09b23c05d56532e9d44a50868eb4b12ff4f74a F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f F mkextw.sh 4123480947681d9b434a5e7b1ee08135abe409ac @@ -190,8 +191,8 @@ F src/sqliteInt.h 2a83cfec9963372b636b0cabd4b200c1f1074a99 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e -F src/tclsqlite.c 9a716c737590d2f129d71c8fc7065e5aba0e7222 -F src/test1.c 6784fdacb35c33ba564ef749b62c4718fe515484 +F src/tclsqlite.c fd60f5950535fd880ce32f357fa9da62db65c490 +F src/test1.c 317741fa810b8acbdf849891202175ea762d68e9 F src/test2.c 29e7154112f7448d64204e8d31179cf497ecf425 F src/test3.c 96aed72a8e1d542fed127e3e8350ae357712fa82 F src/test4.c cea2c55110241e4674e66d476d29c914627999f5 @@ -225,7 +226,6 @@ F src/test_osinst.c 90a845c8183013d80eccb1f29e8805608516edba F src/test_pcache.c a5cd24730cb43c5b18629043314548c9169abb00 F src/test_quota.c 1ec82e02fd3643899e9a5de9684515e84641c91f F src/test_quota.h 8761e463b25e75ebc078bd67d70e39b9c817a0cb -F src/test_regexp.c 06ae8138d41a793330f62351283dd6f6f21104f4 F src/test_rtree.c aba603c949766c4193f1068b91c787f57274e0d9 F src/test_schema.c 8c06ef9ddb240c7a0fcd31bc221a6a2aade58bf0 F src/test_server.c 2f99eb2837dfa06a4aacf24af24c6affdf66a84f @@ -684,7 +684,7 @@ F test/quote.test 215897dbe8de1a6f701265836d6601cc6ed103e6 F test/randexpr1.tcl 40dec52119ed3a2b8b2a773bce24b63a3a746459 F test/randexpr1.test eda062a97e60f9c38ae8d806b03b0ddf23d796df F test/rdonly.test c267d050a1d9a6a321de502b737daf28821a518d -F test/regexp1.test 5cbb6e7122ca51260d71079cf9245b63b8f64e1a +F test/regexp1.test 497ea812f264d12b6198d6e50a76be4a1973a9d8 F test/reindex.test 44edd3966b474468b823d481eafef0c305022254 F test/releasetest.mk 2eced2f9ae701fd0a29e714a241760503ccba25a F test/releasetest.tcl 06d289d8255794073a58d2850742f627924545ce @@ -1054,7 +1054,10 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 49cfa14fceeef2d55b449eb927c283ce6f650c07 -R 83c630a254d9cc209b10e1a83d85002b +P 1a1cf5aa86734c832d845e07780262a178188d56 +R d55f2fee19b165fc6c487ed43598c861 +T *branch * std-ext +T *sym-std-ext * +T -sym-trunk * U drh -Z a7a8588b07b6fce1e57eced049fe41f1 +Z 87544ad6dda5ce727b201c9348b14476 diff --git a/manifest.uuid b/manifest.uuid index fa38fa5b2c..f18da3e0e5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1a1cf5aa86734c832d845e07780262a178188d56 \ No newline at end of file +860fc393bff27045e0593c3c51bf5577accd0b79 \ No newline at end of file diff --git a/src/tclsqlite.c b/src/tclsqlite.c index c0c4fb6499..f8d428e7a8 100644 --- a/src/tclsqlite.c +++ b/src/tclsqlite.c @@ -3685,7 +3685,6 @@ static void init_all(Tcl_Interp *interp){ extern int SqlitetestSyscall_Init(Tcl_Interp*); extern int Sqlitetestfuzzer_Init(Tcl_Interp*); extern int Sqlitetestwholenumber_Init(Tcl_Interp*); - extern int Sqlitetestregexp_Init(Tcl_Interp*); #if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) extern int Sqlitetestfts3_Init(Tcl_Interp *interp); @@ -3730,7 +3729,6 @@ static void init_all(Tcl_Interp *interp){ SqlitetestSyscall_Init(interp); Sqlitetestfuzzer_Init(interp); Sqlitetestwholenumber_Init(interp); - Sqlitetestregexp_Init(interp); #if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) Sqlitetestfts3_Init(interp); diff --git a/src/test1.c b/src/test1.c index 63d014cc17..178aba809d 100644 --- a/src/test1.c +++ b/src/test1.c @@ -6045,6 +6045,53 @@ static int optimization_control( return TCL_OK; } +typedef struct sqlite3_api_routines sqlite3_api_routines; +/* +** load_static_extension DB NAME +** +** Load an extension that is statically linked. +*/ +static int tclLoadStaticExtensionCmd( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + extern int sqlite3_regexp_init(sqlite3*,char**,const sqlite3_api_routines*); + static const struct { + const char *zExtName; + int (*pInit)(sqlite3*,char**,const sqlite3_api_routines*); + } aExtension[] = { + { "regexp", sqlite3_regexp_init }, + }; + sqlite3 *db; + const char *zName; + int i, rc; + char *zErrMsg = 0; + if( objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB NAME"); + return TCL_ERROR; + } + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; + zName = Tcl_GetString(objv[2]); + for(i=0; i=ArraySize(aExtension) ){ + Tcl_AppendResult(interp, "no such extension: ", zName, (char*)0); + return TCL_ERROR; + } + rc = aExtension[i].pInit(db, &zErrMsg, 0); + if( rc!=SQLITE_OK || zErrMsg ){ + Tcl_AppendResult(interp, "initialization of ", zName, " failed: ", zErrMsg, + (char*)0); + sqlite3_free(zErrMsg); + return TCL_ERROR; + } + return TCL_OK; +} + + /* ** Register commands with the TCL interpreter. */ @@ -6266,6 +6313,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ #if SQLITE_OS_UNIX { "getrusage", test_getrusage }, #endif + { "load_static_extension", tclLoadStaticExtensionCmd }, }; static int bitmask_size = sizeof(Bitmask)*8; int i; diff --git a/test/regexp1.test b/test/regexp1.test index b7ec8fdeab..0e63cd98c8 100644 --- a/test/regexp1.test +++ b/test/regexp1.test @@ -16,7 +16,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl do_test regexp1-1.1 { - sqlite3_add_regexp_func db + load_static_extension db regexp db eval { CREATE TABLE t1(x INTEGER PRIMARY KEY, y TEXT); INSERT INTO t1 VALUES(1, 'For since by man came death,'); From ed17167e1dc966bfb871aea8297206991eac2794 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 24 Apr 2013 13:50:09 +0000 Subject: [PATCH 02/32] Fix a simple comment typo. No changes to code. FossilOrigin-Name: f136bd95824dc95b9e6acdc4d55db263ba13fbaa --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/select.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index f702eb82f2..7f10b643a0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarnings. -D 2013-04-22T23:38:50.007 +C Fix\sa\ssimple\scomment\stypo.\s\sNo\schanges\sto\scode. +D 2013-04-24T13:50:09.308 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 87591ea5bf7d6ed521ad42d5bc69c124debe11a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -181,7 +181,7 @@ F src/printf.c 4a9f882f1c1787a8b494a2987765acf9d97ac21f F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 F src/resolve.c 10a1b332e3eb36e5d561085e18c58a8578cd7d73 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c 01540bcd3df3c8f1187158e77986028b1c667258 +F src/select.c 8d097454ff56bdda38c4d877757f592a3c823d15 F src/shell.c aca9d94653decd4496846dee0c7ba83eaf96a46d F src/sqlite.h.in ec279b782bea05db63b8b29481f9642b406004af F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 @@ -1054,7 +1054,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 49cfa14fceeef2d55b449eb927c283ce6f650c07 -R 83c630a254d9cc209b10e1a83d85002b +P 1a1cf5aa86734c832d845e07780262a178188d56 +R 52301974d4ec9029bc00a43d1015330a U drh -Z a7a8588b07b6fce1e57eced049fe41f1 +Z d70e5c91a8e7a901cc898018c72d77a1 diff --git a/manifest.uuid b/manifest.uuid index fa38fa5b2c..6407b99609 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1a1cf5aa86734c832d845e07780262a178188d56 \ No newline at end of file +f136bd95824dc95b9e6acdc4d55db263ba13fbaa \ No newline at end of file diff --git a/src/select.c b/src/select.c index c7948096c9..1daa5bed36 100644 --- a/src/select.c +++ b/src/select.c @@ -4047,7 +4047,7 @@ int sqlite3Select( pItem->addrFillSub = topAddr+1; VdbeNoopComment((v, "materialize %s", pItem->pTab->zName)); if( pItem->isCorrelated==0 ){ - /* If the subquery is no correlated and if we are not inside of + /* If the subquery is not correlated and if we are not inside of ** a trigger, then we only need to compute the value of the subquery ** once. */ onceAddr = sqlite3CodeOnce(pParse); From aa87f9a68b5ac0305699507574cb2ed074e722b8 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 25 Apr 2013 00:57:10 +0000 Subject: [PATCH 03/32] Make sure the affinity and datatype of sub-subqueries are initialized prior to subqueries as the latter relies on the former. FossilOrigin-Name: 39b4e6ff9316cc78ea88349091e195b8104d1e9e --- manifest | 22 +++++++++++----------- manifest.uuid | 2 +- src/expr.c | 3 ++- src/resolve.c | 2 ++ src/select.c | 3 +++ src/sqliteInt.h | 1 + src/walker.c | 17 ++++++++++++++--- test/selectD.test | 19 +++++++++++++++++++ 8 files changed, 53 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 7f10b643a0..31cc673a2f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\ssimple\scomment\stypo.\s\sNo\schanges\sto\scode. -D 2013-04-24T13:50:09.308 +C Make\ssure\sthe\saffinity\sand\sdatatype\sof\ssub-subqueries\sare\sinitialized\nprior\sto\ssubqueries\sas\sthe\slatter\srelies\son\sthe\sformer. +D 2013-04-25T00:57:10.497 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 87591ea5bf7d6ed521ad42d5bc69c124debe11a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -137,7 +137,7 @@ F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 4262c227bc91cecc61ae37ed3a40f08069cfa267 F src/date.c 067a81c9942c497aafd2c260e13add8a7d0c7dd4 F src/delete.c aeabdabeeeaa0584127f291baa9617153d334778 -F src/expr.c 48048fca951eedbc74aa32262154410d56c83812 +F src/expr.c 437c03d5bb4fe3a53ecab3ad0286d6c5260da7ed F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c e16942bd5c8a868ac53287886464a5ed0e72b179 F src/func.c d3fdcff9274bc161152e67ed3f626841c247f4b9 @@ -179,14 +179,14 @@ F src/pragma.c 3eacf001cbf4becbd494f8d82d08fdf1648cf8cb F src/prepare.c 743e484233c51109666d402f470523553b41797c F src/printf.c 4a9f882f1c1787a8b494a2987765acf9d97ac21f F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 -F src/resolve.c 10a1b332e3eb36e5d561085e18c58a8578cd7d73 +F src/resolve.c 83cc2d942ee216bc56956c6e6fadb691c1727fa1 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c 8d097454ff56bdda38c4d877757f592a3c823d15 +F src/select.c 6bfbe11e2fef81c5e18d30513ab6c69f171667eb F src/shell.c aca9d94653decd4496846dee0c7ba83eaf96a46d F src/sqlite.h.in ec279b782bea05db63b8b29481f9642b406004af F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h d936f797812c28b81b26ed18345baf8db28a21a5 -F src/sqliteInt.h 2a83cfec9963372b636b0cabd4b200c1f1074a99 +F src/sqliteInt.h 3585ea1bb8776baa9f2675e9ef3d9170e7aeda29 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -258,7 +258,7 @@ F src/vdbetrace.c 3ad1b4e92b60c082a02ac563da4a2735cc7d297c F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h a4d3da523d55a226a0b28e9058ef88d0a8051887 -F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b +F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 F src/where.c d54e63087b52c309550aa2defdb20ef27add9f9a F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -719,7 +719,7 @@ F test/select9.test c0ca3cd87a8ebb04de2cb1402c77df55d911a0ea F test/selectA.test 06d1032fa9009314c95394f2ca2e60d9f7ae8532 F test/selectB.test 954e4e49cf1f896d61794e440669e03a27ceea25 F test/selectC.test 871fb55d884d3de5943c4057ebd22c2459e71977 -F test/selectD.test 03f7c1ea8d5ab3c637cbc30fcbbbac96b988c162 +F test/selectD.test b0f02a04ef7737decb24e08be2c39b9664b43394 F test/server1.test 46803bd3fe8b99b30dbc5ff38ffc756f5c13a118 F test/shared.test 1da9dbad400cee0d93f252ccf76e1ae007a63746 F test/shared2.test 03eb4a8d372e290107d34b6ce1809919a698e879 @@ -1054,7 +1054,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 1a1cf5aa86734c832d845e07780262a178188d56 -R 52301974d4ec9029bc00a43d1015330a +P f136bd95824dc95b9e6acdc4d55db263ba13fbaa +R 35e39801836ac191fc1f292f3b81674a U drh -Z d70e5c91a8e7a901cc898018c72d77a1 +Z 424df927045edc37db2bc00e1032f86c diff --git a/manifest.uuid b/manifest.uuid index 6407b99609..4aeba6cd08 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f136bd95824dc95b9e6acdc4d55db263ba13fbaa \ No newline at end of file +39b4e6ff9316cc78ea88349091e195b8104d1e9e \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index ae6a1dec10..a974c5a61f 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1214,6 +1214,7 @@ static int selectNodeIsConstant(Walker *pWalker, Select *NotUsed){ } static int exprIsConst(Expr *p, int initFlag){ Walker w; + memset(&w, 0, sizeof(w)); w.u.i = initFlag; w.xExprCallback = exprNodeIsConstant; w.xSelectCallback = selectNodeIsConstant; @@ -3428,8 +3429,8 @@ void sqlite3ExprCodeConstants(Parse *pParse, Expr *pExpr){ Walker w; if( pParse->cookieGoto ) return; if( OptimizationDisabled(pParse->db, SQLITE_FactorOutConst) ) return; + memset(&w, 0, sizeof(w)); w.xExprCallback = evalConstExpr; - w.xSelectCallback = 0; w.pParse = pParse; sqlite3WalkExpr(&w, pExpr); } diff --git a/src/resolve.c b/src/resolve.c index 9b350caf80..a8e196926c 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -1283,6 +1283,7 @@ int sqlite3ResolveExprNames( #endif savedHasAgg = pNC->ncFlags & NC_HasAgg; pNC->ncFlags &= ~NC_HasAgg; + memset(&w, 0, sizeof(w)); w.xExprCallback = resolveExprStep; w.xSelectCallback = resolveSelectStep; w.pParse = pNC->pParse; @@ -1323,6 +1324,7 @@ void sqlite3ResolveSelectNames( Walker w; assert( p!=0 ); + memset(&w, 0, sizeof(w)); w.xExprCallback = resolveExprStep; w.xSelectCallback = resolveSelectStep; w.pParse = pParse; diff --git a/src/select.c b/src/select.c index 1daa5bed36..a745dc370a 100644 --- a/src/select.c +++ b/src/select.c @@ -3576,6 +3576,7 @@ static int exprWalkNoop(Walker *NotUsed, Expr *NotUsed2){ */ static void sqlite3SelectExpand(Parse *pParse, Select *pSelect){ Walker w; + memset(&w, 0, sizeof(w)); w.xSelectCallback = selectExpander; w.xExprCallback = exprWalkNoop; w.pParse = pParse; @@ -3634,9 +3635,11 @@ static int selectAddSubqueryTypeInfo(Walker *pWalker, Select *p){ static void sqlite3SelectAddTypeInfo(Parse *pParse, Select *pSelect){ #ifndef SQLITE_OMIT_SUBQUERY Walker w; + memset(&w, 0, sizeof(w)); w.xSelectCallback = selectAddSubqueryTypeInfo; w.xExprCallback = exprWalkNoop; w.pParse = pParse; + w.bSelectDepthFirst = 1; sqlite3WalkSelect(&w, pSelect); #endif } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 3722041b0a..876629a8c8 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2588,6 +2588,7 @@ struct Walker { int (*xSelectCallback)(Walker*,Select*); /* Callback for SELECTs */ Parse *pParse; /* Parser context. */ int walkerDepth; /* Number of subqueries */ + u8 bSelectDepthFirst; /* Do subqueries first */ union { /* Extra data for callback */ NameContext *pNC; /* Naming context */ int i; /* Integer value */ diff --git a/src/walker.c b/src/walker.c index eab96ea24d..e71ed2ac48 100644 --- a/src/walker.c +++ b/src/walker.c @@ -113,7 +113,9 @@ int sqlite3WalkSelectFrom(Walker *pWalker, Select *p){ /* ** Call sqlite3WalkExpr() for every expression in Select statement p. ** Invoke sqlite3WalkSelect() for subqueries in the FROM clause and -** on the compound select chain, p->pPrior. +** on the compound select chain, p->pPrior. Invoke the xSelectCallback() +** either before or after the walk of expressions and FROM clause, depending +** on whether pWalker->bSelectDepthFirst is false or true, respectively. ** ** Return WRC_Continue under normal conditions. Return WRC_Abort if ** there is an abort request. @@ -127,14 +129,23 @@ int sqlite3WalkSelect(Walker *pWalker, Select *p){ rc = WRC_Continue; pWalker->walkerDepth++; while( p ){ - rc = pWalker->xSelectCallback(pWalker, p); - if( rc ) break; + if( !pWalker->bSelectDepthFirst ){ + rc = pWalker->xSelectCallback(pWalker, p); + if( rc ) break; + } if( sqlite3WalkSelectExpr(pWalker, p) || sqlite3WalkSelectFrom(pWalker, p) ){ pWalker->walkerDepth--; return WRC_Abort; } + if( pWalker->bSelectDepthFirst ){ + rc = pWalker->xSelectCallback(pWalker, p); + /* Depth-first search is currently only used for + ** selectAddSubqueryTypeInfo() and that routine always returns + ** WRC_Continue (0). So the following branch is never taken. */ + if( NEVER(rc) ) break; + } p = p->pPrior; } pWalker->walkerDepth--; diff --git a/test/selectD.test b/test/selectD.test index aa8c328ee9..89f999eb6d 100644 --- a/test/selectD.test +++ b/test/selectD.test @@ -152,4 +152,23 @@ for {set i 1} {$i<=2} {incr i} { } {111 x1 111 x2 222 x3 {}} } +# The following test was added on 2013-04-24 in order to verify that +# the datatypes and affinities of sub-sub-queries are set prior to computing +# the datatypes and affinities of the parent sub-queries because the +# latter computation depends on the former. +# +do_execsql_test selectD-4.1 { + CREATE TABLE t41(a INTEGER PRIMARY KEY, b INTEGER); + CREATE TABLE t42(d INTEGER PRIMARY KEY, e INTEGER); + CREATE TABLE t43(f INTEGER PRIMARY KEY, g INTEGER); + EXPLAIN QUERY PLAN + SELECT * + FROM t41 + LEFT JOIN (SELECT count(*) AS cnt, x1.d + FROM (t42 INNER JOIN t43 ON d=g) AS x1 + WHERE x1.d>5 + GROUP BY x1.d) AS x2 + ON t41.b=x2.d; +} {/.*SEARCH SUBQUERY 1 AS x2 USING AUTOMATIC.*/} + finish_test From 24b6422dcb66ec9ad8828b2a65e6e5afc224be22 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 25 Apr 2013 11:58:36 +0000 Subject: [PATCH 04/32] Make test_wholenumber.c into a loadable extension and move it to ext/misc/wholenumber.c. FossilOrigin-Name: efcc9dd012b5f193324dfc2ee9c2410c16fc1b3b --- .../misc/wholenumber.c | 65 ++++--------------- main.mk | 2 +- manifest | 33 +++++----- manifest.uuid | 2 +- src/tclsqlite.c | 2 - src/test1.c | 4 +- test/8_3_names.test | 4 +- test/analyze7.test | 2 +- test/memdb.test | 2 +- test/tkt-2d1a5c67d.test | 2 +- test/zerodamage.test | 2 +- 11 files changed, 37 insertions(+), 83 deletions(-) rename src/test_wholenumber.c => ext/misc/wholenumber.c (83%) diff --git a/src/test_wholenumber.c b/ext/misc/wholenumber.c similarity index 83% rename from src/test_wholenumber.c rename to ext/misc/wholenumber.c index 7c42d01691..a6d79507c4 100644 --- a/src/test_wholenumber.c +++ b/ext/misc/wholenumber.c @@ -22,7 +22,8 @@ ** ** 1 2 3 4 5 6 7 8 9 */ -#include "sqlite3.h" +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 #include #include @@ -250,62 +251,18 @@ static sqlite3_module wholenumberModule = { #endif /* SQLITE_OMIT_VIRTUALTABLE */ - -/* -** Register the wholenumber virtual table -*/ -int wholenumber_register(sqlite3 *db){ +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_wholenumber_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); #ifndef SQLITE_OMIT_VIRTUALTABLE rc = sqlite3_create_module(db, "wholenumber", &wholenumberModule, 0); #endif return rc; } - -#ifdef SQLITE_TEST -#include -/* -** Decode a pointer to an sqlite3 object. -*/ -extern int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite3 **ppDb); - -/* -** Register the echo virtual table module. -*/ -static int register_wholenumber_module( - ClientData clientData, /* Pointer to sqlite3_enable_XXX function */ - Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ - int objc, /* Number of arguments */ - Tcl_Obj *CONST objv[] /* Command arguments */ -){ - sqlite3 *db; - if( objc!=2 ){ - Tcl_WrongNumArgs(interp, 1, objv, "DB"); - return TCL_ERROR; - } - if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; - wholenumber_register(db); - return TCL_OK; -} - - -/* -** Register commands with the TCL interpreter. -*/ -int Sqlitetestwholenumber_Init(Tcl_Interp *interp){ - static struct { - char *zName; - Tcl_ObjCmdProc *xProc; - void *clientData; - } aObjCmd[] = { - { "register_wholenumber_module", register_wholenumber_module, 0 }, - }; - int i; - for(i=0; i Date: Thu, 25 Apr 2013 14:31:46 +0000 Subject: [PATCH 05/32] Convert the fuzzer virtual table into a loadable extension and move it to the ext/misc/fuzzer.c file. FossilOrigin-Name: c8c69307f60c1d07ac666ae3797b7e3f286fd491 --- Makefile.in | 10 ++-- Makefile.msc | 11 +++-- src/test_fuzzer.c => ext/misc/fuzzer.c | 68 +++++--------------------- main.mk | 11 +++-- manifest | 28 +++++------ manifest.uuid | 2 +- src/tclsqlite.c | 2 - src/test1.c | 2 + test/fuzzer1.test | 7 +-- test/fuzzerfault.test | 2 +- 10 files changed, 54 insertions(+), 89 deletions(-) rename src/test_fuzzer.c => ext/misc/fuzzer.c (96%) diff --git a/Makefile.in b/Makefile.in index 5c636209d8..597994c15c 100644 --- a/Makefile.in +++ b/Makefile.in @@ -361,7 +361,6 @@ TESTSRC = \ $(TOP)/src/test_devsym.c \ $(TOP)/src/test_fs.c \ $(TOP)/src/test_func.c \ - $(TOP)/src/test_fuzzer.c \ $(TOP)/src/test_hexio.c \ $(TOP)/src/test_init.c \ $(TOP)/src/test_intarray.c \ @@ -373,7 +372,6 @@ TESTSRC = \ $(TOP)/src/test_osinst.c \ $(TOP)/src/test_pcache.c \ $(TOP)/src/test_quota.c \ - $(TOP)/src/test_regexp.c \ $(TOP)/src/test_rtree.c \ $(TOP)/src/test_schema.c \ $(TOP)/src/test_server.c \ @@ -383,11 +381,17 @@ TESTSRC = \ $(TOP)/src/test_tclvar.c \ $(TOP)/src/test_thread.c \ $(TOP)/src/test_vfs.c \ - $(TOP)/src/test_wholenumber.c \ $(TOP)/src/test_wsd.c \ $(TOP)/ext/fts3/fts3_term.c \ $(TOP)/ext/fts3/fts3_test.c +# Statically linked extensions +# +TESTSRC += \ + $(TOP)/ext/misc/fuzzer.c \ + $(TOP)/ext/misc/regexp.c \ + $(TOP)/ext/misc/wholenumber.c + # Source code to the library files needed by the test fixture # TESTSRC2 = \ diff --git a/Makefile.msc b/Makefile.msc index 47f815e677..982a959dc6 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -681,7 +681,6 @@ TESTSRC = \ $(TOP)\src\test_devsym.c \ $(TOP)\src\test_fs.c \ $(TOP)\src\test_func.c \ - $(TOP)\src\test_fuzzer.c \ $(TOP)\src\test_hexio.c \ $(TOP)\src\test_init.c \ $(TOP)\src\test_intarray.c \ @@ -693,7 +692,6 @@ TESTSRC = \ $(TOP)\src\test_osinst.c \ $(TOP)\src\test_pcache.c \ $(TOP)\src\test_quota.c \ - $(TOP)\src\test_regexp.c \ $(TOP)\src\test_rtree.c \ $(TOP)\src\test_schema.c \ $(TOP)\src\test_server.c \ @@ -703,11 +701,18 @@ TESTSRC = \ $(TOP)\src\test_tclvar.c \ $(TOP)\src\test_thread.c \ $(TOP)\src\test_vfs.c \ - $(TOP)\src\test_wholenumber.c \ $(TOP)\src\test_wsd.c \ $(TOP)\ext\fts3\fts3_term.c \ $(TOP)\ext\fts3\fts3_test.c +# Statically linked extensions +# +TESTSRC += \ + $(TOP)\ext\misc\fuzzer.c \ + $(TOP)\ext\misc\regexp.c \ + $(TOP)\ext\misc\wholenumber.c + + # Source code to the library files needed by the test fixture # TESTSRC2 = \ diff --git a/src/test_fuzzer.c b/ext/misc/fuzzer.c similarity index 96% rename from src/test_fuzzer.c rename to ext/misc/fuzzer.c index 10496f2ea7..c0c294b11c 100644 --- a/src/test_fuzzer.c +++ b/ext/misc/fuzzer.c @@ -141,13 +141,14 @@ ** of the strings in the second or third column of the fuzzer data table ** is 50 bytes. The maximum cost on a rule is 1000. */ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 /* If SQLITE_DEBUG is not defined, disable assert statements. */ #if !defined(NDEBUG) && !defined(SQLITE_DEBUG) # define NDEBUG #endif -#include "sqlite3.h" #include #include #include @@ -1155,61 +1156,16 @@ static sqlite3_module fuzzerModule = { #endif /* SQLITE_OMIT_VIRTUALTABLE */ -/* -** Register the fuzzer virtual table -*/ -int fuzzer_register(sqlite3 *db){ - int rc = SQLITE_OK; -#ifndef SQLITE_OMIT_VIRTUALTABLE - rc = sqlite3_create_module(db, "fuzzer", &fuzzerModule, 0); +#ifdef _WIN32 +__declspec(dllexport) #endif +int sqlite3_fuzzer_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + rc = sqlite3_create_module(db, "fuzzer", &fuzzerModule, 0); return rc; } - -#ifdef SQLITE_TEST -#include -/* -** Decode a pointer to an sqlite3 object. -*/ -extern int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite3 **ppDb); - -/* -** Register the echo virtual table module. -*/ -static int register_fuzzer_module( - ClientData clientData, /* Pointer to sqlite3_enable_XXX function */ - Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ - int objc, /* Number of arguments */ - Tcl_Obj *CONST objv[] /* Command arguments */ -){ - sqlite3 *db; - if( objc!=2 ){ - Tcl_WrongNumArgs(interp, 1, objv, "DB"); - return TCL_ERROR; - } - getDbPointer(interp, Tcl_GetString(objv[1]), &db); - fuzzer_register(db); - return TCL_OK; -} - - -/* -** Register commands with the TCL interpreter. -*/ -int Sqlitetestfuzzer_Init(Tcl_Interp *interp){ - static struct { - char *zName; - Tcl_ObjCmdProc *xProc; - void *clientData; - } aObjCmd[] = { - { "register_fuzzer_module", register_fuzzer_module, 0 }, - }; - int i; - for(i=0; i Date: Thu, 25 Apr 2013 14:36:28 +0000 Subject: [PATCH 06/32] Fix the MSVC makefile so that it works with the ext/misc/*.c extensions. FossilOrigin-Name: 680822e892f3efdb702eea3b321bc5785239dd56 --- Makefile.msc | 6 +++--- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Makefile.msc b/Makefile.msc index 982a959dc6..ecdac8c412 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -707,7 +707,7 @@ TESTSRC = \ # Statically linked extensions # -TESTSRC += \ +TESTEXT = \ $(TOP)\ext\misc\fuzzer.c \ $(TOP)\ext\misc\regexp.c \ $(TOP)\ext\misc\wholenumber.c @@ -1209,8 +1209,8 @@ rtree.lo: $(TOP)\ext\rtree\rtree.c $(HDR) $(EXTHDR) TESTFIXTURE_FLAGS = -DTCLSH=1 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE -TESTFIXTURE_SRC0 = $(TESTSRC2) libsqlite3.lib -TESTFIXTURE_SRC1 = sqlite3.c +TESTFIXTURE_SRC0 = $(TESTEXT) $(TESTSRC2) libsqlite3.lib +TESTFIXTURE_SRC1 = $(TESTEXT) sqlite3.c !IF $(USE_AMALGAMATION)==0 TESTFIXTURE_SRC = $(TESTSRC) $(TOP)\src\tclsqlite.c $(TESTFIXTURE_SRC0) !ELSE diff --git a/manifest b/manifest index 9878de9b34..355653e47b 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Convert\sthe\sfuzzer\svirtual\stable\sinto\sa\sloadable\sextension\sand\smove\sit\nto\sthe\sext/misc/fuzzer.c\sfile. -D 2013-04-25T14:31:46.866 +C Fix\sthe\sMSVC\smakefile\sso\sthat\sit\sworks\swith\sthe\sext/misc/*.c\sextensions. +D 2013-04-25T14:36:28.444 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 422a8541e3595e4fd4d962a46f52fbe095a31bd2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 -F Makefile.msc ef00ccb017f6da580ab934d096b08da40d81f1a5 +F Makefile.msc 3657134d4304934fd9a2c81ff6a2bd7e2187954b F Makefile.vxworks db21ed42a01d5740e656b16f92cb5d8d5e5dd315 F README cd04a36fbc7ea56932a4052d7d0b7f09f27c33d6 F VERSION 05c7bd63b96f31cfdef5c766ed91307ac121f5aa @@ -83,7 +83,7 @@ F ext/fts3/unicode/mkunicode.tcl 7a9bc018e2962abb79563c5a39fe581fcbf2f675 F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43 F ext/icu/icu.c eb9ae1d79046bd7871aa97ee6da51eb770134b5a F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37 -F ext/misc/fuzzer.c fb64a15af978ae73fa9075b9b1dfbe82b8defc6f w src/test_fuzzer.c +F ext/misc/fuzzer.c fb64a15af978ae73fa9075b9b1dfbe82b8defc6f F ext/misc/regexp.c c0fdb8af86981ff9890d776cfb97fe66297cc3b2 F ext/misc/wholenumber.c ce362368b9381ea48cbd951ade8df867eeeab014 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 @@ -1054,7 +1054,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P efcc9dd012b5f193324dfc2ee9c2410c16fc1b3b -R 2b859f3329d083b32ee90e4538687497 +P c8c69307f60c1d07ac666ae3797b7e3f286fd491 +R 3a7355d9d00cf8a32d1b35340d234e67 U drh -Z 672d17ed58c8e3fac84edc0141a68b79 +Z 8dc283cf0c63ecf74b33a1f0701aa7d4 diff --git a/manifest.uuid b/manifest.uuid index 6f090d8535..b8ed72b0f8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c8c69307f60c1d07ac666ae3797b7e3f286fd491 \ No newline at end of file +680822e892f3efdb702eea3b321bc5785239dd56 \ No newline at end of file From b7045ab2eda7ad2179d15a5d1ae9a4e807148066 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 25 Apr 2013 14:59:01 +0000 Subject: [PATCH 07/32] Move the test_spellfix.c module to ext/misc/spellfix.c. FossilOrigin-Name: de556add10150140981a2e34b3712e96a7c262e3 --- Makefile.in | 1 + Makefile.msc | 1 + src/test_spellfix.c => ext/misc/spellfix.c | 42 ++++++++-------------- main.mk | 1 + manifest | 26 +++++++------- manifest.uuid | 2 +- src/shell.c | 12 ------- src/test1.c | 2 ++ src/test8.c | 24 ------------- test/spellfix.test | 2 +- 10 files changed, 35 insertions(+), 78 deletions(-) rename src/test_spellfix.c => ext/misc/spellfix.c (99%) diff --git a/Makefile.in b/Makefile.in index 597994c15c..74bf58e31f 100644 --- a/Makefile.in +++ b/Makefile.in @@ -390,6 +390,7 @@ TESTSRC = \ TESTSRC += \ $(TOP)/ext/misc/fuzzer.c \ $(TOP)/ext/misc/regexp.c \ + $(TOP)/ext/misc/spellfix.c \ $(TOP)/ext/misc/wholenumber.c # Source code to the library files needed by the test fixture diff --git a/Makefile.msc b/Makefile.msc index ecdac8c412..ea8fc10a2e 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -710,6 +710,7 @@ TESTSRC = \ TESTEXT = \ $(TOP)\ext\misc\fuzzer.c \ $(TOP)\ext\misc\regexp.c \ + $(TOP)\ext\misc\spellfix.c \ $(TOP)\ext\misc\wholenumber.c diff --git a/src/test_spellfix.c b/ext/misc/spellfix.c similarity index 99% rename from src/test_spellfix.c rename to ext/misc/spellfix.c index 16376244a3..a8492185b3 100644 --- a/src/test_spellfix.c +++ b/ext/misc/spellfix.c @@ -14,20 +14,17 @@ ** to search a large vocabulary for close matches. See separate ** documentation files (spellfix1.wiki and editdist3.wiki) for details. */ -#if SQLITE_CORE -# include "sqliteInt.h" -#else -# include -# include -# include -# include "sqlite3ext.h" -# include -# define ALWAYS(X) 1 -# define NEVER(X) 0 - typedef unsigned char u8; - typedef unsigned short u16; - SQLITE_EXTENSION_INIT1 -#endif /* !SQLITE_CORE */ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 + +#include +#include +#include +#include +#define ALWAYS(X) 1 +#define NEVER(X) 0 +typedef unsigned char u8; +typedef unsigned short u16; #include /* @@ -2822,21 +2819,13 @@ static int spellfix1Register(sqlite3 *db){ return rc; } -#if SQLITE_CORE || defined(SQLITE_TEST) -/* -** Register the spellfix1 virtual table and its associated functions. -*/ -int sqlite3_spellfix1_register(sqlite3 *db){ - return spellfix1Register(db); -} -#endif - - -#if !SQLITE_CORE /* ** Extension load function. */ -int sqlite3_spellfix1_init( +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_spellfix_init( sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi @@ -2844,4 +2833,3 @@ int sqlite3_spellfix1_init( SQLITE_EXTENSION_INIT2(pApi); return spellfix1Register(db); } -#endif /* !SQLITE_CORE */ diff --git a/main.mk b/main.mk index dd85a9acbe..eb3da5eaee 100644 --- a/main.mk +++ b/main.mk @@ -272,6 +272,7 @@ TESTSRC = \ TESTSRC += \ $(TOP)/ext/misc/fuzzer.c \ $(TOP)/ext/misc/regexp.c \ + $(TOP)/ext/misc/spellfix.c \ $(TOP)/ext/misc/wholenumber.c diff --git a/manifest b/manifest index 355653e47b..4c3502f113 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Fix\sthe\sMSVC\smakefile\sso\sthat\sit\sworks\swith\sthe\sext/misc/*.c\sextensions. -D 2013-04-25T14:36:28.444 +C Move\sthe\stest_spellfix.c\smodule\sto\sext/misc/spellfix.c. +D 2013-04-25T14:59:01.366 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f -F Makefile.in 422a8541e3595e4fd4d962a46f52fbe095a31bd2 +F Makefile.in 10c635460b6c3a20741d71c3a1b65b0ebec7558b F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 -F Makefile.msc 3657134d4304934fd9a2c81ff6a2bd7e2187954b +F Makefile.msc 97a23e910afb8c2d23db0217c343481d59045acb F Makefile.vxworks db21ed42a01d5740e656b16f92cb5d8d5e5dd315 F README cd04a36fbc7ea56932a4052d7d0b7f09f27c33d6 F VERSION 05c7bd63b96f31cfdef5c766ed91307ac121f5aa @@ -85,6 +85,7 @@ F ext/icu/icu.c eb9ae1d79046bd7871aa97ee6da51eb770134b5a F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37 F ext/misc/fuzzer.c fb64a15af978ae73fa9075b9b1dfbe82b8defc6f F ext/misc/regexp.c c0fdb8af86981ff9890d776cfb97fe66297cc3b2 +F ext/misc/spellfix.c 8bb699116e36cc5e68d7ddf1810b638a3090c744 w src/test_spellfix.c F ext/misc/wholenumber.c ce362368b9381ea48cbd951ade8df867eeeab014 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 757abea591d4ff67c0ff4e8f9776aeda86b18c14 @@ -107,7 +108,7 @@ F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 -F main.mk bf3cf721c54c97e0eb02856d7d67828ac9738c4e +F main.mk a6183110637d782988fdf3735f7d125b19639967 F mkdll.sh 7d09b23c05d56532e9d44a50868eb4b12ff4f74a F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f F mkextw.sh 4123480947681d9b434a5e7b1ee08135abe409ac @@ -185,7 +186,7 @@ F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 F src/resolve.c 10a1b332e3eb36e5d561085e18c58a8578cd7d73 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 01540bcd3df3c8f1187158e77986028b1c667258 -F src/shell.c aca9d94653decd4496846dee0c7ba83eaf96a46d +F src/shell.c 5d527e5d08f05ec2c43ff194ea44bf62b974f4c9 F src/sqlite.h.in ec279b782bea05db63b8b29481f9642b406004af F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h d936f797812c28b81b26ed18345baf8db28a21a5 @@ -194,14 +195,14 @@ F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c 2ecec9937e69bc17560ad886da35195daa7261b8 -F src/test1.c aba32f14372febad3e7c14798b25bdd9fd0b7c61 +F src/test1.c 1c7fa0a36703508130d7ed65325883a9e453bf72 F src/test2.c 29e7154112f7448d64204e8d31179cf497ecf425 F src/test3.c 96aed72a8e1d542fed127e3e8350ae357712fa82 F src/test4.c cea2c55110241e4674e66d476d29c914627999f5 F src/test5.c a6d1ac55ac054d0b2b8f37b5e655b6c92645a013 F src/test6.c a437f76f9874d2563352a7e6cd0d43217663c220 F src/test7.c f4b894b7931f8cf9f5cbf37cfa0727703f526a40 -F src/test8.c 58ea1d9698f3947e4662107ef98f429e84ae20e0 +F src/test8.c f7e729e3e1973f68e6d98f5aa65046e3e2cb0bad F src/test9.c bea1e8cf52aa93695487badedd6e1886c321ea60 F src/test_async.c 0612a752896fad42d55c3999a5122af10dcf22ad F src/test_autoext.c 5c95b5d435eaa09d6c0e7d90371c5ca8cd567701 @@ -230,7 +231,6 @@ F src/test_quota.h 8761e463b25e75ebc078bd67d70e39b9c817a0cb F src/test_rtree.c aba603c949766c4193f1068b91c787f57274e0d9 F src/test_schema.c 8c06ef9ddb240c7a0fcd31bc221a6a2aade58bf0 F src/test_server.c 2f99eb2837dfa06a4aacf24af24c6affdf66a84f -F src/test_spellfix.c 56dfa6d583ac34f61af0834d7b58d674e7e18e13 F src/test_sqllog.c c1c1bbedbcaf82b93d83e4f9dd990e62476a680e F src/test_stat.c d1569c7a4839f13e80187e2c26b2ab4da2d03935 F src/test_superlock.c 2b97936ca127d13962c3605dbc9a4ef269c424cd @@ -750,7 +750,7 @@ F test/speed3.test d32043614c08c53eafdc80f33191d5bd9b920523 F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa F test/speed4p.test 0e51908951677de5a969b723e03a27a1c45db38b -F test/spellfix.test 52ae2680b1247c52b9e2b2116de3fd26a78e6bd2 +F test/spellfix.test a85915ab25af7fcfb0d99cb1951e6ef15e26202c F test/sqllimits1.test b1aae27cc98eceb845e7f7adf918561256e31298 F test/stat.test be8d477306006ec696bc86757cfb34bec79447ce F test/stmt.test 25d64e3dbf9a3ce89558667d7f39d966fe2a71b9 @@ -1054,7 +1054,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P c8c69307f60c1d07ac666ae3797b7e3f286fd491 -R 3a7355d9d00cf8a32d1b35340d234e67 +P 680822e892f3efdb702eea3b321bc5785239dd56 +R f6325e95c7057e7590cfe9f4379a5b36 U drh -Z 8dc283cf0c63ecf74b33a1f0701aa7d4 +Z 2d983c030ad222dceb5985301e41e8c4 diff --git a/manifest.uuid b/manifest.uuid index b8ed72b0f8..2730c8b8c1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -680822e892f3efdb702eea3b321bc5785239dd56 \ No newline at end of file +de556add10150140981a2e34b3712e96a7c262e3 \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index 13a2f70cbc..09c3b810c8 100644 --- a/src/shell.c +++ b/src/shell.c @@ -1480,18 +1480,6 @@ static void open_db(struct callback_data *p){ } #ifndef SQLITE_OMIT_LOAD_EXTENSION sqlite3_enable_load_extension(p->db, 1); -#endif -#ifdef SQLITE_ENABLE_REGEXP - { - extern int sqlite3_add_regexp_func(sqlite3*); - sqlite3_add_regexp_func(db); - } -#endif -#ifdef SQLITE_ENABLE_SPELLFIX - { - extern int sqlite3_spellfix1_register(sqlite3*); - sqlite3_spellfix1_register(db); - } #endif } } diff --git a/src/test1.c b/src/test1.c index 328d0a8e45..7c30962b75 100644 --- a/src/test1.c +++ b/src/test1.c @@ -6059,6 +6059,7 @@ static int tclLoadStaticExtensionCmd( ){ extern int sqlite3_fuzzer_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_regexp_init(sqlite3*,char**,const sqlite3_api_routines*); + extern int sqlite3_spellfix_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_wholenumber_init(sqlite3*,char**,const sqlite3_api_routines*); static const struct { const char *zExtName; @@ -6066,6 +6067,7 @@ static int tclLoadStaticExtensionCmd( } aExtension[] = { { "fuzzer", sqlite3_fuzzer_init }, { "regexp", sqlite3_regexp_init }, + { "spellfix", sqlite3_spellfix_init }, { "wholenumber", sqlite3_wholenumber_init }, }; sqlite3 *db; diff --git a/src/test8.c b/src/test8.c index 6bd4e396b8..f7b2e6813d 100644 --- a/src/test8.c +++ b/src/test8.c @@ -1370,29 +1370,6 @@ static int declare_vtab( return TCL_OK; } -#include "test_spellfix.c" - -/* -** Register the spellfix virtual table module. -*/ -static int register_spellfix_module( - ClientData clientData, - Tcl_Interp *interp, - int objc, - Tcl_Obj *CONST objv[] -){ - sqlite3 *db; - - if( objc!=2 ){ - Tcl_WrongNumArgs(interp, 1, objv, "DB"); - return TCL_ERROR; - } - if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; - - sqlite3_spellfix1_register(db); - return TCL_OK; -} - #endif /* ifndef SQLITE_OMIT_VIRTUALTABLE */ /* @@ -1406,7 +1383,6 @@ int Sqlitetest8_Init(Tcl_Interp *interp){ void *clientData; } aObjCmd[] = { { "register_echo_module", register_echo_module, 0 }, - { "register_spellfix_module", register_spellfix_module, 0 }, { "sqlite3_declare_vtab", declare_vtab, 0 }, }; int i; diff --git a/test/spellfix.test b/test/spellfix.test index 6fb32b6d37..dbe8bd60e9 100644 --- a/test/spellfix.test +++ b/test/spellfix.test @@ -16,7 +16,7 @@ set testprefix spellfix ifcapable !vtab { finish_test ; return } -register_spellfix_module db +load_static_extension db spellfix set vocab { rabbi rabbit rabbits rabble rabid rabies raccoon raccoons race raced racer From 8416fc7fc7a590451c32a5b1f89180d065d37a00 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 25 Apr 2013 16:42:55 +0000 Subject: [PATCH 08/32] Added the transitive_closure, ieee754, and amatch extensions. FossilOrigin-Name: 84018099c8715b982cd24ce9221f93c7379e8c08 --- Makefile.in | 3 + Makefile.msc | 3 + ext/misc/amatch.c | 1477 +++++++++++++++++++++++++++++++++++++++++++ ext/misc/closure.c | 942 +++++++++++++++++++++++++++ ext/misc/ieee754.c | 131 ++++ ext/misc/regexp.c | 11 +- main.mk | 3 + manifest | 26 +- manifest.uuid | 2 +- src/test1.c | 6 + test/closure01.test | 224 +++++++ 11 files changed, 2815 insertions(+), 13 deletions(-) create mode 100644 ext/misc/amatch.c create mode 100644 ext/misc/closure.c create mode 100644 ext/misc/ieee754.c create mode 100644 test/closure01.test diff --git a/Makefile.in b/Makefile.in index 74bf58e31f..c713daaebc 100644 --- a/Makefile.in +++ b/Makefile.in @@ -388,7 +388,10 @@ TESTSRC = \ # Statically linked extensions # TESTSRC += \ + $(TOP)/ext/misc/amatch.c \ + $(TOP)/ext/misc/closure.c \ $(TOP)/ext/misc/fuzzer.c \ + $(TOP)/ext/misc/ieee754.c \ $(TOP)/ext/misc/regexp.c \ $(TOP)/ext/misc/spellfix.c \ $(TOP)/ext/misc/wholenumber.c diff --git a/Makefile.msc b/Makefile.msc index ea8fc10a2e..383368c44c 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -708,7 +708,10 @@ TESTSRC = \ # Statically linked extensions # TESTEXT = \ + $(TOP)\ext\misc\amatch.c \ + $(TOP)\ext\misc\closure.c \ $(TOP)\ext\misc\fuzzer.c \ + $(TOP)\ext\misc\ieee754.c \ $(TOP)\ext\misc\regexp.c \ $(TOP)\ext\misc\spellfix.c \ $(TOP)\ext\misc\wholenumber.c diff --git a/ext/misc/amatch.c b/ext/misc/amatch.c new file mode 100644 index 0000000000..f91c84f06e --- /dev/null +++ b/ext/misc/amatch.c @@ -0,0 +1,1477 @@ +/* +** 2013-03-14 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains code for a demonstration virtual table that finds +** "approximate matches" - strings from a finite set that are nearly the +** same as a single input string. The virtual table is called "amatch". +** +** A amatch virtual table is created like this: +** +** CREATE VIRTUAL TABLE f USING approximate_match( +** vocabulary_table=, -- V +** vocabulary_word=, -- W +** vocabulary_language=, -- L +** edit_distances= +** ); +** +** When it is created, the new amatch table must be supplied with the +** the name of a table V and columns V.W and V.L such that +** +** SELECT W FROM V WHERE L=$language +** +** returns the allowed vocabulary for the match. If the "vocabulary_language" +** or L columnname is left unspecified or is an empty string, then no +** filtering of the vocabulary by language is performed. +** +** For efficiency, it is essential that the vocabulary table be indexed: +** +** CREATE vocab_index ON V(W) +** +** A separate edit-cost-table provides scoring information that defines +** what it means for one string to be "close" to another. +** +** The edit-cost-table must contain exactly four columns (more precisely, +** the statement "SELECT * FROM " must return records +** that consist of four columns). It does not matter what the columns are +** named. +** +** Each row in the edit-cost-table represents a single character +** transformation going from user input to the vocabulary. The leftmost +** column of the row (column 0) contains an integer identifier of the +** language to which the transformation rule belongs (see "MULTIPLE LANGUAGES" +** below). The second column of the row (column 1) contains the input +** character or characters - the characters of user input. The third +** column contains characters as they appear in the vocabulary table. +** And the fourth column contains the integer cost of making the +** transformation. For example: +** +** CREATE TABLE f_data(iLang, cFrom, cTo, Cost); +** INSERT INTO f_data(iLang, cFrom, cTo, Cost) VALUES(0, '', 'a', 100); +** INSERT INTO f_data(iLang, cFrom, cTo, Cost) VALUES(0, 'b', '', 87); +** INSERT INTO f_data(iLang, cFrom, cTo, Cost) VALUES(0, 'o', 'oe', 38); +** INSERT INTO f_data(iLang, cFrom, cTo, Cost) VALUES(0, 'oe', 'o', 40); +** +** The first row inserted into the edit-cost-table by the SQL script +** above indicates that the cost of having an extra 'a' in the vocabulary +** table that is missing in the user input 100. (All costs are integers. +** Overall cost must not exceed 16777216.) The second INSERT statement +** creates a rule saying that the cost of having a single letter 'b' in +** user input which is missing in the vocabulary table is 87. The third +** INSERT statement mean that the cost of matching an 'o' in user input +** against an 'oe' in the vocabulary table is 38. And so forth. +** +** The following rules are special: +** +** INSERT INTO f_data(iLang, cFrom, cTo, Cost) VALUES(0, '?', '', 97); +** INSERT INTO f_data(iLang, cFrom, cTo, Cost) VALUES(0, '', '?', 98); +** INSERT INTO f_data(iLang, cFrom, cTo, Cost) VALUES(0, '?', '?', 99); +** +** The '?' to '' rule is the cost of having any single character in the input +** that is not found in the vocabular. The '' to '?' rule is the cost of +** having a character in the vocabulary table that is missing from input. +** And the '?' to '?' rule is the cost of doing an arbitrary character +** substitution. These three generic rules apply across all languages. +** In other words, the iLang field is ignored for the generic substitution +** rules. If more than one cost is given for a generic substitution rule, +** then the lowest cost is used. +** +** Once it has been created, the amatch virtual table can be queried +** as follows: +** +** SELECT word, distance FROM f +** WHERE word MATCH 'abcdefg' +** AND distance<200; +** +** This query outputs the strings contained in the T(F) field that +** are close to "abcdefg" and in order of increasing distance. No string +** is output more than once. If there are multiple ways to transform the +** target string ("abcdefg") into a string in the vocabulary table then +** the lowest cost transform is the one that is returned. In this example, +** the search is limited to strings with a total distance of less than 200. +** +** For efficiency, it is important to put tight bounds on the distance. +** The time and memory space needed to perform this query is exponential +** in the maximum distance. A good rule of thumb is to limit the distance +** to no more than 1.5 or 2 times the maximum cost of any rule in the +** edit-cost-table. +** +** The amatch is a read-only table. Any attempt to DELETE, INSERT, or +** UPDATE on a amatch table will throw an error. +** +** It is important to put some kind of a limit on the amatch output. This +** can be either in the form of a LIMIT clause at the end of the query, +** or better, a "distance +#include +#include +#include +#include + +/* +** Forward declaration of objects used by this implementation +*/ +typedef struct amatch_vtab amatch_vtab; +typedef struct amatch_cursor amatch_cursor; +typedef struct amatch_rule amatch_rule; +typedef struct amatch_word amatch_word; +typedef struct amatch_avl amatch_avl; + + +/***************************************************************************** +** AVL Tree implementation +*/ +/* +** Objects that want to be members of the AVL tree should embedded an +** instance of this structure. +*/ +struct amatch_avl { + amatch_word *pWord; /* Points to the object being stored in the tree */ + char *zKey; /* Key. zero-terminated string. Must be unique */ + amatch_avl *pBefore; /* Other elements less than zKey */ + amatch_avl *pAfter; /* Other elements greater than zKey */ + amatch_avl *pUp; /* Parent element */ + short int height; /* Height of this node. Leaf==1 */ + short int imbalance; /* Height difference between pBefore and pAfter */ +}; + +/* Recompute the amatch_avl.height and amatch_avl.imbalance fields for p. +** Assume that the children of p have correct heights. +*/ +static void amatchAvlRecomputeHeight(amatch_avl *p){ + short int hBefore = p->pBefore ? p->pBefore->height : 0; + short int hAfter = p->pAfter ? p->pAfter->height : 0; + p->imbalance = hBefore - hAfter; /* -: pAfter higher. +: pBefore higher */ + p->height = (hBefore>hAfter ? hBefore : hAfter)+1; +} + +/* +** P B +** / \ / \ +** B Z ==> X P +** / \ / \ +** X Y Y Z +** +*/ +static amatch_avl *amatchAvlRotateBefore(amatch_avl *pP){ + amatch_avl *pB = pP->pBefore; + amatch_avl *pY = pB->pAfter; + pB->pUp = pP->pUp; + pB->pAfter = pP; + pP->pUp = pB; + pP->pBefore = pY; + if( pY ) pY->pUp = pP; + amatchAvlRecomputeHeight(pP); + amatchAvlRecomputeHeight(pB); + return pB; +} + +/* +** P A +** / \ / \ +** X A ==> P Z +** / \ / \ +** Y Z X Y +** +*/ +static amatch_avl *amatchAvlRotateAfter(amatch_avl *pP){ + amatch_avl *pA = pP->pAfter; + amatch_avl *pY = pA->pBefore; + pA->pUp = pP->pUp; + pA->pBefore = pP; + pP->pUp = pA; + pP->pAfter = pY; + if( pY ) pY->pUp = pP; + amatchAvlRecomputeHeight(pP); + amatchAvlRecomputeHeight(pA); + return pA; +} + +/* +** Return a pointer to the pBefore or pAfter pointer in the parent +** of p that points to p. Or if p is the root node, return pp. +*/ +static amatch_avl **amatchAvlFromPtr(amatch_avl *p, amatch_avl **pp){ + amatch_avl *pUp = p->pUp; + if( pUp==0 ) return pp; + if( pUp->pAfter==p ) return &pUp->pAfter; + return &pUp->pBefore; +} + +/* +** Rebalance all nodes starting with p and working up to the root. +** Return the new root. +*/ +static amatch_avl *amatchAvlBalance(amatch_avl *p){ + amatch_avl *pTop = p; + amatch_avl **pp; + while( p ){ + amatchAvlRecomputeHeight(p); + if( p->imbalance>=2 ){ + amatch_avl *pB = p->pBefore; + if( pB->imbalance<0 ) p->pBefore = amatchAvlRotateAfter(pB); + pp = amatchAvlFromPtr(p,&p); + p = *pp = amatchAvlRotateBefore(p); + }else if( p->imbalance<=(-2) ){ + amatch_avl *pA = p->pAfter; + if( pA->imbalance>0 ) p->pAfter = amatchAvlRotateBefore(pA); + pp = amatchAvlFromPtr(p,&p); + p = *pp = amatchAvlRotateAfter(p); + } + pTop = p; + p = p->pUp; + } + return pTop; +} + +/* Search the tree rooted at p for an entry with zKey. Return a pointer +** to the entry or return NULL. +*/ +static amatch_avl *amatchAvlSearch(amatch_avl *p, const char *zKey){ + int c; + while( p && (c = strcmp(zKey, p->zKey))!=0 ){ + p = (c<0) ? p->pBefore : p->pAfter; + } + return p; +} + +/* Find the first node (the one with the smallest key). +*/ +static amatch_avl *amatchAvlFirst(amatch_avl *p){ + if( p ) while( p->pBefore ) p = p->pBefore; + return p; +} + +#if 0 /* NOT USED */ +/* Return the node with the next larger key after p. +*/ +static amatch_avl *amatchAvlNext(amatch_avl *p){ + amatch_avl *pPrev = 0; + while( p && p->pAfter==pPrev ){ + pPrev = p; + p = p->pUp; + } + if( p && pPrev==0 ){ + p = amatchAvlFirst(p->pAfter); + } + return p; +} +#endif + +#if 0 /* NOT USED */ +/* Verify AVL tree integrity +*/ +static int amatchAvlIntegrity(amatch_avl *pHead){ + amatch_avl *p; + if( pHead==0 ) return 1; + if( (p = pHead->pBefore)!=0 ){ + assert( p->pUp==pHead ); + assert( amatchAvlIntegrity(p) ); + assert( strcmp(p->zKey, pHead->zKey)<0 ); + while( p->pAfter ) p = p->pAfter; + assert( strcmp(p->zKey, pHead->zKey)<0 ); + } + if( (p = pHead->pAfter)!=0 ){ + assert( p->pUp==pHead ); + assert( amatchAvlIntegrity(p) ); + assert( strcmp(p->zKey, pHead->zKey)>0 ); + p = amatchAvlFirst(p); + assert( strcmp(p->zKey, pHead->zKey)>0 ); + } + return 1; +} +static int amatchAvlIntegrity2(amatch_avl *pHead){ + amatch_avl *p, *pNext; + for(p=amatchAvlFirst(pHead); p; p=pNext){ + pNext = amatchAvlNext(p); + if( pNext==0 ) break; + assert( strcmp(p->zKey, pNext->zKey)<0 ); + } + return 1; +} +#endif + +/* Insert a new node pNew. Return NULL on success. If the key is not +** unique, then do not perform the insert but instead leave pNew unchanged +** and return a pointer to an existing node with the same key. +*/ +static amatch_avl *amatchAvlInsert(amatch_avl **ppHead, amatch_avl *pNew){ + int c; + amatch_avl *p = *ppHead; + if( p==0 ){ + p = pNew; + pNew->pUp = 0; + }else{ + while( p ){ + c = strcmp(pNew->zKey, p->zKey); + if( c<0 ){ + if( p->pBefore ){ + p = p->pBefore; + }else{ + p->pBefore = pNew; + pNew->pUp = p; + break; + } + }else if( c>0 ){ + if( p->pAfter ){ + p = p->pAfter; + }else{ + p->pAfter = pNew; + pNew->pUp = p; + break; + } + }else{ + return p; + } + } + } + pNew->pBefore = 0; + pNew->pAfter = 0; + pNew->height = 1; + pNew->imbalance = 0; + *ppHead = amatchAvlBalance(p); + /* assert( amatchAvlIntegrity(*ppHead) ); */ + /* assert( amatchAvlIntegrity2(*ppHead) ); */ + return 0; +} + +/* Remove node pOld from the tree. pOld must be an element of the tree or +** the AVL tree will become corrupt. +*/ +static void amatchAvlRemove(amatch_avl **ppHead, amatch_avl *pOld){ + amatch_avl **ppParent; + amatch_avl *pBalance; + /* assert( amatchAvlSearch(*ppHead, pOld->zKey)==pOld ); */ + ppParent = amatchAvlFromPtr(pOld, ppHead); + if( pOld->pBefore==0 && pOld->pAfter==0 ){ + *ppParent = 0; + pBalance = pOld->pUp; + }else if( pOld->pBefore && pOld->pAfter ){ + amatch_avl *pX, *pY; + pX = amatchAvlFirst(pOld->pAfter); + *amatchAvlFromPtr(pX, 0) = pX->pAfter; + if( pX->pAfter ) pX->pAfter->pUp = pX->pUp; + pBalance = pX->pUp; + pX->pAfter = pOld->pAfter; + if( pX->pAfter ){ + pX->pAfter->pUp = pX; + }else{ + assert( pBalance==pOld ); + pBalance = pX; + } + pX->pBefore = pY = pOld->pBefore; + if( pY ) pY->pUp = pX; + pX->pUp = pOld->pUp; + *ppParent = pX; + }else if( pOld->pBefore==0 ){ + *ppParent = pBalance = pOld->pAfter; + pBalance->pUp = pOld->pUp; + }else if( pOld->pAfter==0 ){ + *ppParent = pBalance = pOld->pBefore; + pBalance->pUp = pOld->pUp; + } + *ppHead = amatchAvlBalance(pBalance); + pOld->pUp = 0; + pOld->pBefore = 0; + pOld->pAfter = 0; + /* assert( amatchAvlIntegrity(*ppHead) ); */ + /* assert( amatchAvlIntegrity2(*ppHead) ); */ +} +/* +** End of the AVL Tree implementation +******************************************************************************/ + + +/* +** Various types. +** +** amatch_cost is the "cost" of an edit operation. +** +** amatch_len is the length of a matching string. +** +** amatch_langid is an ruleset identifier. +*/ +typedef int amatch_cost; +typedef signed char amatch_len; +typedef int amatch_langid; + +/* +** Limits +*/ +#define AMATCH_MX_LENGTH 50 /* Maximum length of a rule string */ +#define AMATCH_MX_LANGID 2147483647 /* Maximum rule ID */ +#define AMATCH_MX_COST 1000 /* Maximum single-rule cost */ + +/* +** A match or partial match +*/ +struct amatch_word { + amatch_word *pNext; /* Next on a list of all amatch_words */ + amatch_avl sCost; /* Linkage of this node into the cost tree */ + amatch_avl sWord; /* Linkage of this node into the word tree */ + amatch_cost rCost; /* Cost of the match so far */ + int iSeq; /* Sequence number */ + char zCost[10]; /* Cost key (text rendering of rCost) */ + short int nMatch; /* Input characters matched */ + char zWord[4]; /* Text of the word. Extra space appended as needed */ +}; + +/* +** Each transformation rule is stored as an instance of this object. +** All rules are kept on a linked list sorted by rCost. +*/ +struct amatch_rule { + amatch_rule *pNext; /* Next rule in order of increasing rCost */ + char *zFrom; /* Transform from (a string from user input) */ + amatch_cost rCost; /* Cost of this transformation */ + amatch_langid iLang; /* The langauge to which this rule belongs */ + amatch_len nFrom, nTo; /* Length of the zFrom and zTo strings */ + char zTo[4]; /* Tranform to V.W value (extra space appended) */ +}; + +/* +** A amatch virtual-table object +*/ +struct amatch_vtab { + sqlite3_vtab base; /* Base class - must be first */ + char *zClassName; /* Name of this class. Default: "amatch" */ + char *zDb; /* Name of database. (ex: "main") */ + char *zSelf; /* Name of this virtual table */ + char *zCostTab; /* Name of edit-cost-table */ + char *zVocabTab; /* Name of vocabulary table */ + char *zVocabWord; /* Name of vocabulary table word column */ + char *zVocabLang; /* Name of vocabulary table language column */ + amatch_rule *pRule; /* All active rules in this amatch */ + amatch_cost rIns; /* Generic insertion cost '' -> ? */ + amatch_cost rDel; /* Generic deletion cost ? -> '' */ + amatch_cost rSub; /* Generic substitution cost ? -> ? */ + sqlite3 *db; /* The database connection */ + sqlite3_stmt *pVCheck; /* Query to check zVocabTab */ + int nCursor; /* Number of active cursors */ +}; + +/* A amatch cursor object */ +struct amatch_cursor { + sqlite3_vtab_cursor base; /* Base class - must be first */ + sqlite3_int64 iRowid; /* The rowid of the current word */ + amatch_langid iLang; /* Use this language ID */ + amatch_cost rLimit; /* Maximum cost of any term */ + int nBuf; /* Space allocated for zBuf */ + int oomErr; /* True following an OOM error */ + int nWord; /* Number of amatch_word objects */ + char *zBuf; /* Temp-use buffer space */ + char *zInput; /* Input word to match against */ + amatch_vtab *pVtab; /* The virtual table this cursor belongs to */ + amatch_word *pAllWords; /* List of all amatch_word objects */ + amatch_word *pCurrent; /* Most recent solution */ + amatch_avl *pCost; /* amatch_word objects keyed by iCost */ + amatch_avl *pWord; /* amatch_word objects keyed by zWord */ +}; + +/* +** The two input rule lists are both sorted in order of increasing +** cost. Merge them together into a single list, sorted by cost, and +** return a pointer to the head of that list. +*/ +static amatch_rule *amatchMergeRules(amatch_rule *pA, amatch_rule *pB){ + amatch_rule head; + amatch_rule *pTail; + + pTail = &head; + while( pA && pB ){ + if( pA->rCost<=pB->rCost ){ + pTail->pNext = pA; + pTail = pA; + pA = pA->pNext; + }else{ + pTail->pNext = pB; + pTail = pB; + pB = pB->pNext; + } + } + if( pA==0 ){ + pTail->pNext = pB; + }else{ + pTail->pNext = pA; + } + return head.pNext; +} + +/* +** Statement pStmt currently points to a row in the amatch data table. This +** function allocates and populates a amatch_rule structure according to +** the content of the row. +** +** If successful, *ppRule is set to point to the new object and SQLITE_OK +** is returned. Otherwise, *ppRule is zeroed, *pzErr may be set to point +** to an error message and an SQLite error code returned. +*/ +static int amatchLoadOneRule( + amatch_vtab *p, /* Fuzzer virtual table handle */ + sqlite3_stmt *pStmt, /* Base rule on statements current row */ + amatch_rule **ppRule, /* OUT: New rule object */ + char **pzErr /* OUT: Error message */ +){ + sqlite3_int64 iLang = sqlite3_column_int64(pStmt, 0); + const char *zFrom = (const char *)sqlite3_column_text(pStmt, 1); + const char *zTo = (const char *)sqlite3_column_text(pStmt, 2); + amatch_cost rCost = sqlite3_column_int(pStmt, 3); + + int rc = SQLITE_OK; /* Return code */ + int nFrom; /* Size of string zFrom, in bytes */ + int nTo; /* Size of string zTo, in bytes */ + amatch_rule *pRule = 0; /* New rule object to return */ + + if( zFrom==0 ) zFrom = ""; + if( zTo==0 ) zTo = ""; + nFrom = (int)strlen(zFrom); + nTo = (int)strlen(zTo); + + /* Silently ignore null transformations */ + if( strcmp(zFrom, zTo)==0 ){ + if( zFrom[0]=='?' && zFrom[1]==0 ){ + if( p->rSub==0 || p->rSub>rCost ) p->rSub = rCost; + } + *ppRule = 0; + return SQLITE_OK; + } + + if( rCost<=0 || rCost>AMATCH_MX_COST ){ + *pzErr = sqlite3_mprintf("%s: cost must be between 1 and %d", + p->zClassName, AMATCH_MX_COST + ); + rc = SQLITE_ERROR; + }else + if( nFrom>AMATCH_MX_LENGTH || nTo>AMATCH_MX_LENGTH ){ + *pzErr = sqlite3_mprintf("%s: maximum string length is %d", + p->zClassName, AMATCH_MX_LENGTH + ); + rc = SQLITE_ERROR; + }else + if( iLang<0 || iLang>AMATCH_MX_LANGID ){ + *pzErr = sqlite3_mprintf("%s: iLang must be between 0 and %d", + p->zClassName, AMATCH_MX_LANGID + ); + rc = SQLITE_ERROR; + }else + if( strcmp(zFrom,"")==0 && strcmp(zTo,"?")==0 ){ + if( p->rIns==0 || p->rIns>rCost ) p->rIns = rCost; + }else + if( strcmp(zFrom,"?")==0 && strcmp(zTo,"")==0 ){ + if( p->rDel==0 || p->rDel>rCost ) p->rDel = rCost; + }else + { + pRule = sqlite3_malloc( sizeof(*pRule) + nFrom + nTo ); + if( pRule==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pRule, 0, sizeof(*pRule)); + pRule->zFrom = &pRule->zTo[nTo+1]; + pRule->nFrom = nFrom; + memcpy(pRule->zFrom, zFrom, nFrom+1); + memcpy(pRule->zTo, zTo, nTo+1); + pRule->nTo = nTo; + pRule->rCost = rCost; + pRule->iLang = (int)iLang; + } + } + + *ppRule = pRule; + return rc; +} + +/* +** Free all the content in the edit-cost-table +*/ +static void amatchFreeRules(amatch_vtab *p){ + while( p->pRule ){ + amatch_rule *pRule = p->pRule; + p->pRule = pRule->pNext; + sqlite3_free(pRule); + } + p->pRule = 0; +} + +/* +** Load the content of the amatch data table into memory. +*/ +static int amatchLoadRules( + sqlite3 *db, /* Database handle */ + amatch_vtab *p, /* Virtual amatch table to configure */ + char **pzErr /* OUT: Error message */ +){ + int rc = SQLITE_OK; /* Return code */ + char *zSql; /* SELECT used to read from rules table */ + amatch_rule *pHead = 0; + + zSql = sqlite3_mprintf("SELECT * FROM %Q.%Q", p->zDb, p->zCostTab); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + int rc2; /* finalize() return code */ + sqlite3_stmt *pStmt = 0; + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( rc!=SQLITE_OK ){ + *pzErr = sqlite3_mprintf("%s: %s", p->zClassName, sqlite3_errmsg(db)); + }else if( sqlite3_column_count(pStmt)!=4 ){ + *pzErr = sqlite3_mprintf("%s: %s has %d columns, expected 4", + p->zClassName, p->zCostTab, sqlite3_column_count(pStmt) + ); + rc = SQLITE_ERROR; + }else{ + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + amatch_rule *pRule = 0; + rc = amatchLoadOneRule(p, pStmt, &pRule, pzErr); + if( pRule ){ + pRule->pNext = pHead; + pHead = pRule; + } + } + } + rc2 = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ) rc = rc2; + } + sqlite3_free(zSql); + + /* All rules are now in a singly linked list starting at pHead. This + ** block sorts them by cost and then sets amatch_vtab.pRule to point to + ** point to the head of the sorted list. + */ + if( rc==SQLITE_OK ){ + unsigned int i; + amatch_rule *pX; + amatch_rule *a[15]; + for(i=0; ipNext; + pX->pNext = 0; + for(i=0; a[i] && ipRule = amatchMergeRules(p->pRule, pX); + }else{ + /* An error has occurred. Setting p->pRule to point to the head of the + ** allocated list ensures that the list will be cleaned up in this case. + */ + assert( p->pRule==0 ); + p->pRule = pHead; + } + + return rc; +} + +/* +** This function converts an SQL quoted string into an unquoted string +** and returns a pointer to a buffer allocated using sqlite3_malloc() +** containing the result. The caller should eventually free this buffer +** using sqlite3_free. +** +** Examples: +** +** "abc" becomes abc +** 'xyz' becomes xyz +** [pqr] becomes pqr +** `mno` becomes mno +*/ +static char *amatchDequote(const char *zIn){ + int nIn; /* Size of input string, in bytes */ + char *zOut; /* Output (dequoted) string */ + + nIn = (int)strlen(zIn); + zOut = sqlite3_malloc(nIn+1); + if( zOut ){ + char q = zIn[0]; /* Quote character (if any ) */ + + if( q!='[' && q!= '\'' && q!='"' && q!='`' ){ + memcpy(zOut, zIn, nIn+1); + }else{ + int iOut = 0; /* Index of next byte to write to output */ + int iIn; /* Index of next byte to read from input */ + + if( q=='[' ) q = ']'; + for(iIn=1; iInpVCheck ){ + sqlite3_finalize(p->pVCheck); + p->pVCheck = 0; + } +} + +/* +** Deallocate an amatch_vtab object +*/ +static void amatchFree(amatch_vtab *p){ + if( p ){ + amatchFreeRules(p); + amatchVCheckClear(p); + sqlite3_free(p->zClassName); + sqlite3_free(p->zDb); + sqlite3_free(p->zCostTab); + sqlite3_free(p->zVocabTab); + sqlite3_free(p->zVocabWord); + sqlite3_free(p->zVocabLang); + memset(p, 0, sizeof(*p)); + sqlite3_free(p); + } +} + +/* +** xDisconnect/xDestroy method for the amatch module. +*/ +static int amatchDisconnect(sqlite3_vtab *pVtab){ + amatch_vtab *p = (amatch_vtab*)pVtab; + assert( p->nCursor==0 ); + amatchFree(p); + return SQLITE_OK; +} + +/* +** Check to see if the argument is of the form: +** +** KEY = VALUE +** +** If it is, return a pointer to the first character of VALUE. +** If not, return NULL. Spaces around the = are ignored. +*/ +static const char *amatchValueOfKey(const char *zKey, const char *zStr){ + int nKey = (int)strlen(zKey); + int nStr = (int)strlen(zStr); + int i; + if( nStr module name ("approximate_match") +** argv[1] -> database name +** argv[2] -> table name +** argv[3...] -> arguments +*/ +static int amatchConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + int rc = SQLITE_OK; /* Return code */ + amatch_vtab *pNew = 0; /* New virtual table */ + const char *zModule = argv[0]; + const char *zDb = argv[1]; + const char *zVal; + int i; + + (void)pAux; + *ppVtab = 0; + pNew = sqlite3_malloc( sizeof(*pNew) ); + if( pNew==0 ) return SQLITE_NOMEM; + rc = SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); + pNew->db = db; + pNew->zClassName = sqlite3_mprintf("%s", zModule); + if( pNew->zClassName==0 ) goto amatchConnectError; + pNew->zDb = sqlite3_mprintf("%s", zDb); + if( pNew->zDb==0 ) goto amatchConnectError; + pNew->zSelf = sqlite3_mprintf("%s", argv[2]); + if( pNew->zSelf==0 ) goto amatchConnectError; + for(i=3; izVocabTab); + pNew->zVocabTab = amatchDequote(zVal); + if( pNew->zVocabTab==0 ) goto amatchConnectError; + continue; + } + zVal = amatchValueOfKey("vocabulary_word", argv[i]); + if( zVal ){ + sqlite3_free(pNew->zVocabWord); + pNew->zVocabWord = amatchDequote(zVal); + if( pNew->zVocabWord==0 ) goto amatchConnectError; + continue; + } + zVal = amatchValueOfKey("vocabulary_language", argv[i]); + if( zVal ){ + sqlite3_free(pNew->zVocabLang); + pNew->zVocabLang = amatchDequote(zVal); + if( pNew->zVocabLang==0 ) goto amatchConnectError; + continue; + } + zVal = amatchValueOfKey("edit_distances", argv[i]); + if( zVal ){ + sqlite3_free(pNew->zCostTab); + pNew->zCostTab = amatchDequote(zVal); + if( pNew->zCostTab==0 ) goto amatchConnectError; + continue; + } + *pzErr = sqlite3_mprintf("unrecognized argument: [%s]\n", argv[i]); + amatchFree(pNew); + *ppVtab = 0; + return SQLITE_ERROR; + } + rc = SQLITE_OK; + if( pNew->zCostTab==0 ){ + *pzErr = sqlite3_mprintf("no edit_distances table specified"); + rc = SQLITE_ERROR; + }else{ + rc = amatchLoadRules(db, pNew, pzErr); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_declare_vtab(db, + "CREATE TABLE x(word,distance,language," + "command HIDDEN,nword HIDDEN)" + ); +#define AMATCH_COL_WORD 0 +#define AMATCH_COL_DISTANCE 1 +#define AMATCH_COL_LANGUAGE 2 +#define AMATCH_COL_COMMAND 3 +#define AMATCH_COL_NWORD 4 + } + if( rc!=SQLITE_OK ){ + amatchFree(pNew); + } + *ppVtab = &pNew->base; + return rc; + +amatchConnectError: + amatchFree(pNew); + return rc; +} + +/* +** Open a new amatch cursor. +*/ +static int amatchOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ + amatch_vtab *p = (amatch_vtab*)pVTab; + amatch_cursor *pCur; + pCur = sqlite3_malloc( sizeof(*pCur) ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + pCur->pVtab = p; + *ppCursor = &pCur->base; + p->nCursor++; + return SQLITE_OK; +} + +/* +** Free up all the memory allocated by a cursor. Set it rLimit to 0 +** to indicate that it is at EOF. +*/ +static void amatchClearCursor(amatch_cursor *pCur){ + amatch_word *pWord, *pNextWord; + for(pWord=pCur->pAllWords; pWord; pWord=pNextWord){ + pNextWord = pWord->pNext; + sqlite3_free(pWord); + } + pCur->pAllWords = 0; + sqlite3_free(pCur->zInput); + pCur->zInput = 0; + pCur->pCost = 0; + pCur->pWord = 0; + pCur->pCurrent = 0; + pCur->rLimit = 1000000; + pCur->iLang = 0; + pCur->nWord = 0; +} + +/* +** Close a amatch cursor. +*/ +static int amatchClose(sqlite3_vtab_cursor *cur){ + amatch_cursor *pCur = (amatch_cursor *)cur; + amatchClearCursor(pCur); + pCur->pVtab->nCursor--; + sqlite3_free(pCur); + return SQLITE_OK; +} + +/* +** Render a 24-bit unsigned integer as a 4-byte base-64 number. +*/ +static void amatchEncodeInt(int x, char *z){ + static const char a[] = + "0123456789" + "ABCDEFGHIJ" + "KLMNOPQRST" + "UVWXYZ^abc" + "defghijklm" + "nopqrstuvw" + "xyz~"; + z[0] = a[(x>>18)&0x3f]; + z[1] = a[(x>>12)&0x3f]; + z[2] = a[(x>>6)&0x3f]; + z[3] = a[x&0x3f]; +} + +/* +** Write the zCost[] field for a amatch_word object +*/ +static void amatchWriteCost(amatch_word *pWord){ + amatchEncodeInt(pWord->rCost, pWord->zCost); + amatchEncodeInt(pWord->iSeq, pWord->zCost+4); + pWord->zCost[8] = 0; +} + +/* +** Add a new amatch_word object to the queue. +** +** If a prior amatch_word object with the same zWord, and nMatch +** already exists, update its rCost (if the new rCost is less) but +** otherwise leave it unchanged. Do not add a duplicate. +** +** Do nothing if the cost exceeds threshold. +*/ +static void amatchAddWord( + amatch_cursor *pCur, + amatch_cost rCost, + int nMatch, + const char *zWordBase, + const char *zWordTail +){ + amatch_word *pWord; + amatch_avl *pNode; + amatch_avl *pOther; + int nBase, nTail; + char zBuf[4]; + + if( rCost>pCur->rLimit ){ + return; + } + nBase = (int)strlen(zWordBase); + nTail = (int)strlen(zWordTail); + if( nBase+nTail+3>pCur->nBuf ){ + pCur->nBuf = nBase+nTail+100; + pCur->zBuf = sqlite3_realloc(pCur->zBuf, pCur->nBuf); + if( pCur->zBuf==0 ){ + pCur->nBuf = 0; + return; + } + } + amatchEncodeInt(nMatch, zBuf); + memcpy(pCur->zBuf, zBuf+2, 2); + memcpy(pCur->zBuf+2, zWordBase, nBase); + memcpy(pCur->zBuf+2+nBase, zWordTail, nTail+1); + pNode = amatchAvlSearch(pCur->pWord, pCur->zBuf); + if( pNode ){ + pWord = pNode->pWord; + if( pWord->rCost>rCost ){ +#ifdef AMATCH_TRACE_1 + printf("UPDATE [%s][%.*s^%s] %d (\"%s\" \"%s\")\n", + pWord->zWord+2, pWord->nMatch, pCur->zInput, pCur->zInput, + pWord->rCost, pWord->zWord, pWord->zCost); +#endif + amatchAvlRemove(&pCur->pCost, &pWord->sCost); + pWord->rCost = rCost; + amatchWriteCost(pWord); +#ifdef AMATCH_TRACE_1 + printf(" ---> %d (\"%s\" \"%s\")\n", + pWord->rCost, pWord->zWord, pWord->zCost); +#endif + pOther = amatchAvlInsert(&pCur->pCost, &pWord->sCost); + assert( pOther==0 ); (void)pOther; + } + return; + } + pWord = sqlite3_malloc( sizeof(*pWord) + nBase + nTail - 1 ); + if( pWord==0 ) return; + memset(pWord, 0, sizeof(*pWord)); + pWord->rCost = rCost; + pWord->iSeq = pCur->nWord++; + amatchWriteCost(pWord); + pWord->nMatch = nMatch; + pWord->pNext = pCur->pAllWords; + pCur->pAllWords = pWord; + pWord->sCost.zKey = pWord->zCost; + pWord->sCost.pWord = pWord; + pOther = amatchAvlInsert(&pCur->pCost, &pWord->sCost); + assert( pOther==0 ); (void)pOther; + pWord->sWord.zKey = pWord->zWord; + pWord->sWord.pWord = pWord; + strcpy(pWord->zWord, pCur->zBuf); + pOther = amatchAvlInsert(&pCur->pWord, &pWord->sWord); + assert( pOther==0 ); (void)pOther; +#ifdef AMATCH_TRACE_1 + printf("INSERT [%s][%.*s^%s] %d (\"%s\" \"%s\")\n", pWord->zWord+2, + pWord->nMatch, pCur->zInput, pCur->zInput+pWord->nMatch, rCost, + pWord->zWord, pWord->zCost); +#endif +} + +/* +** Advance a cursor to its next row of output +*/ +static int amatchNext(sqlite3_vtab_cursor *cur){ + amatch_cursor *pCur = (amatch_cursor*)cur; + amatch_word *pWord = 0; + amatch_avl *pNode; + int isMatch = 0; + amatch_vtab *p = pCur->pVtab; + int nWord; + int rc; + int i; + const char *zW; + amatch_rule *pRule; + char *zBuf = 0; + char nBuf = 0; + char zNext[8]; + char zNextIn[8]; + int nNextIn; + + if( p->pVCheck==0 ){ + char *zSql; + if( p->zVocabLang && p->zVocabLang[0] ){ + zSql = sqlite3_mprintf( + "SELECT \"%s\" FROM \"%s\"", + " WHERE \"%w\">=?1 AND \"%w\"=?2" + " ORDER BY 1", + p->zVocabWord, p->zVocabTab, + p->zVocabWord, p->zVocabLang + ); + }else{ + zSql = sqlite3_mprintf( + "SELECT \"%s\" FROM \"%s\"" + " WHERE \"%w\">=?1" + " ORDER BY 1", + p->zVocabWord, p->zVocabTab, + p->zVocabWord + ); + } + rc = sqlite3_prepare_v2(p->db, zSql, -1, &p->pVCheck, 0); + sqlite3_free(zSql); + if( rc ) return rc; + } + sqlite3_bind_int(p->pVCheck, 2, pCur->iLang); + + do{ + pNode = amatchAvlFirst(pCur->pCost); + if( pNode==0 ){ + pWord = 0; + break; + } + pWord = pNode->pWord; + amatchAvlRemove(&pCur->pCost, &pWord->sCost); + +#ifdef AMATCH_TRACE_1 + printf("PROCESS [%s][%.*s^%s] %d (\"%s\" \"%s\")\n", + pWord->zWord+2, pWord->nMatch, pCur->zInput, pCur->zInput+pWord->nMatch, + pWord->rCost, pWord->zWord, pWord->zCost); +#endif + nWord = (int)strlen(pWord->zWord+2); + if( nWord+20>nBuf ){ + nBuf = nWord+100; + zBuf = sqlite3_realloc(zBuf, nBuf); + if( zBuf==0 ) return SQLITE_NOMEM; + } + strcpy(zBuf, pWord->zWord+2); + zNext[0] = 0; + zNextIn[0] = pCur->zInput[pWord->nMatch]; + if( zNextIn[0] ){ + for(i=1; i<=4 && (pCur->zInput[pWord->nMatch+i]&0xc0)==0x80; i++){ + zNextIn[i] = pCur->zInput[pWord->nMatch+i]; + } + zNextIn[i] = 0; + nNextIn = i; + }else{ + nNextIn = 0; + } + + if( zNextIn[0] && zNextIn[0]!='*' ){ + sqlite3_reset(p->pVCheck); + strcat(zBuf, zNextIn); + sqlite3_bind_text(p->pVCheck, 1, zBuf, nWord+nNextIn, SQLITE_STATIC); + rc = sqlite3_step(p->pVCheck); + if( rc==SQLITE_ROW ){ + zW = (const char*)sqlite3_column_text(p->pVCheck, 0); + if( strncmp(zBuf, zW, nWord+nNextIn)==0 ){ + amatchAddWord(pCur, pWord->rCost, pWord->nMatch+nNextIn, zBuf, ""); + } + } + zBuf[nWord] = 0; + } + + while( 1 ){ + strcpy(zBuf+nWord, zNext); + sqlite3_reset(p->pVCheck); + sqlite3_bind_text(p->pVCheck, 1, zBuf, -1, SQLITE_TRANSIENT); + rc = sqlite3_step(p->pVCheck); + if( rc!=SQLITE_ROW ) break; + zW = (const char*)sqlite3_column_text(p->pVCheck, 0); + strcpy(zBuf+nWord, zNext); + if( strncmp(zW, zBuf, nWord)!=0 ) break; + if( (zNextIn[0]=='*' && zNextIn[1]==0) + || (zNextIn[0]==0 && zW[nWord]==0) + ){ + isMatch = 1; + zNextIn[0] = 0; + nNextIn = 0; + break; + } + zNext[0] = zW[nWord]; + for(i=1; i<=4 && (zW[nWord+i]&0xc0)==0x80; i++){ + zNext[i] = zW[nWord+i]; + } + zNext[i] = 0; + zBuf[nWord] = 0; + if( p->rIns>0 ){ + amatchAddWord(pCur, pWord->rCost+p->rIns, pWord->nMatch, + zBuf, zNext); + } + if( p->rSub>0 ){ + amatchAddWord(pCur, pWord->rCost+p->rSub, pWord->nMatch+nNextIn, + zBuf, zNext); + } + if( p->rIns<0 && p->rSub<0 ) break; + zNext[i-1]++; /* FIX ME */ + } + sqlite3_reset(p->pVCheck); + + if( p->rDel>0 ){ + zBuf[nWord] = 0; + amatchAddWord(pCur, pWord->rCost+p->rDel, pWord->nMatch+nNextIn, + zBuf, ""); + } + + for(pRule=p->pRule; pRule; pRule=pRule->pNext){ + if( pRule->iLang!=pCur->iLang ) continue; + if( strncmp(pRule->zFrom, pCur->zInput+pWord->nMatch, pRule->nFrom)==0 ){ + amatchAddWord(pCur, pWord->rCost+pRule->rCost, + pWord->nMatch+pRule->nFrom, pWord->zWord+2, pRule->zTo); + } + } + }while( !isMatch ); + pCur->pCurrent = pWord; + sqlite3_free(zBuf); + return SQLITE_OK; +} + +/* +** Called to "rewind" a cursor back to the beginning so that +** it starts its output over again. Always called at least once +** prior to any amatchColumn, amatchRowid, or amatchEof call. +*/ +static int amatchFilter( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + amatch_cursor *pCur = (amatch_cursor *)pVtabCursor; + const char *zWord = "*"; + int idx; + + amatchClearCursor(pCur); + idx = 0; + if( idxNum & 1 ){ + zWord = (const char*)sqlite3_value_text(argv[0]); + idx++; + } + if( idxNum & 2 ){ + pCur->rLimit = (amatch_cost)sqlite3_value_int(argv[idx]); + idx++; + } + if( idxNum & 4 ){ + pCur->iLang = (amatch_cost)sqlite3_value_int(argv[idx]); + idx++; + } + pCur->zInput = sqlite3_mprintf("%s", zWord); + if( pCur->zInput==0 ) return SQLITE_NOMEM; + amatchAddWord(pCur, 0, 0, "", ""); + amatchNext(pVtabCursor); + + return SQLITE_OK; +} + +/* +** Only the word and distance columns have values. All other columns +** return NULL +*/ +static int amatchColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){ + amatch_cursor *pCur = (amatch_cursor*)cur; + switch( i ){ + case AMATCH_COL_WORD: { + sqlite3_result_text(ctx, pCur->pCurrent->zWord+2, -1, SQLITE_STATIC); + break; + } + case AMATCH_COL_DISTANCE: { + sqlite3_result_int(ctx, pCur->pCurrent->rCost); + break; + } + case AMATCH_COL_LANGUAGE: { + sqlite3_result_int(ctx, pCur->iLang); + break; + } + case AMATCH_COL_NWORD: { + sqlite3_result_int(ctx, pCur->nWord); + break; + } + default: { + sqlite3_result_null(ctx); + break; + } + } + return SQLITE_OK; +} + +/* +** The rowid. +*/ +static int amatchRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + amatch_cursor *pCur = (amatch_cursor*)cur; + *pRowid = pCur->iRowid; + return SQLITE_OK; +} + +/* +** EOF indicator +*/ +static int amatchEof(sqlite3_vtab_cursor *cur){ + amatch_cursor *pCur = (amatch_cursor*)cur; + return pCur->pCurrent==0; +} + +/* +** Search for terms of these forms: +** +** (A) word MATCH $str +** (B1) distance < $value +** (B2) distance <= $value +** (C) language == $language +** +** The distance< and distance<= are both treated as distance<=. +** The query plan number is a bit vector: +** +** bit 1: Term of the form (A) found +** bit 2: Term like (B1) or (B2) found +** bit 3: Term like (C) found +** +** If bit-1 is set, $str is always in filter.argv[0]. If bit-2 is set +** then $value is in filter.argv[0] if bit-1 is clear and is in +** filter.argv[1] if bit-1 is set. If bit-3 is set, then $ruleid is +** in filter.argv[0] if bit-1 and bit-2 are both zero, is in +** filter.argv[1] if exactly one of bit-1 and bit-2 are set, and is in +** filter.argv[2] if both bit-1 and bit-2 are set. +*/ +static int amatchBestIndex( + sqlite3_vtab *tab, + sqlite3_index_info *pIdxInfo +){ + int iPlan = 0; + int iDistTerm = -1; + int iLangTerm = -1; + int i; + const struct sqlite3_index_constraint *pConstraint; + + (void)tab; + pConstraint = pIdxInfo->aConstraint; + for(i=0; inConstraint; i++, pConstraint++){ + if( pConstraint->usable==0 ) continue; + if( (iPlan & 1)==0 + && pConstraint->iColumn==0 + && pConstraint->op==SQLITE_INDEX_CONSTRAINT_MATCH + ){ + iPlan |= 1; + pIdxInfo->aConstraintUsage[i].argvIndex = 1; + pIdxInfo->aConstraintUsage[i].omit = 1; + } + if( (iPlan & 2)==0 + && pConstraint->iColumn==1 + && (pConstraint->op==SQLITE_INDEX_CONSTRAINT_LT + || pConstraint->op==SQLITE_INDEX_CONSTRAINT_LE) + ){ + iPlan |= 2; + iDistTerm = i; + } + if( (iPlan & 4)==0 + && pConstraint->iColumn==2 + && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ + ){ + iPlan |= 4; + pIdxInfo->aConstraintUsage[i].omit = 1; + iLangTerm = i; + } + } + if( iPlan & 2 ){ + pIdxInfo->aConstraintUsage[iDistTerm].argvIndex = 1+((iPlan&1)!=0); + } + if( iPlan & 4 ){ + int idx = 1; + if( iPlan & 1 ) idx++; + if( iPlan & 2 ) idx++; + pIdxInfo->aConstraintUsage[iLangTerm].argvIndex = idx; + } + pIdxInfo->idxNum = iPlan; + if( pIdxInfo->nOrderBy==1 + && pIdxInfo->aOrderBy[0].iColumn==1 + && pIdxInfo->aOrderBy[0].desc==0 + ){ + pIdxInfo->orderByConsumed = 1; + } + pIdxInfo->estimatedCost = (double)10000; + + return SQLITE_OK; +} + +/* +** The xUpdate() method. +** +** This implementation disallows DELETE and UPDATE. The only thing +** allowed is INSERT into the "command" column. +*/ +static int amatchUpdate( + sqlite3_vtab *pVTab, + int argc, + sqlite3_value **argv, + sqlite_int64 *pRowid +){ + amatch_vtab *p = (amatch_vtab*)pVTab; + const unsigned char *zCmd; + (void)pRowid; + if( argc==1 ){ + pVTab->zErrMsg = sqlite3_mprintf("DELETE from %s is not allowed", + p->zSelf); + return SQLITE_ERROR; + } + if( sqlite3_value_type(argv[0])!=SQLITE_NULL ){ + pVTab->zErrMsg = sqlite3_mprintf("UPDATE of %s is not allowed", + p->zSelf); + return SQLITE_ERROR; + } + if( sqlite3_value_type(argv[2+AMATCH_COL_WORD])!=SQLITE_NULL + || sqlite3_value_type(argv[2+AMATCH_COL_DISTANCE])!=SQLITE_NULL + || sqlite3_value_type(argv[2+AMATCH_COL_LANGUAGE])!=SQLITE_NULL + ){ + pVTab->zErrMsg = sqlite3_mprintf( + "INSERT INTO %s allowed for column [command] only", p->zSelf); + return SQLITE_ERROR; + } + zCmd = sqlite3_value_text(argv[2+AMATCH_COL_COMMAND]); + if( zCmd==0 ) return SQLITE_OK; + + return SQLITE_OK; +} + +/* +** A virtual table module that implements the "approximate_match". +*/ +static sqlite3_module amatchModule = { + 0, /* iVersion */ + amatchConnect, /* xCreate */ + amatchConnect, /* xConnect */ + amatchBestIndex, /* xBestIndex */ + amatchDisconnect, /* xDisconnect */ + amatchDisconnect, /* xDestroy */ + amatchOpen, /* xOpen - open a cursor */ + amatchClose, /* xClose - close a cursor */ + amatchFilter, /* xFilter - configure scan constraints */ + amatchNext, /* xNext - advance a cursor */ + amatchEof, /* xEof - check for end of scan */ + amatchColumn, /* xColumn - read data */ + amatchRowid, /* xRowid - read data */ + amatchUpdate, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0 /* xRollbackTo */ +}; + +/* +** Register the amatch virtual table +*/ +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_amatch_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Not used */ + rc = sqlite3_create_module(db, "approximate_match", &amatchModule, 0); + return rc; +} diff --git a/ext/misc/closure.c b/ext/misc/closure.c new file mode 100644 index 0000000000..7b3d06492e --- /dev/null +++ b/ext/misc/closure.c @@ -0,0 +1,942 @@ +/* +** 2013-04-16 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains code for a virtual table that finds the transitive +** closure of a parent/child relationship in a real table. The virtual +** table is called "transitive_closure". +** +** A transitive_closure virtual table is created like this: +** +** CREATE VIRTUAL TABLE x USING transitive_closure( +** tablename=, -- T +** idcolumn=, -- X +** parentcolumn= -- P +** ); +** +** When it is created, the new transitive_closure table may be supplied +** with default values for the name of a table T and columns T.X and T.P. +** The T.X and T.P columns must contain integers. The ideal case is for +** T.X to be the INTEGER PRIMARY KEY. The T.P column should reference +** the T.X column. The row referenced by T.P is the parent of the current row. +** +** The tablename, idcolumn, and parentcolumn supplied by the CREATE VIRTUAL +** TABLE statement may be overridden in individual queries by including +** terms like tablename='newtable', idcolumn='id2', or +** parentcolumn='parent3' in the WHERE clause of the query. +** +** For efficiency, it is essential that there be an index on the P column: +** +** CREATE Tidx1 ON T(P) +** +** Suppose a specific instance of the closure table is as follows: +** +** CREATE VIRTUAL TABLE ct1 USING transitive_closure( +** tablename='group', +** idcolumn='groupId', +** parentcolumn='parentId' +** ); +** +** Such an instance of the transitive_closure virtual table would be +** appropriate for walking a tree defined using a table like this, for example: +** +** CREATE TABLE group( +** groupId INTEGER PRIMARY KEY, +** parentId INTEGER REFERENCES group +** ); +** CREATE INDEX group_idx1 ON group(parentId); +** +** The group table above would presumably have other application-specific +** fields. The key point here is that rows of the group table form a +** tree. The purpose of the ct1 virtual table is to easily extract +** branches of that tree. +** +** Once it has been created, the ct1 virtual table can be queried +** as follows: +** +** SELECT * FROM element +** WHERE element.groupId IN (SELECT id FROM ct1 WHERE root=?1); +** +** The above query will return all elements that are part of group ?1 +** or children of group ?1 or grand-children of ?1 and so forth for all +** descendents of group ?1. The same query can be formulated as a join: +** +** SELECT element.* FROM element, ct1 +** WHERE element.groupid=ct1.id +** AND ct1.root=?1; +** +** The depth of the transitive_closure (the number of generations of +** parent/child relations to follow) can be limited by setting "depth" +** column in the WHERE clause. So, for example, the following query +** finds only children and grandchildren but no further descendents: +** +** SELECT element.* FROM element, ct1 +** WHERE element.groupid=ct1.id +** AND ct1.root=?1 +** AND ct1.depth<=2; +** +** The "ct1.depth<=2" term could be a strict equality "ct1.depth=2" in +** order to find only the grandchildren of ?1, not ?1 itself or the +** children of ?1. +** +** The root=?1 term must be supplied in WHERE clause or else the query +** of the ct1 virtual table will return an empty set. The tablename, +** idcolumn, and parentcolumn attributes can be overridden in the WHERE +** clause if desired. So, for example, the ct1 table could be repurposed +** to find ancestors rather than descendents by inverting the roles of +** the idcolumn and parentcolumn: +** +** SELECT element.* FROM element, ct1 +** WHERE element.groupid=ct1.id +** AND ct1.root=?1 +** AND ct1.idcolumn='parentId' +** AND ct1.parentcolumn='groupId'; +** +** Multiple calls to ct1 could be combined. For example, the following +** query finds all elements that "cousins" of groupId ?1. That is to say +** elements where the groupId is a grandchild of the grandparent of ?1. +** (This definition of "cousins" also includes siblings and self.) +** +** SELECT element.* FROM element, ct1 +** WHERE element.groupId=ct1.id +** AND ct1.depth=2 +** AND ct1.root IN (SELECT id FROM ct1 +** WHERE root=?1 +** AND depth=2 +** AND idcolumn='parentId' +** AND parentcolumn='groupId'); +** +** In our example, the group.groupId column is unique and thus the +** subquery will return exactly one row. For that reason, the IN +** operator could be replaced by "=" to get the same result. But +** in the general case where the idcolumn is not unique, an IN operator +** would be required for this kind of query. +** +** Note that because the tablename, idcolumn, and parentcolumn can +** all be specified in the query, it is possible for an application +** to define a single transitive_closure virtual table for use on lots +** of different hierarchy tables. One might say: +** +** CREATE VIRTUAL TABLE temp.closure USING transitive_closure; +** +** As each database connection is being opened. Then the application +** would always have a "closure" virtual table handy to use for querying. +** +** SELECT element.* FROM element, closure +** WHERE element.groupid=ct1.id +** AND closure.root=?1 +** AND closure.tablename='group' +** AND closure.idname='groupId' +** AND closure.parentname='parentId'; +** +** See the documentation at http://www.sqlite.org/loadext.html for information +** on how to compile and use loadable extensions such as this one. +*/ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include +#include +#include +#include +#include + +/* +** Forward declaration of objects used by this implementation +*/ +typedef struct closure_vtab closure_vtab; +typedef struct closure_cursor closure_cursor; +typedef struct closure_queue closure_queue; +typedef struct closure_avl closure_avl; + +/***************************************************************************** +** AVL Tree implementation +*/ +/* +** Objects that want to be members of the AVL tree should embedded an +** instance of this structure. +*/ +struct closure_avl { + sqlite3_int64 id; /* Id of this entry in the table */ + int iGeneration; /* Which generation is this entry part of */ + closure_avl *pList; /* A linked list of nodes */ + closure_avl *pBefore; /* Other elements less than id */ + closure_avl *pAfter; /* Other elements greater than id */ + closure_avl *pUp; /* Parent element */ + short int height; /* Height of this node. Leaf==1 */ + short int imbalance; /* Height difference between pBefore and pAfter */ +}; + +/* Recompute the closure_avl.height and closure_avl.imbalance fields for p. +** Assume that the children of p have correct heights. +*/ +static void closureAvlRecomputeHeight(closure_avl *p){ + short int hBefore = p->pBefore ? p->pBefore->height : 0; + short int hAfter = p->pAfter ? p->pAfter->height : 0; + p->imbalance = hBefore - hAfter; /* -: pAfter higher. +: pBefore higher */ + p->height = (hBefore>hAfter ? hBefore : hAfter)+1; +} + +/* +** P B +** / \ / \ +** B Z ==> X P +** / \ / \ +** X Y Y Z +** +*/ +static closure_avl *closureAvlRotateBefore(closure_avl *pP){ + closure_avl *pB = pP->pBefore; + closure_avl *pY = pB->pAfter; + pB->pUp = pP->pUp; + pB->pAfter = pP; + pP->pUp = pB; + pP->pBefore = pY; + if( pY ) pY->pUp = pP; + closureAvlRecomputeHeight(pP); + closureAvlRecomputeHeight(pB); + return pB; +} + +/* +** P A +** / \ / \ +** X A ==> P Z +** / \ / \ +** Y Z X Y +** +*/ +static closure_avl *closureAvlRotateAfter(closure_avl *pP){ + closure_avl *pA = pP->pAfter; + closure_avl *pY = pA->pBefore; + pA->pUp = pP->pUp; + pA->pBefore = pP; + pP->pUp = pA; + pP->pAfter = pY; + if( pY ) pY->pUp = pP; + closureAvlRecomputeHeight(pP); + closureAvlRecomputeHeight(pA); + return pA; +} + +/* +** Return a pointer to the pBefore or pAfter pointer in the parent +** of p that points to p. Or if p is the root node, return pp. +*/ +static closure_avl **closureAvlFromPtr(closure_avl *p, closure_avl **pp){ + closure_avl *pUp = p->pUp; + if( pUp==0 ) return pp; + if( pUp->pAfter==p ) return &pUp->pAfter; + return &pUp->pBefore; +} + +/* +** Rebalance all nodes starting with p and working up to the root. +** Return the new root. +*/ +static closure_avl *closureAvlBalance(closure_avl *p){ + closure_avl *pTop = p; + closure_avl **pp; + while( p ){ + closureAvlRecomputeHeight(p); + if( p->imbalance>=2 ){ + closure_avl *pB = p->pBefore; + if( pB->imbalance<0 ) p->pBefore = closureAvlRotateAfter(pB); + pp = closureAvlFromPtr(p,&p); + p = *pp = closureAvlRotateBefore(p); + }else if( p->imbalance<=(-2) ){ + closure_avl *pA = p->pAfter; + if( pA->imbalance>0 ) p->pAfter = closureAvlRotateBefore(pA); + pp = closureAvlFromPtr(p,&p); + p = *pp = closureAvlRotateAfter(p); + } + pTop = p; + p = p->pUp; + } + return pTop; +} + +/* Search the tree rooted at p for an entry with id. Return a pointer +** to the entry or return NULL. +*/ +static closure_avl *closureAvlSearch(closure_avl *p, sqlite3_int64 id){ + while( p && id!=p->id ){ + p = (idid) ? p->pBefore : p->pAfter; + } + return p; +} + +/* Find the first node (the one with the smallest key). +*/ +static closure_avl *closureAvlFirst(closure_avl *p){ + if( p ) while( p->pBefore ) p = p->pBefore; + return p; +} + +/* Return the node with the next larger key after p. +*/ +closure_avl *closureAvlNext(closure_avl *p){ + closure_avl *pPrev = 0; + while( p && p->pAfter==pPrev ){ + pPrev = p; + p = p->pUp; + } + if( p && pPrev==0 ){ + p = closureAvlFirst(p->pAfter); + } + return p; +} + +/* Insert a new node pNew. Return NULL on success. If the key is not +** unique, then do not perform the insert but instead leave pNew unchanged +** and return a pointer to an existing node with the same key. +*/ +static closure_avl *closureAvlInsert( + closure_avl **ppHead, /* Head of the tree */ + closure_avl *pNew /* New node to be inserted */ +){ + closure_avl *p = *ppHead; + if( p==0 ){ + p = pNew; + pNew->pUp = 0; + }else{ + while( p ){ + if( pNew->idid ){ + if( p->pBefore ){ + p = p->pBefore; + }else{ + p->pBefore = pNew; + pNew->pUp = p; + break; + } + }else if( pNew->id>p->id ){ + if( p->pAfter ){ + p = p->pAfter; + }else{ + p->pAfter = pNew; + pNew->pUp = p; + break; + } + }else{ + return p; + } + } + } + pNew->pBefore = 0; + pNew->pAfter = 0; + pNew->height = 1; + pNew->imbalance = 0; + *ppHead = closureAvlBalance(p); + return 0; +} + +/* Walk the tree can call xDestroy on each node +*/ +static void closureAvlDestroy(closure_avl *p, void (*xDestroy)(closure_avl*)){ + if( p ){ + closureAvlDestroy(p->pBefore, xDestroy); + closureAvlDestroy(p->pAfter, xDestroy); + xDestroy(p); + } +} +/* +** End of the AVL Tree implementation +******************************************************************************/ + +/* +** A closure virtual-table object +*/ +struct closure_vtab { + sqlite3_vtab base; /* Base class - must be first */ + char *zDb; /* Name of database. (ex: "main") */ + char *zSelf; /* Name of this virtual table */ + char *zTableName; /* Name of table holding parent/child relation */ + char *zIdColumn; /* Name of ID column of zTableName */ + char *zParentColumn; /* Name of PARENT column in zTableName */ + sqlite3 *db; /* The database connection */ + int nCursor; /* Number of pending cursors */ +}; + +/* A closure cursor object */ +struct closure_cursor { + sqlite3_vtab_cursor base; /* Base class - must be first */ + closure_vtab *pVtab; /* The virtual table this cursor belongs to */ + char *zTableName; /* Name of table holding parent/child relation */ + char *zIdColumn; /* Name of ID column of zTableName */ + char *zParentColumn; /* Name of PARENT column in zTableName */ + closure_avl *pCurrent; /* Current element of output */ + closure_avl *pClosure; /* The complete closure tree */ +}; + +/* A queue of AVL nodes */ +struct closure_queue { + closure_avl *pFirst; /* Oldest node on the queue */ + closure_avl *pLast; /* Youngest node on the queue */ +}; + +/* +** Add a node to the end of the queue +*/ +static void queuePush(closure_queue *pQueue, closure_avl *pNode){ + pNode->pList = 0; + if( pQueue->pLast ){ + pQueue->pLast->pList = pNode; + }else{ + pQueue->pFirst = pNode; + } + pQueue->pLast = pNode; +} + +/* +** Extract the oldest element (the front element) from the queue. +*/ +static closure_avl *queuePull(closure_queue *pQueue){ + closure_avl *p = pQueue->pFirst; + if( p ){ + pQueue->pFirst = p->pList; + if( pQueue->pFirst==0 ) pQueue->pLast = 0; + } + return p; +} + +/* +** This function converts an SQL quoted string into an unquoted string +** and returns a pointer to a buffer allocated using sqlite3_malloc() +** containing the result. The caller should eventually free this buffer +** using sqlite3_free. +** +** Examples: +** +** "abc" becomes abc +** 'xyz' becomes xyz +** [pqr] becomes pqr +** `mno` becomes mno +*/ +static char *closureDequote(const char *zIn){ + int nIn; /* Size of input string, in bytes */ + char *zOut; /* Output (dequoted) string */ + + nIn = (int)strlen(zIn); + zOut = sqlite3_malloc(nIn+1); + if( zOut ){ + char q = zIn[0]; /* Quote character (if any ) */ + + if( q!='[' && q!= '\'' && q!='"' && q!='`' ){ + memcpy(zOut, zIn, nIn+1); + }else{ + int iOut = 0; /* Index of next byte to write to output */ + int iIn; /* Index of next byte to read from input */ + + if( q=='[' ) q = ']'; + for(iIn=1; iInzDb); + sqlite3_free(p->zSelf); + sqlite3_free(p->zTableName); + sqlite3_free(p->zIdColumn); + sqlite3_free(p->zParentColumn); + memset(p, 0, sizeof(*p)); + sqlite3_free(p); + } +} + +/* +** xDisconnect/xDestroy method for the closure module. +*/ +static int closureDisconnect(sqlite3_vtab *pVtab){ + closure_vtab *p = (closure_vtab*)pVtab; + assert( p->nCursor==0 ); + closureFree(p); + return SQLITE_OK; +} + +/* +** Check to see if the argument is of the form: +** +** KEY = VALUE +** +** If it is, return a pointer to the first character of VALUE. +** If not, return NULL. Spaces around the = are ignored. +*/ +static const char *closureValueOfKey(const char *zKey, const char *zStr){ + int nKey = (int)strlen(zKey); + int nStr = (int)strlen(zStr); + int i; + if( nStr module name ("approximate_match") +** argv[1] -> database name +** argv[2] -> table name +** argv[3...] -> arguments +*/ +static int closureConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + int rc = SQLITE_OK; /* Return code */ + closure_vtab *pNew = 0; /* New virtual table */ + const char *zDb = argv[1]; + const char *zVal; + int i; + + (void)pAux; + *ppVtab = 0; + pNew = sqlite3_malloc( sizeof(*pNew) ); + if( pNew==0 ) return SQLITE_NOMEM; + rc = SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); + pNew->db = db; + pNew->zDb = sqlite3_mprintf("%s", zDb); + if( pNew->zDb==0 ) goto closureConnectError; + pNew->zSelf = sqlite3_mprintf("%s", argv[2]); + if( pNew->zSelf==0 ) goto closureConnectError; + for(i=3; izTableName); + pNew->zTableName = closureDequote(zVal); + if( pNew->zTableName==0 ) goto closureConnectError; + continue; + } + zVal = closureValueOfKey("idcolumn", argv[i]); + if( zVal ){ + sqlite3_free(pNew->zIdColumn); + pNew->zIdColumn = closureDequote(zVal); + if( pNew->zIdColumn==0 ) goto closureConnectError; + continue; + } + zVal = closureValueOfKey("parentcolumn", argv[i]); + if( zVal ){ + sqlite3_free(pNew->zParentColumn); + pNew->zParentColumn = closureDequote(zVal); + if( pNew->zParentColumn==0 ) goto closureConnectError; + continue; + } + *pzErr = sqlite3_mprintf("unrecognized argument: [%s]\n", argv[i]); + closureFree(pNew); + *ppVtab = 0; + return SQLITE_ERROR; + } + rc = sqlite3_declare_vtab(db, + "CREATE TABLE x(id,depth,root HIDDEN,tablename HIDDEN," + "idcolumn HIDDEN,parentcolumn HIDDEN)" + ); +#define CLOSURE_COL_ID 0 +#define CLOSURE_COL_DEPTH 1 +#define CLOSURE_COL_ROOT 2 +#define CLOSURE_COL_TABLENAME 3 +#define CLOSURE_COL_IDCOLUMN 4 +#define CLOSURE_COL_PARENTCOLUMN 5 + if( rc!=SQLITE_OK ){ + closureFree(pNew); + } + *ppVtab = &pNew->base; + return rc; + +closureConnectError: + closureFree(pNew); + return rc; +} + +/* +** Open a new closure cursor. +*/ +static int closureOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ + closure_vtab *p = (closure_vtab*)pVTab; + closure_cursor *pCur; + pCur = sqlite3_malloc( sizeof(*pCur) ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + pCur->pVtab = p; + *ppCursor = &pCur->base; + p->nCursor++; + return SQLITE_OK; +} + +/* +** Free up all the memory allocated by a cursor. Set it rLimit to 0 +** to indicate that it is at EOF. +*/ +static void closureClearCursor(closure_cursor *pCur){ + closureAvlDestroy(pCur->pClosure, (void(*)(closure_avl*))sqlite3_free); + sqlite3_free(pCur->zTableName); + sqlite3_free(pCur->zIdColumn); + sqlite3_free(pCur->zParentColumn); + pCur->zTableName = 0; + pCur->zIdColumn = 0; + pCur->zParentColumn = 0; + pCur->pCurrent = 0; + pCur->pClosure = 0; +} + +/* +** Close a closure cursor. +*/ +static int closureClose(sqlite3_vtab_cursor *cur){ + closure_cursor *pCur = (closure_cursor *)cur; + closureClearCursor(pCur); + pCur->pVtab->nCursor--; + sqlite3_free(pCur); + return SQLITE_OK; +} + +/* +** Advance a cursor to its next row of output +*/ +static int closureNext(sqlite3_vtab_cursor *cur){ + closure_cursor *pCur = (closure_cursor*)cur; + pCur->pCurrent = closureAvlNext(pCur->pCurrent); + return SQLITE_OK; +} + +/* +** Allocate and insert a node +*/ +static int closureInsertNode( + closure_queue *pQueue, /* Add new node to this queue */ + closure_cursor *pCur, /* The cursor into which to add the node */ + sqlite3_int64 id, /* The node ID */ + int iGeneration /* The generation number for this node */ +){ + closure_avl *pNew = sqlite3_malloc( sizeof(*pNew) ); + if( pNew==0 ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); + pNew->id = id; + pNew->iGeneration = iGeneration; + closureAvlInsert(&pCur->pClosure, pNew); + queuePush(pQueue, pNew); + return SQLITE_OK; +} + +/* +** Called to "rewind" a cursor back to the beginning so that +** it starts its output over again. Always called at least once +** prior to any closureColumn, closureRowid, or closureEof call. +** +** This routine actually computes the closure. +** +** See the comment at the beginning of closureBestIndex() for a +** description of the meaning of idxNum. The idxStr parameter is +** not used. +*/ +static int closureFilter( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + closure_cursor *pCur = (closure_cursor *)pVtabCursor; + closure_vtab *pVtab = pCur->pVtab; + sqlite3_int64 iRoot; + int mxGen = 999999999; + char *zSql; + sqlite3_stmt *pStmt; + closure_avl *pAvl; + int rc = SQLITE_OK; + const char *zTableName = pVtab->zTableName; + const char *zIdColumn = pVtab->zIdColumn; + const char *zParentColumn = pVtab->zParentColumn; + closure_queue sQueue; + + (void)idxStr; /* Unused parameter */ + (void)argc; /* Unused parameter */ + closureClearCursor(pCur); + memset(&sQueue, 0, sizeof(sQueue)); + if( (idxNum & 1)==0 ){ + /* No root=$root in the WHERE clause. Return an empty set */ + return SQLITE_OK; + } + iRoot = sqlite3_value_int64(argv[0]); + if( (idxNum & 0x000f0)!=0 ){ + mxGen = sqlite3_value_int(argv[(idxNum>>4)&0x0f]); + if( (idxNum & 0x00002)!=0 ) mxGen--; + } + if( (idxNum & 0x00f00)!=0 ){ + zTableName = (const char*)sqlite3_value_text(argv[(idxNum>>8)&0x0f]); + pCur->zTableName = sqlite3_mprintf("%s", zTableName); + } + if( (idxNum & 0x0f000)!=0 ){ + zIdColumn = (const char*)sqlite3_value_text(argv[(idxNum>>12)&0x0f]); + pCur->zIdColumn = sqlite3_mprintf("%s", zIdColumn); + } + if( (idxNum & 0x0f0000)!=0 ){ + zParentColumn = (const char*)sqlite3_value_text(argv[(idxNum>>16)&0x0f]); + pCur->zParentColumn = sqlite3_mprintf("%s", zParentColumn); + } + + zSql = sqlite3_mprintf( + "SELECT \"%w\".\"%w\" FROM \"%w\" WHERE \"%w\".\"%w\"=?1", + zTableName, zIdColumn, zTableName, zTableName, zParentColumn); + if( zSql==0 ){ + return SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(pVtab->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + if( rc ){ + sqlite3_free(pVtab->base.zErrMsg); + pVtab->base.zErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pVtab->db)); + return rc; + } + } + if( rc==SQLITE_OK ){ + rc = closureInsertNode(&sQueue, pCur, iRoot, 0); + } + while( (pAvl = queuePull(&sQueue))!=0 ){ + if( pAvl->iGeneration>=mxGen ) continue; + sqlite3_bind_int64(pStmt, 1, pAvl->id); + while( rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){ + if( sqlite3_column_type(pStmt,0)==SQLITE_INTEGER ){ + sqlite3_int64 iNew = sqlite3_column_int64(pStmt, 0); + if( closureAvlSearch(pCur->pClosure, iNew)==0 ){ + rc = closureInsertNode(&sQueue, pCur, iNew, pAvl->iGeneration+1); + } + } + } + sqlite3_reset(pStmt); + } + sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ){ + pCur->pCurrent = closureAvlFirst(pCur->pClosure); + } + + return rc; +} + +/* +** Only the word and distance columns have values. All other columns +** return NULL +*/ +static int closureColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){ + closure_cursor *pCur = (closure_cursor*)cur; + switch( i ){ + case CLOSURE_COL_ID: { + sqlite3_result_int64(ctx, pCur->pCurrent->id); + break; + } + case CLOSURE_COL_DEPTH: { + sqlite3_result_int(ctx, pCur->pCurrent->iGeneration); + break; + } + case CLOSURE_COL_ROOT: { + sqlite3_result_null(ctx); + break; + } + case CLOSURE_COL_TABLENAME: { + sqlite3_result_text(ctx, + pCur->zTableName ? pCur->zTableName : pCur->pVtab->zTableName, + -1, SQLITE_TRANSIENT); + break; + } + case CLOSURE_COL_IDCOLUMN: { + sqlite3_result_text(ctx, + pCur->zIdColumn ? pCur->zIdColumn : pCur->pVtab->zIdColumn, + -1, SQLITE_TRANSIENT); + break; + } + case CLOSURE_COL_PARENTCOLUMN: { + sqlite3_result_text(ctx, + pCur->zParentColumn ? pCur->zParentColumn : pCur->pVtab->zParentColumn, + -1, SQLITE_TRANSIENT); + break; + } + } + return SQLITE_OK; +} + +/* +** The rowid. For the closure table, this is the same as the "id" column. +*/ +static int closureRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + closure_cursor *pCur = (closure_cursor*)cur; + *pRowid = pCur->pCurrent->id; + return SQLITE_OK; +} + +/* +** EOF indicator +*/ +static int closureEof(sqlite3_vtab_cursor *cur){ + closure_cursor *pCur = (closure_cursor*)cur; + return pCur->pCurrent==0; +} + +/* +** Search for terms of these forms: +** +** (A) root = $root +** (B1) depth < $depth +** (B2) depth <= $depth +** (B3) depth = $depth +** (C) tablename = $tablename +** (D) idcolumn = $idcolumn +** (E) parentcolumn = $parentcolumn +** +** +** +** idxNum meaning +** ---------- ------------------------------------------------------ +** 0x00000001 Term of the form (A) found +** 0x00000002 The term of bit-2 is like (B1) +** 0x000000f0 Index in filter.argv[] of $depth. 0 if not used. +** 0x00000f00 Index in filter.argv[] of $tablename. 0 if not used. +** 0x0000f000 Index in filter.argv[] of $idcolumn. 0 if not used +** 0x000f0000 Index in filter.argv[] of $parentcolumn. 0 if not used. +** +** There must be a term of type (A). If there is not, then the index type +** is 0 and the query will return an empty set. +*/ +static int closureBestIndex( + sqlite3_vtab *pTab, /* The virtual table */ + sqlite3_index_info *pIdxInfo /* Information about the query */ +){ + int iPlan = 0; + int i; + int idx = 1; + const struct sqlite3_index_constraint *pConstraint; + closure_vtab *pVtab = (closure_vtab*)pTab; + + pConstraint = pIdxInfo->aConstraint; + for(i=0; inConstraint; i++, pConstraint++){ + if( pConstraint->usable==0 ) continue; + if( (iPlan & 1)==0 + && pConstraint->iColumn==CLOSURE_COL_ROOT + && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ + ){ + iPlan |= 1; + pIdxInfo->aConstraintUsage[i].argvIndex = 1; + pIdxInfo->aConstraintUsage[i].omit = 1; + } + if( (iPlan & 0x0000f0)==0 + && pConstraint->iColumn==CLOSURE_COL_DEPTH + && (pConstraint->op==SQLITE_INDEX_CONSTRAINT_LT + || pConstraint->op==SQLITE_INDEX_CONSTRAINT_LE + || pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ) + ){ + iPlan |= idx<<4; + pIdxInfo->aConstraintUsage[i].argvIndex = ++idx; + if( pConstraint->op==SQLITE_INDEX_CONSTRAINT_LT ) iPlan |= 0x000002; + } + if( (iPlan & 0x000f00)==0 + && pConstraint->iColumn==CLOSURE_COL_TABLENAME + && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ + ){ + iPlan |= idx<<8; + pIdxInfo->aConstraintUsage[i].argvIndex = ++idx; + pIdxInfo->aConstraintUsage[i].omit = 1; + } + if( (iPlan & 0x00f000)==0 + && pConstraint->iColumn==CLOSURE_COL_IDCOLUMN + && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ + ){ + iPlan |= idx<<12; + pIdxInfo->aConstraintUsage[i].argvIndex = ++idx; + pIdxInfo->aConstraintUsage[i].omit = 1; + } + if( (iPlan & 0x0f0000)==0 + && pConstraint->iColumn==CLOSURE_COL_PARENTCOLUMN + && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ + ){ + iPlan |= idx<<16; + pIdxInfo->aConstraintUsage[i].argvIndex = ++idx; + pIdxInfo->aConstraintUsage[i].omit = 1; + } + } + if( (pVtab->zTableName==0 && (iPlan & 0x000f00)==0) + || (pVtab->zIdColumn==0 && (iPlan & 0x00f000)==0) + || (pVtab->zParentColumn==0 && (iPlan & 0x0f0000)==0) + ){ + /* All of tablename, idcolumn, and parentcolumn must be specified + ** in either the CREATE VIRTUAL TABLE or in the WHERE clause constraints + ** or else the result is an empty set. */ + iPlan = 0; + } + pIdxInfo->idxNum = iPlan; + if( pIdxInfo->nOrderBy==1 + && pIdxInfo->aOrderBy[0].iColumn==CLOSURE_COL_ID + && pIdxInfo->aOrderBy[0].desc==0 + ){ + pIdxInfo->orderByConsumed = 1; + } + pIdxInfo->estimatedCost = (double)10000; + + return SQLITE_OK; +} + +/* +** A virtual table module that implements the "approximate_match". +*/ +static sqlite3_module closureModule = { + 0, /* iVersion */ + closureConnect, /* xCreate */ + closureConnect, /* xConnect */ + closureBestIndex, /* xBestIndex */ + closureDisconnect, /* xDisconnect */ + closureDisconnect, /* xDestroy */ + closureOpen, /* xOpen - open a cursor */ + closureClose, /* xClose - close a cursor */ + closureFilter, /* xFilter - configure scan constraints */ + closureNext, /* xNext - advance a cursor */ + closureEof, /* xEof - check for end of scan */ + closureColumn, /* xColumn - read data */ + closureRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0 /* xRollbackTo */ +}; + +/* +** Register the closure virtual table +*/ +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_closure_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; + rc = sqlite3_create_module(db, "transitive_closure", &closureModule, 0); + return rc; +} diff --git a/ext/misc/ieee754.c b/ext/misc/ieee754.c new file mode 100644 index 0000000000..436b11e0c5 --- /dev/null +++ b/ext/misc/ieee754.c @@ -0,0 +1,131 @@ +/* +** 2013-04-17 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This SQLite extension implements functions for the exact display +** and input of IEEE754 Binary64 floating-point numbers. +** +** ieee754(X) +** ieee754(Y,Z) +** +** In the first form, the value X should be a floating-point number. +** The function will return a string of the form 'ieee754(Y,Z)' where +** Y and Z are integers such that X==Y*pow(w.0,Z). +** +** In the second form, Y and Z are integers which are the mantissa and +** base-2 exponent of a new floating point number. The function returns +** a floating-point value equal to Y*pow(2.0,Z). +** +** Examples: +** +** ieee754(2.0) -> 'ieee754(2,0)' +** ieee754(45.25) -> 'ieee754(181,-2)' +** ieee754(2, 0) -> 2.0 +** ieee754(181, -2) -> 45.25 +*/ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include +#include + +/* +** Implementation of the ieee754() function +*/ +static void ieee754func( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + if( argc==1 ){ + sqlite3_int64 m, a; + double r; + int e; + int isNeg; + char zResult[100]; + assert( sizeof(m)==sizeof(r) ); + if( sqlite3_value_type(argv[0])!=SQLITE_FLOAT ) return; + r = sqlite3_value_double(argv[0]); + if( r<0.0 ){ + isNeg = 1; + r = -r; + }else{ + isNeg = 0; + } + memcpy(&a,&r,sizeof(a)); + if( a==0 ){ + e = 0; + m = 0; + }else{ + e = a>>52; + m = a & ((((sqlite3_int64)1)<<52)-1); + m |= ((sqlite3_int64)1)<<52; + while( e<1075 && m>0 && (m&1)==0 ){ + m >>= 1; + e++; + } + if( isNeg ) m = -m; + } + sqlite3_snprintf(sizeof(zResult), zResult, "ieee754(%lld,%d)", + m, e-1075); + sqlite3_result_text(context, zResult, -1, SQLITE_TRANSIENT); + }else if( argc==2 ){ + sqlite3_int64 m, e, a; + double r; + int isNeg = 0; + m = sqlite3_value_int64(argv[0]); + e = sqlite3_value_int64(argv[1]); + if( m<0 ){ + isNeg = 1; + m = -m; + if( m<0 ) return; + }else if( m==0 && e>1000 && e<1000 ){ + sqlite3_result_double(context, 0.0); + return; + } + while( (m>>32)&0xffe00000 ){ + m >>= 1; + e++; + } + while( ((m>>32)&0xfff00000)==0 ){ + m <<= 1; + e--; + } + e += 1075; + if( e<0 ) e = m = 0; + if( e>0x7ff ) m = 0; + a = m & ((((sqlite3_int64)1)<<52)-1); + a |= e<<52; + if( isNeg ) a |= ((sqlite3_int64)1)<<63; + memcpy(&r, &a, sizeof(r)); + sqlite3_result_double(context, r); + } +} + + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_ieee_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Unused parameter */ + rc = sqlite3_create_function(db, "ieee754", 1, SQLITE_UTF8, 0, + ieee754func, 0, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "ieee754", 2, SQLITE_UTF8, 0, + ieee754func, 0, 0); + } + return rc; +} diff --git a/ext/misc/regexp.c b/ext/misc/regexp.c index cb39956da5..16fa7d0b96 100644 --- a/ext/misc/regexp.c +++ b/ext/misc/regexp.c @@ -12,7 +12,16 @@ ** ** The code in this file implements a compact but reasonably ** efficient regular-expression matcher for posix extended regular -** expressions against UTF8 text. The following syntax is supported: +** expressions against UTF8 text. +** +** This file is an SQLite extension. It registers a single function +** named "regexp(A,B)" where A is the regular expression and B is the +** string to be matched. By registering this function, SQLite will also +** then implement the "B regexp A" operator. Note that with the function +** the regular expression comes first, but with the operator it comes +** second. +** +** The following regular expression syntax is supported: ** ** X* zero or more occurrences of X ** X+ one or more occurrences of X diff --git a/main.mk b/main.mk index eb3da5eaee..90571b0456 100644 --- a/main.mk +++ b/main.mk @@ -270,7 +270,10 @@ TESTSRC = \ # Extensions to be statically loaded. # TESTSRC += \ + $(TOP)/ext/misc/amatch.c \ + $(TOP)/ext/misc/closure.c \ $(TOP)/ext/misc/fuzzer.c \ + $(TOP)/ext/misc/ieee754.c \ $(TOP)/ext/misc/regexp.c \ $(TOP)/ext/misc/spellfix.c \ $(TOP)/ext/misc/wholenumber.c diff --git a/manifest b/manifest index 4c3502f113..1e2aa91074 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Move\sthe\stest_spellfix.c\smodule\sto\sext/misc/spellfix.c. -D 2013-04-25T14:59:01.366 +C Added\sthe\stransitive_closure,\sieee754,\sand\samatch\sextensions. +D 2013-04-25T16:42:55.478 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f -F Makefile.in 10c635460b6c3a20741d71c3a1b65b0ebec7558b +F Makefile.in 38a45083d4df568eefd31d27d67381850f35a016 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 -F Makefile.msc 97a23e910afb8c2d23db0217c343481d59045acb +F Makefile.msc d03cde9073d5752d6ddcaa6807a8bdda277e48f2 F Makefile.vxworks db21ed42a01d5740e656b16f92cb5d8d5e5dd315 F README cd04a36fbc7ea56932a4052d7d0b7f09f27c33d6 F VERSION 05c7bd63b96f31cfdef5c766ed91307ac121f5aa @@ -83,9 +83,12 @@ F ext/fts3/unicode/mkunicode.tcl 7a9bc018e2962abb79563c5a39fe581fcbf2f675 F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43 F ext/icu/icu.c eb9ae1d79046bd7871aa97ee6da51eb770134b5a F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37 +F ext/misc/amatch.c 3369b2b544066e620d986f0085d039c77d1ef17f +F ext/misc/closure.c fec0c8537c69843e0b7631d500a14c0527962cd6 F ext/misc/fuzzer.c fb64a15af978ae73fa9075b9b1dfbe82b8defc6f -F ext/misc/regexp.c c0fdb8af86981ff9890d776cfb97fe66297cc3b2 -F ext/misc/spellfix.c 8bb699116e36cc5e68d7ddf1810b638a3090c744 w src/test_spellfix.c +F ext/misc/ieee754.c 2565ce373d842977efe0922dc50b8a41b3289556 +F ext/misc/regexp.c c25c65fe775f5d9801fb8573e36ebe73f2c0c2e0 +F ext/misc/spellfix.c 8bb699116e36cc5e68d7ddf1810b638a3090c744 F ext/misc/wholenumber.c ce362368b9381ea48cbd951ade8df867eeeab014 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 757abea591d4ff67c0ff4e8f9776aeda86b18c14 @@ -108,7 +111,7 @@ F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 -F main.mk a6183110637d782988fdf3735f7d125b19639967 +F main.mk 9535835509ac58a1902ba85ed77bcce9840d62d4 F mkdll.sh 7d09b23c05d56532e9d44a50868eb4b12ff4f74a F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f F mkextw.sh 4123480947681d9b434a5e7b1ee08135abe409ac @@ -195,7 +198,7 @@ F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c 2ecec9937e69bc17560ad886da35195daa7261b8 -F src/test1.c 1c7fa0a36703508130d7ed65325883a9e453bf72 +F src/test1.c e9562428421bf0cdbf11bf5f60ab0cdabee45885 F src/test2.c 29e7154112f7448d64204e8d31179cf497ecf425 F src/test3.c 96aed72a8e1d542fed127e3e8350ae357712fa82 F src/test4.c cea2c55110241e4674e66d476d29c914627999f5 @@ -332,6 +335,7 @@ F test/capi3d.test 17b57ca28be3e37e14c2ba8f787d292d84b724a1 F test/capi3e.test f7408dda65c92b9056199fdc180f893015f83dde F test/cast.test 4c275cbdc8202d6f9c54a3596701719868ac7dc3 F test/check.test 2eb93611139a7dfaed3be80067c7dc5ceb5fb287 +F test/closure01.test 6194a899cdbba561d0439c0d6cc7bcdf4fc413e7 F test/coalesce.test cee0dccb9fbd2d494b77234bccf9dc6c6786eb91 F test/collate1.test fd02c4d8afc71879c4bb952586389961a21fb0ce F test/collate2.test 04cebe4a033be319d6ddbb3bbc69464e01700b49 @@ -1054,7 +1058,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 680822e892f3efdb702eea3b321bc5785239dd56 -R f6325e95c7057e7590cfe9f4379a5b36 +P de556add10150140981a2e34b3712e96a7c262e3 +R 9481b1895542c76cd21821bef5dac0b1 U drh -Z 2d983c030ad222dceb5985301e41e8c4 +Z 540c4ff1b6a2b03d38328f53c71b1af7 diff --git a/manifest.uuid b/manifest.uuid index 2730c8b8c1..31b1041913 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -de556add10150140981a2e34b3712e96a7c262e3 \ No newline at end of file +84018099c8715b982cd24ce9221f93c7379e8c08 \ No newline at end of file diff --git a/src/test1.c b/src/test1.c index 7c30962b75..3cbe2bd543 100644 --- a/src/test1.c +++ b/src/test1.c @@ -6057,7 +6057,10 @@ static int tclLoadStaticExtensionCmd( int objc, Tcl_Obj *CONST objv[] ){ + extern int sqlite3_amatch_init(sqlite3*,char**,const sqlite3_api_routines*); + extern int sqlite3_closure_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_fuzzer_init(sqlite3*,char**,const sqlite3_api_routines*); + extern int sqlite3_ieee_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_regexp_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_spellfix_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_wholenumber_init(sqlite3*,char**,const sqlite3_api_routines*); @@ -6065,7 +6068,10 @@ static int tclLoadStaticExtensionCmd( const char *zExtName; int (*pInit)(sqlite3*,char**,const sqlite3_api_routines*); } aExtension[] = { + { "amatch", sqlite3_amatch_init }, + { "closure", sqlite3_closure_init }, { "fuzzer", sqlite3_fuzzer_init }, + { "ieee754", sqlite3_ieee_init }, { "regexp", sqlite3_regexp_init }, { "spellfix", sqlite3_spellfix_init }, { "wholenumber", sqlite3_wholenumber_init }, diff --git a/test/closure01.test b/test/closure01.test new file mode 100644 index 0000000000..abae85c3ac --- /dev/null +++ b/test/closure01.test @@ -0,0 +1,224 @@ +# 2013-04-25 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Test cases for transitive_closure virtual table. + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix closure01 + +load_static_extension db closure + +do_execsql_test 1.0 { + BEGIN; + CREATE TABLE t1(x INTEGER PRIMARY KEY, y INTEGER); + CREATE INDEX t1y ON t1(y); + INSERT INTO t1(x) VALUES(1),(2); + INSERT INTO t1(x) SELECT x+2 FROM t1; + INSERT INTO t1(x) SELECT x+4 FROM t1; + INSERT INTO t1(x) SELECT x+8 FROM t1; + INSERT INTO t1(x) SELECT x+16 FROM t1; + INSERT INTO t1(x) SELECT x+32 FROM t1; + INSERT INTO t1(x) SELECT x+64 FROM t1; + INSERT INTO t1(x) SELECT x+128 FROM t1; + INSERT INTO t1(x) SELECT x+256 FROM t1; + INSERT INTO t1(x) SELECT x+512 FROM t1; + INSERT INTO t1(x) SELECT x+1024 FROM t1; + INSERT INTO t1(x) SELECT x+2048 FROM t1; + INSERT INTO t1(x) SELECT x+4096 FROM t1; + INSERT INTO t1(x) SELECT x+8192 FROM t1; + INSERT INTO t1(x) SELECT x+16384 FROM t1; + INSERT INTO t1(x) SELECT x+32768 FROM t1; + INSERT INTO t1(x) SELECT x+65536 FROM t1; + UPDATE t1 SET y=x/2 WHERE x>1; + COMMIT; + CREATE VIRTUAL TABLE cx + USING transitive_closure(tablename=t1, idcolumn=x, parentcolumn=y); +} {} + +# The entire table +do_execsql_test 1.1 { + SELECT count(*), depth FROM cx WHERE root=1 GROUP BY depth ORDER BY 1; +} {/1 0 1 17 2 1 4 2 8 3 16 4 .* 65536 16/} + +# descendents of 32768 +do_execsql_test 1.2 { + SELECT * FROM cx WHERE root=32768 ORDER BY id; +} {32768 0 65536 1 65537 1 131072 2} + +# descendents of 16384 +do_execsql_test 1.3 { + SELECT * FROM cx WHERE root=16384 AND depth<=2 ORDER BY id; +} {16384 0 32768 1 32769 1 65536 2 65537 2 65538 2 65539 2} + +# children of 16384 +do_execsql_test 1.4 { + SELECT id, depth, root, tablename, idcolumn, parentcolumn FROM cx + WHERE root=16384 + AND depth=1 + ORDER BY id; +} {32768 1 {} t1 x y 32769 1 {} t1 x y} + +# great-grandparent of 16384 +do_execsql_test 1.5 { + SELECT id, depth, root, tablename, idcolumn, parentcolumn FROM cx + WHERE root=16384 + AND depth=3 + AND idcolumn='Y' + AND parentcolumn='X'; +} {2048 3 {} t1 Y X} + +# depth<5 +do_execsql_test 1.6 { + SELECT count(*), depth FROM cx WHERE root=1 AND depth<5 + GROUP BY depth ORDER BY 1; +} {1 0 2 1 4 2 8 3 16 4} + +# depth<=5 +do_execsql_test 1.7 { + SELECT count(*), depth FROM cx WHERE root=1 AND depth<=5 + GROUP BY depth ORDER BY 1; +} {1 0 2 1 4 2 8 3 16 4 32 5} + +# depth==5 +do_execsql_test 1.8 { + SELECT count(*), depth FROM cx WHERE root=1 AND depth=5 + GROUP BY depth ORDER BY 1; +} {32 5} + +# depth BETWEEN 3 AND 5 +do_execsql_test 1.9 { + SELECT count(*), depth FROM cx WHERE root=1 AND depth BETWEEN 3 AND 5 + GROUP BY depth ORDER BY 1; +} {8 3 16 4 32 5} + +# depth==5 with min() and max() +do_execsql_test 1.10 { + SELECT count(*), min(id), max(id) FROM cx WHERE root=1 AND depth=5; +} {32 32 63} + +# Create a much smaller table t2 with only 32 elements +db eval { + CREATE TABLE t2(x INTEGER PRIMARY KEY, y INTEGER); + INSERT INTO t2 SELECT x, y FROM t1 WHERE x<32; + CREATE INDEX t2y ON t2(y); + CREATE VIRTUAL TABLE c2 + USING transitive_closure(tablename=t2, idcolumn=x, parentcolumn=y); +} + +# t2 full-table +do_execsql_test 2.1 { + SELECT count(*), min(id), max(id) FROM c2 WHERE root=1; +} {31 1 31} +# t2 root=10 +do_execsql_test 2.2 { + SELECT id FROM c2 WHERE root=10; +} {10 20 21} +# t2 root=11 +do_execsql_test 2.3 { + SELECT id FROM c2 WHERE root=12; +} {12 24 25} +# t2 root IN [10,12] +do_execsql_test 2.4 { + SELECT id FROM c2 WHERE root IN (10,12) ORDER BY id; +} {10 12 20 21 24 25} +# t2 root IN [10,12] (sorted) +do_execsql_test 2.5 { + SELECT id FROM c2 WHERE root IN (10,12) ORDER BY +id; +} {10 12 20 21 24 25} + +# t2 c2up from 20 +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE c2up USING transitive_closure( + tablename = t2, + idcolumn = y, + parentcolumn = x + ); + SELECT id FROM c2up WHERE root=20; +} {1 2 5 10 20} + +# cx as c2up +do_execsql_test 3.1 { + SELECT id FROM cx + WHERE root=20 + AND tablename='t2' + AND idcolumn='y' + AND parentcolumn='x'; +} {1 2 5 10 20} + +# t2 first cousins of 20 +do_execsql_test 3.2 { + SELECT DISTINCT id FROM c2 + WHERE root IN (SELECT id FROM c2up + WHERE root=20 AND depth<=2) + ORDER BY id; +} {5 10 11 20 21 22 23} + +# t2 first cousins of 20 +do_execsql_test 3.3 { + SELECT id FROM c2 + WHERE root=(SELECT id FROM c2up + WHERE root=20 AND depth=2) + AND depth=2 + EXCEPT + SELECT id FROM c2 + WHERE root=(SELECT id FROM c2up + WHERE root=20 AND depth=1) + AND depth<=1 + ORDER BY id; +} {22 23} + +# missing tablename. +do_test 4.1 { + catchsql { + SELECT id FROM cx + WHERE root=20 + AND tablename='t3' + AND idcolumn='y' + AND parentcolumn='x'; + } +} {1 {no such table: t3}} + +# missing idcolumn +do_test 4.2 { + catchsql { + SELECT id FROM cx + WHERE root=20 + AND tablename='t2' + AND idcolumn='xyz' + AND parentcolumn='x'; + } +} {1 {no such column: t2.xyz}} + +# missing parentcolumn +do_test 4.3 { + catchsql { + SELECT id FROM cx + WHERE root=20 + AND tablename='t2' + AND idcolumn='x' + AND parentcolumn='pqr'; + } +} {1 {no such column: t2.pqr}} + +# generic closure +do_execsql_test 5.1 { + CREATE VIRTUAL TABLE temp.closure USING transitive_closure; + SELECT id FROM closure + WHERE root=1 + AND depth=3 + AND tablename='t1' + AND idcolumn='x' + AND parentcolumn='y' + ORDER BY id; +} {8 9 10 11 12 13 14 15} + +finish_test From d9555a7927ad0751e3539987ba4acbf264fa7c1e Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 25 Apr 2013 17:07:26 +0000 Subject: [PATCH 09/32] Add wiki documentation files for the spellfix1 virtual table. FossilOrigin-Name: 381564e91bbf619f99a48b0b7a94ac586cb9ee79 --- ext/misc/editdist3.wiki | 114 ++++++++++ ext/misc/spellfix1.wiki | 464 ++++++++++++++++++++++++++++++++++++++++ manifest | 20 +- manifest.uuid | 2 +- 4 files changed, 590 insertions(+), 10 deletions(-) create mode 100644 ext/misc/editdist3.wiki create mode 100644 ext/misc/spellfix1.wiki diff --git a/ext/misc/editdist3.wiki b/ext/misc/editdist3.wiki new file mode 100644 index 0000000000..494922c5e9 --- /dev/null +++ b/ext/misc/editdist3.wiki @@ -0,0 +1,114 @@ +The editdist3 algorithm + +The editdist3 algorithm is a function that computes the minimum edit distance +(a.k.a. the Levenshtein distance) between two input strings. Features of +editdist3 include: + + * It works with unicode (UTF8) text. + + * A table of insertion, deletion, and substitution costs can be + provided by the application. + + * Multi-character insertsions, deletions, and substitutions can be + enumerated in the cost table. + +

The COST table

+ +To program the costs of editdist3, create a table such as the following: + +
+CREATE TABLE editcost(
+  iLang INT,   -- The language ID
+  cFrom TEXT,  -- Convert text from this
+  cTo   TEXT,  -- Convert text into this
+  iCost INT    -- The cost of doing the conversionnn
+);
+
+ +The cost table can be named anything you want - it does not have to be called +"editcost". And the table can contain additional columns. However, it the +table must contain the four columns show above, with exactly the names shown. + +The iLang column is a non-negative integer that identifies a set of costs +appropriate for a particular language. The editdist3 function will only use +a single iLang value for any given edit-distance computation. The default +value is 0. It is recommended that applications that only need to use a +single langauge always use iLang==0 for all entries. + +The iCost column is the numeric cost of transforming cFrom into cTo. This +value should be a non-negative integer, and should probably be less than 100. +The default single-character insertion and deletion costs are 100 and the +default single-character to single-character substitution cost is 150. A +cost of 10000 or more is considered "infinite" and causes the rule to be +ignored. + +The cFrom and cTo columns show edit transformation strings. Either or both +columns may contain more than one character. Or either column (but not both) +may hold an empty string. When cFrom is empty, that is the cost of inserting +cTo. When cTo is empty, that is the cost of deleting cFrom. + +In the spellfix1 algorithm, cFrom is the text as the user entered it and +cTo is the correctly spelled text as it exists in the database. The goal +of the editdist3 algorithm is to determine how close the user-entered text is +to the dictionary text. + +There are three special-case entries in the cost table: + + + + + + +
cFromcToMeaning
'''?'The default insertion cost
'?'''The default deletion cost
'?''?'The default substitution cost
+ +If any of the special-case entries shows above are omitted, then the +value of 100 is used for insertion and deletion and 150 is used for +substitution. To disable the default insertion, deletion, and/or substitution +set their respective cost to 10000 or more. + +Other entries in the cost table specific transforms for particular characters. +The cost of specific transforms should be less than the default costs, or else +the default costs will take precedence and the specific transforms will never +be used. + +Some example, cost table entries: + +
+INSERT INTO editcost(iLang, cFrom, cTo, iCost)
+VALUES(0, 'a', 'ä', 5);
+
+ +The rule above says that the letter "a" in user input can be matched against +the letter "ä" in the dictionary with a penalty of 5. + +
+INSERT INTO editcost(iLang, cFrom, cTo, iCost)
+VALUES(0, 'ss', 'ß', 8);
+
+ +The number of characters in cFrom and cTo do not need to be the same. The +rule above says that "ss" on user input will match "ß" with a penalty of 8. + +

Experimenting with the editcost3() function

+ +The [./spellfix1.wiki | spellfix1 virtual table] +uses editdist3 if the "edit_cost_table=TABLE" option +is specified as an argument when the spellfix1 virtual table is created. +But editdist3 can also be tested directly using the built-in "editdist3()" +SQL function. The editdist3() SQL function has 3 forms: + + 1. editdist3('TABLENAME'); + 2. editdist3('string1', 'string2'); + 3. editdist3('string1', 'string2', langid); + +The first form loads the edit distance coefficients from a table called +'TABLENAME'. Any prior coefficients are discarded. So when experimenting +with weights and the weight table changes, simply rerun the single-argument +form of editdist3() to reload revised coefficients. Note that the +edit distance +weights used by the editdist3() SQL function are independent from the +weights used by the spellfix1 virtual table. + +The second and third forms return the computed edit distance between strings +'string1' and "string2'. In the second form, an language id of 0 is used. +The language id is specified in the third form. diff --git a/ext/misc/spellfix1.wiki b/ext/misc/spellfix1.wiki new file mode 100644 index 0000000000..288c55dd2e --- /dev/null +++ b/ext/misc/spellfix1.wiki @@ -0,0 +1,464 @@ +The Spellfix1 Virtual Table + +This spellfix1 virtual table is used to search +a large vocabulary for close matches. For example, spellfix1 +can be used to suggest corrections to misspelled words. Or, +it could be used with FTS4 to do full-text search using potentially +misspelled words. + +Create an instance of the spellfix1 virtual table like this: + +
+CREATE VIRTUAL TABLE demo USING spellfix1;
+
+ +The "spellfix1" term is the name of this module and must be entered as +shown. The "demo" term is the +name of the virtual table you will be creating and can be altered +to suit the needs of your application. The virtual table is initially +empty. In order for the virtual table to be useful, you will need to +populate it with your vocabulary. Suppose you +have a list of words in a table named "big_vocabulary". Then do this: + +
+INSERT INTO demo(word) SELECT word FROM big_vocabulary;
+
+ +If you intend to use this virtual table in cooperation with an FTS4 +table (for spelling correctly of search terms) then you might extract +the vocabulary using an fts3aux table: + +
+INSERT INTO demo(word) SELECT term FROM search_aux WHERE col='*';
+
+ +You can also provide the virtual table with a "rank" for each word. +The "rank" is an estimate of how common the word is. Larger numbers +mean the word is more common. If you omit the rank when populating +the table, then a rank of 1 is assumed. But if you have rank +information, you can supply it and the virtual table will show a +slight preference for selecting more commonly used terms. To +populate the rank from an fts4aux table "search_aux" do something +like this: + +
+INSERT INTO demo(word,rank)
+   SELECT term, documents FROM search_aux WHERE col='*';
+
+ +To query the virtual table, include a MATCH operator in the WHERE +clause. For example: + +
+SELECT word FROM demo WHERE word MATCH 'kennasaw';
+
+ +Using a dataset of American place names (derived from +[http://geonames.usgs.gov/domestic/download_data.htm]) the query above +returns 20 results beginning with: + +
+kennesaw
+kenosha
+kenesaw
+kenaga
+keanak
+
+ +If you append the character '*' to the end of the pattern, then +a prefix search is performed. For example: + +
+SELECT word FROM demo WHERE word MATCH 'kennes*';
+
+ +Yields 20 results beginning with: + +
+kennesaw
+kennestone
+kenneson
+kenneys
+keanes
+keenes
+
+ +

Search Refinements

+ +By default, the spellfix1 table returns no more than 20 results. +(It might return less than 20 if there were fewer good matches.) +You can change the upper bound on the number of returned rows by +adding a "top=N" term to the WHERE clause of your query, where N +is the new maximum. For example, to see the 5 best matches: + +
+SELECT word FROM demo WHERE word MATCH 'kennes*' AND top=5;
+
+ +Each entry in the spellfix1 virtual table is associated with a +a particular language, identified by the integer "langid" column. +The default langid is 0 and if no other actions are taken, the +entire vocabulary is a part of the 0 language. But if your application +needs to operate in multiple languages, then you can specify different +vocabulary items for each language by specifying the langid field +when populating the table. For example: + +
+INSERT INTO demo(word,langid) SELECT word, 0 FROM en_vocabulary;
+INSERT INTO demo(word,langid) SELECT word, 1 FROM de_vocabulary;
+INSERT INTO demo(word,langid) SELECT word, 2 FROM fr_vocabulary;
+INSERT INTO demo(word,langid) SELECT word, 3 FROM ru_vocabulary;
+INSERT INTO demo(word,langid) SELECT word, 4 FROM cn_vocabulary;
+
+ +After the virtual table has been populated with items from multiple +languages, specify the language of interest using a "langid=N" term +in the WHERE clause of the query: + +
+SELECT word FROM demo WHERE word MATCH 'hildes*' AND langid=1;
+
+ +Note that if you do not include the "langid=N" term in the WHERE clause, +the search will be against language 0 (English in the example above.) +All spellfix1 searches are against a single language id. There is no +way to search all languages at once. + + +

Virtual Table Details

+ +The virtual table actually has a unique rowid with seven columns plus five +extra hidden columns. The columns are as follows: + +
+
rowid
+A unique integer number associated with each +vocabulary item in the table. This can be used +as a foreign key on other tables in the database. + +
word
+The text of the word that matches the pattern. +Both word and pattern can contains unicode characters +and can be mixed case. + +
rank
+This is the rank of the word, as specified in the +original INSERT statement. + + +
distance
+This is an edit distance or Levensthein distance going +from the pattern to the word. + +
langid
+This is the language-id of the word. All queries are +against a single language-id, which defaults to 0. +For any given query this value is the same on all rows. + +
score
+The score is a combination of rank and distance. The +idea is that a lower score is better. The virtual table +attempts to find words with the lowest score and +by default (unless overridden by ORDER BY) returns +results in order of increasing score. + +
matchlen
+In a prefix search, the matchlen is the number of characters in +the string that match against the prefix. For a non-prefix search, +this is the same as length(word). + +
phonehash
+This column shows the phonetic hash prefix that was used to restrict +the search. For any given query, this column should be the same for +every row. This information is available for diagnostic purposes and +is not normally considered useful in real applications. + +
top
+(HIDDEN) For any query, this value is the same on all +rows. It is an integer which is the maximum number of +rows that will be output. The actually number of rows +output might be less than this number, but it will never +be greater. The default value for top is 20, but that +can be changed for each query by including a term of +the form "top=N" in the WHERE clause of the query. + +
scope
+(HIDDEN) For any query, this value is the same on all +rows. The scope is a measure of how widely the virtual +table looks for matching words. Smaller values of +scope cause a broader search. The scope is normally +choosen automatically and is capped at 4. Applications +can change the scope by including a term of the form +"scope=N" in the WHERE clause of the query. Increasing +the scope will make the query run faster, but will reduce +the possible corrections. + +
srchcnt
+(HIDDEN) For any query, this value is the same on all +rows. This value is an integer which is the number of +of words examined using the edit-distance algorithm to +find the top matches that are ultimately displayed. This +value is for diagnostic use only. + +
soundslike
+(HIDDEN) When inserting vocabulary entries, this field +can be set to an spelling that matches what the word +sounds like. See the DEALING WITH UNUSUAL AND DIFFICULT +SPELLINGS section below for details. + +
command
+(HIDDEN) The value of the "command" column is always NULL. However, +applications can insert special strings into the "command" column in order +to provoke certain behaviors in the spellfix1 virtual table. +For example, inserting the string 'reset' into the "command" column +will cause the virtual table will reread its edit distance weights +(if there are any). +
+ +

Algorithm

+ +The spellfix1 virtual table creates a single +shadow table named "%_vocab" (where the % is replaced by the name of +the virtual table; Ex: "demo_vocab" for the "demo" virtual table). +the shadow table contains the following columns: + +
+
id
+The unique id (INTEGER PRIMARY KEY) + +
rank
+The rank of word. + +
langid
+The language id for this entry. + +
word
+The original UTF8 text of the vocabulary word + +
k1
+The word transliterated into lower-case ASCII. +There is a standard table of mappings from non-ASCII +characters into ASCII. Examples: "æ" -> "ae", +"þ" -> "th", "ß" -> "ss", "á" -> "a", ... The +accessory function spellfix1_translit(X) will do +the non-ASCII to ASCII mapping. The built-in lower(X) +function will convert to lower-case. Thus: +k1 = lower(spellfix1_translit(word)). + +
k2
+This field holds a phonetic code derived from k1. Letters +that have similar sounds are mapped into the same symbol. +For example, all vowels and vowel clusters become the +single symbol "A". And the letters "p", "b", "f", and +"v" all become "B". All nasal sounds are represented +as "N". And so forth. The mapping is base on +ideas found in Soundex, Metaphone, and other +long-standing phonetic matching systems. This key can +be generated by the function spellfix1_phonehash(X). +Hence: k2 = spellfix1_phonehash(k1) +
+ +There is also a function for computing the Wagner edit distance or the +Levenshtein distance between a pattern and a word. This function +is exposed as spellfix1_editdist(X,Y). The edit distance function +returns the "cost" of converting X into Y. Some transformations +cost more than others. Changing one vowel into a different vowel, +for example is relatively cheap, as is doubling a constant, or +omitting the second character of a double-constant. Other transformations +or more expensive. The idea is that the edit distance function returns +a low cost of words that are similar and a higher cost for words +that are futher apart. In this implementation, the maximum cost +of any single-character edit (delete, insert, or substitute) is 100, +with lower costs for some edits (such as transforming vowels). + +The "score" for a comparison is the edit distance between the pattern +and the word, adjusted down by the base-2 logorithm of the word rank. +For example, a match with distance 100 but rank 1000 would have a +score of 122 (= 100 - log2(1000) + 32) where as a match with distance +100 with a rank of 1 would have a score of 131 (100 - log2(1) + 32). +(NB: The constant 32 is added to each score to keep it from going +negative in case the edit distance is zero.) In this way, frequently +used words get a slightly lower cost which tends to move them toward +the top of the list of alternative spellings. + +A straightforward implementation of a spelling corrector would be +to compare the search term against every word in the vocabulary +and select the 20 with the lowest scores. However, there will +typically be hundreds of thousands or millions of words in the +vocabulary, and so this approach is not fast enough. + +Suppose the term that is being spell-corrected is X. To limit +the search space, X is converted to a k2-like key using the +equivalent of: + +
+   key = spellfix1_phonehash(lower(spellfix1_translit(X)))
+
+ +This key is then limited to "scope" characters. The default scope +value is 4, but an alternative scope can be specified using the +"scope=N" term in the WHERE clause. After the key has been truncated, +the edit distance is run against every term in the vocabulary that +has a k2 value that begins with the abbreviated key. + +For example, suppose the input word is "Paskagula". The phonetic +key is "BACACALA" which is then truncated to 4 characters "BACA". +The edit distance is then run on the 4980 entries (out of +272,597 entries total) of the vocabulary whose k2 values begin with +BACA, yielding "Pascagoula" as the best match. + +Only terms of the vocabulary with a matching langid are searched. +Hence, the same table can contain entries from multiple languages +and only the requested language will be used. The default langid +is 0. + +

Configurable Edit Distance

+ +The built-in Wagner edit-distance function with fixed weights can be +replaced by the [./editdist3.wiki | editdist3()] edit-distance function +with application-defined weights and support for unicode, by specifying +the "edit_cost_table=TABLENAME" parameter to the spellfix1 module +when the virtual table is created. +For example: + +
+CREATE VIRTUAL TABLE demo2 USING spellfix1(edit_cost_table=APPCOST);
+
+ +In the example above, the APPCOST table would be interrogated to find +the edit distance coefficients. It is the presence of the "edit_cost_table=" +parameter to the spellfix1 module name that causes editdist3() to be used +in place of the built-in edit distance function. + +The edit distance coefficients are normally read from the APPCOST table +once and there after stored in memory. Hence, run-time changes to the +APPCOST table will not normally effect the edit distance results. +However, inserting the special string 'reset' into the "command" column of the +virtual table causes the edit distance coefficients to be reread the +APPCOST table. Hence, applications should run a SQL statement similar +to the following when changes to the APPCOST table occur: + +
+INSERT INTO demo2(command) VALUES('reset'); +
+ +The tables used for edit distance costs can be changed using a command +like the following: + +
+INSERT INTO demo2(command) VALUES('edit_cost_table=APPCOST2'); +
+ +In the example above, any prior edit distance costs would be discarded and +all future queries would use the costs found in the APPCOST2 table. If the +name of the table specified by the "edit_cost_table" command is "NULL", then +theh built-in Wagner edit-distance function will be used instead of the +editdist3() function in all future queries. + +

Dealing With Unusual And Difficult Spellings

+ +The algorithm above works quite well for most cases, but there are +exceptions. These exceptions can be dealt with by making additional +entries in the virtual table using the "soundslike" column. + +For example, many words of Greek origin begin with letters "ps" where +the "p" is silent. Ex: psalm, pseudonym, psoriasis, psyche. In +another example, many Scottish surnames can be spelled with an +initial "Mac" or "Mc". Thus, "MacKay" and "McKay" are both pronounced +the same. + +Accommodation can be made for words that are not spelled as they +sound by making additional entries into the virtual table for the +same word, but adding an alternative spelling in the "soundslike" +column. For example, the canonical entry for "psalm" would be this: + +
+  INSERT INTO demo(word) VALUES('psalm');
+
+ +To enhance the ability to correct the spelling of "salm" into +"psalm", make an addition entry like this: + +
+  INSERT INTO demo(word,soundslike) VALUES('psalm','salm');
+
+ +It is ok to make multiple entries for the same word as long as +each entry has a different soundslike value. Note that if no +soundslike value is specified, the soundslike defaults to the word +itself. + +Listed below are some cases where it might make sense to add additional +soundslike entries. The specific entries will depend on the application +and the target language. + + * Silent "p" in words beginning with "ps": psalm, psyche + + * Silent "p" in words beginning with "pn": pneumonia, pneumatic + + * Silent "p" in words beginning with "pt": pterodactyl, ptolemaic + + * Silent "d" in words beginning with "dj": djinn, Djikarta + + * Silent "k" in words beginning with "kn": knight, Knuthson + + * Silent "g" in words beginning with "gn": gnarly, gnome, gnat + + * "Mac" versus "Mc" beginning Scottish surnames + + * "Tch" sounds in Slavic words: Tchaikovsky vs. Chaykovsky + + * The letter "j" pronounced like "h" in Spanish: LaJolla + + * Words beginning with "wr" versus "r": write vs. rite + + * Miscellanous problem words such as "debt", "tsetse", + "Nguyen", "Van Nuyes". + +

Auxiliary Functions

+ +The source code module that implements the spellfix1 virtual table also +implements several SQL functions that might be useful to applications +that employ spellfix1 or for testing or diagnostic work while developing +applications that use spellfix1. The following auxiliary functions are +available: + +
+
editdist3(P,W)
editdist2(P,W,L)
editdist3(T)
+These routines provide direct access to the version of the Wagner +edit-distance function that allows for application-defined weights +on edit operations. The first two forms of this function compare +pattern P against word W and return the edit distance. In the first +function, the langid is assumed to be 0 and in the second, the +langid is given by the L parameter. The third form of this function +reloads edit distance coefficience from the table named by T. + +
spellfix1_editdist(P,W)
+This routine provides access to the built-in Wagner edit-distance +function that uses default, fixed costs. The value returned is +the edit distance needed to transform W into P. + +
spellfix1_phonehash(X)
+This routine constructs a phonetic hash of the pure ascii input word X +and returns that hash. This routine is used internally by spellfix1 in +order to transform the K1 column of the shadow table into the K2 +column. + +
spellfix1_scriptcode(X)
+Given an input string X, this routine attempts to determin the dominant +script of that input and returns the ISO-15924 numeric code for that +script. The current implementation understands the following scripts: +
    +
  • 215 - Latin +
  • 220 - Cyrillic +
  • 200 - Greek +
+Additional language codes might be added in future releases. + +
spellfix1_translit(X)
+This routine transliterates unicode text into pure ascii, returning +the pure ascii representation of the input text X. This is the function +that is used internally to transform vocabulary words into the K1 +column of the shadow table. + +
diff --git a/manifest b/manifest index 89fc7d396e..473037f91a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\sstd-ext\sbranch\sinto\strunk.\s\sThis\smerge\sadds\sseveral\snew\sextensions\nto\sthe\sext/misc\sfolder,\sincluding\stransitive_closure,\sieee754,\sand\samatch,\nand\sit\sconvers\ssome\solder\ssrc/test_*.c\sfile\sinto\sextensions\sin\sthe\sext/misc\nfolder. -D 2013-04-25T16:52:19.390 +C Add\swiki\sdocumentation\sfiles\sfor\sthe\sspellfix1\svirtual\stable. +D 2013-04-25T17:07:26.477 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 38a45083d4df568eefd31d27d67381850f35a016 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -85,11 +85,13 @@ F ext/icu/icu.c eb9ae1d79046bd7871aa97ee6da51eb770134b5a F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37 F ext/misc/amatch.c 3369b2b544066e620d986f0085d039c77d1ef17f F ext/misc/closure.c fec0c8537c69843e0b7631d500a14c0527962cd6 -F ext/misc/fuzzer.c fb64a15af978ae73fa9075b9b1dfbe82b8defc6f w src/test_fuzzer.c +F ext/misc/editdist3.wiki 06100a0c558921a563cbc40e0d0151902b1eef6d +F ext/misc/fuzzer.c fb64a15af978ae73fa9075b9b1dfbe82b8defc6f F ext/misc/ieee754.c 2565ce373d842977efe0922dc50b8a41b3289556 -F ext/misc/regexp.c c25c65fe775f5d9801fb8573e36ebe73f2c0c2e0 w src/test_regexp.c -F ext/misc/spellfix.c 8bb699116e36cc5e68d7ddf1810b638a3090c744 w src/test_spellfix.c -F ext/misc/wholenumber.c ce362368b9381ea48cbd951ade8df867eeeab014 w src/test_wholenumber.c +F ext/misc/regexp.c c25c65fe775f5d9801fb8573e36ebe73f2c0c2e0 +F ext/misc/spellfix.c 8bb699116e36cc5e68d7ddf1810b638a3090c744 +F ext/misc/spellfix1.wiki dd1830444c14cf0f54dd680cc044df2ace2e9d09 +F ext/misc/wholenumber.c ce362368b9381ea48cbd951ade8df867eeeab014 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 757abea591d4ff67c0ff4e8f9776aeda86b18c14 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e @@ -1058,7 +1060,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 39b4e6ff9316cc78ea88349091e195b8104d1e9e 84018099c8715b982cd24ce9221f93c7379e8c08 -R dffac967ebf5545ada46cf15534fed7e +P bbe607c7d17c50b667990360e2ccfab8dd22f161 +R 7f8f6b4f826c162e581af7818370c158 U drh -Z d8995ab3eecb48f05ff755314fd194c9 +Z c739a22e46c39eecc65136d2af6019ff diff --git a/manifest.uuid b/manifest.uuid index 33791ed9df..84c8f03dc9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bbe607c7d17c50b667990360e2ccfab8dd22f161 \ No newline at end of file +381564e91bbf619f99a48b0b7a94ac586cb9ee79 \ No newline at end of file From e67f418d10ba3a1823bf68713bc0f04a9880fab3 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 25 Apr 2013 17:27:08 +0000 Subject: [PATCH 10/32] Fix the tool/build-shell.sh script to remove references to files that are now loadable extensions. FossilOrigin-Name: aabeea98f53edde68f484f1794ae70789dac3889 --- manifest | 12 ++++++------ manifest.uuid | 2 +- tool/build-shell.sh | 4 ---- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 473037f91a..113cfb711f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\swiki\sdocumentation\sfiles\sfor\sthe\sspellfix1\svirtual\stable. -D 2013-04-25T17:07:26.477 +C Fix\sthe\stool/build-shell.sh\sscript\sto\sremove\sreferences\sto\sfiles\sthat\sare\nnow\sloadable\sextensions. +D 2013-04-25T17:27:08.281 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 38a45083d4df568eefd31d27d67381850f35a016 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1017,7 +1017,7 @@ F test/win32lock.test 7a6bd73a5dcdee39b5bb93e92395e1773a194361 F test/zeroblob.test caaecfb4f908f7bc086ed238668049f96774d688 F test/zerodamage.test 209d7ed441f44cc5299e4ebffbef06fd5aabfefd F tool/build-all-msvc.bat 74fb6e5cca66ebdb6c9bbafb2f8b802f08146d38 x -F tool/build-shell.sh a9c34a606e2e522ba9eeca1e07090f67dce8c912 +F tool/build-shell.sh 950f47c6174f1eea171319438b93ba67ff5bf367 F tool/checkSpacing.c 810e51703529a204fc4e1eb060e9ab663e3c06d2 F tool/diffdb.c 7524b1b5df217c20cd0431f6789851a4e0cb191b F tool/extract.c 054069d81b095fbdc189a6f5d4466e40380505e2 @@ -1060,7 +1060,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P bbe607c7d17c50b667990360e2ccfab8dd22f161 -R 7f8f6b4f826c162e581af7818370c158 +P 381564e91bbf619f99a48b0b7a94ac586cb9ee79 +R dfc5d9e4a45d231ac8d29e1b5d10bc7e U drh -Z c739a22e46c39eecc65136d2af6019ff +Z 8c5379cfdd83f28d6a14d88089c63be6 diff --git a/manifest.uuid b/manifest.uuid index 84c8f03dc9..6111651bb6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -381564e91bbf619f99a48b0b7a94ac586cb9ee79 \ No newline at end of file +aabeea98f53edde68f484f1794ae70789dac3889 \ No newline at end of file diff --git a/tool/build-shell.sh b/tool/build-shell.sh index cd2838ebd6..6a48299d73 100644 --- a/tool/build-shell.sh +++ b/tool/build-shell.sh @@ -15,12 +15,8 @@ gcc -o sqlite3 -g -Os -I. \ -DSQLITE_ENABLE_STAT3 \ -DSQLITE_ENABLE_FTS4 \ -DSQLITE_ENABLE_RTREE \ - -DSQLITE_ENABLE_REGEXP \ - -DSQLITE_ENABLE_SPELLFIX -DSQLITE_CORE=1 \ -DHAVE_READLINE \ -DHAVE_USLEEP=1 \ ../sqlite/src/shell.c \ - ../sqlite/src/test_regexp.c \ - ../sqlite/src/test_spellfix.c \ ../sqlite/src/test_vfstrace.c \ sqlite3.c -ldl -lreadline -lncurses From ea41dc44c579fe48b3130c43ef040e07ca2875b0 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 25 Apr 2013 19:31:33 +0000 Subject: [PATCH 11/32] Added the nextchar.c extension. Minor changes to the spellfix.c extension so that it can be appended to an amalgamation and compiled without duplicating symbols. FossilOrigin-Name: 56b9a417f5451631f11c5206d625f11472ee65f9 --- Makefile.in | 1 + Makefile.msc | 1 + ext/misc/nextchar.c | 265 ++++++++++++++++++++++++++++++++++++++++++++ ext/misc/spellfix.c | 20 ++-- main.mk | 1 + manifest | 23 ++-- manifest.uuid | 2 +- src/test1.c | 42 +++---- test/spellfix.test | 22 +++- 9 files changed, 336 insertions(+), 41 deletions(-) create mode 100644 ext/misc/nextchar.c diff --git a/Makefile.in b/Makefile.in index c713daaebc..068da1a212 100644 --- a/Makefile.in +++ b/Makefile.in @@ -392,6 +392,7 @@ TESTSRC += \ $(TOP)/ext/misc/closure.c \ $(TOP)/ext/misc/fuzzer.c \ $(TOP)/ext/misc/ieee754.c \ + $(TOP)/ext/misc/nextchar.c \ $(TOP)/ext/misc/regexp.c \ $(TOP)/ext/misc/spellfix.c \ $(TOP)/ext/misc/wholenumber.c diff --git a/Makefile.msc b/Makefile.msc index 383368c44c..1c86264804 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -712,6 +712,7 @@ TESTEXT = \ $(TOP)\ext\misc\closure.c \ $(TOP)\ext\misc\fuzzer.c \ $(TOP)\ext\misc\ieee754.c \ + $(TOP)\ext\misc\nextchar.c \ $(TOP)\ext\misc\regexp.c \ $(TOP)\ext\misc\spellfix.c \ $(TOP)\ext\misc\wholenumber.c diff --git a/ext/misc/nextchar.c b/ext/misc/nextchar.c new file mode 100644 index 0000000000..e063043e04 --- /dev/null +++ b/ext/misc/nextchar.c @@ -0,0 +1,265 @@ +/* +** 2013-02-28 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains code to implement the next_char(A,T,F,W) SQL function. +** +** The next_char(A,T,F,H) function finds all valid "next" characters for +** string A given the vocabulary in T.F. The T.F field should be indexed. +** If the W value exists and is a non-empty string, then it is an SQL +** expression that limits the entries in T.F that will be considered. +** +** For example, suppose an application has a dictionary like this: +** +** CREATE TABLE dictionary(word TEXT UNIQUE); +** +** Further suppose that for user keypad entry, it is desired to disable +** (gray out) keys that are not valid as the next character. If the +** the user has previously entered (say) 'cha' then to find all allowed +** next characters (and thereby determine when keys should not be grayed +** out) run the following query: +** +** SELECT next_char('cha','dictionary','word'); +*/ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include + +/* +** A structure to hold context of the next_char() computation across +** nested function calls. +*/ +typedef struct nextCharContext nextCharContext; +struct nextCharContext { + sqlite3 *db; /* Database connection */ + sqlite3_stmt *pStmt; /* Prepared statement used to query */ + const unsigned char *zPrefix; /* Prefix to scan */ + int nPrefix; /* Size of zPrefix in bytes */ + int nAlloc; /* Space allocated to aResult */ + int nUsed; /* Space used in aResult */ + unsigned int *aResult; /* Array of next characters */ + int mallocFailed; /* True if malloc fails */ + int otherError; /* True for any other failure */ +}; + +/* +** Append a result character if the character is not already in the +** result. +*/ +static void nextCharAppend(nextCharContext *p, unsigned c){ + int i; + for(i=0; inUsed; i++){ + if( p->aResult[i]==c ) return; + } + if( p->nUsed+1 > p->nAlloc ){ + unsigned int *aNew; + int n = p->nAlloc*2 + 30; + aNew = sqlite3_realloc(p->aResult, n*sizeof(unsigned int)); + if( aNew==0 ){ + p->mallocFailed = 1; + return; + }else{ + p->aResult = aNew; + p->nAlloc = n; + } + } + p->aResult[p->nUsed++] = c; +} + +/* +** Write a character into z[] as UTF8. Return the number of bytes needed +** to hold the character +*/ +static int writeUtf8(unsigned char *z, unsigned c){ + if( c<0x00080 ){ + z[0] = (unsigned char)(c&0xff); + return 1; + } + if( c<0x00800 ){ + z[0] = 0xC0 + (unsigned char)((c>>6)&0x1F); + z[1] = 0x80 + (unsigned char)(c & 0x3F); + return 2; + } + if( c<0x10000 ){ + z[0] = 0xE0 + (unsigned char)((c>>12)&0x0F); + z[1] = 0x80 + (unsigned char)((c>>6) & 0x3F); + z[2] = 0x80 + (unsigned char)(c & 0x3F); + return 3; + } + z[0] = 0xF0 + (unsigned char)((c>>18) & 0x07); + z[1] = 0x80 + (unsigned char)((c>>12) & 0x3F); + z[2] = 0x80 + (unsigned char)((c>>6) & 0x3F); + z[3] = 0x80 + (unsigned char)(c & 0x3F); + return 4; +} + +/* +** Read a UTF8 character out of z[] and write it into *pOut. Return +** the number of bytes in z[] that were used to construct the character. +*/ +static int readUtf8(const unsigned char *z, unsigned *pOut){ + static const unsigned char validBits[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x00, 0x00, + }; + unsigned c = z[0]; + if( c<0xc0 ){ + *pOut = c; + return 1; + }else{ + int n = 1; + c = validBits[c-0xc0]; + while( (z[n] & 0xc0)==0x80 ){ + c = (c<<6) + (0x3f & z[n++]); + } + if( c<0x80 || (c&0xFFFFF800)==0xD800 || (c&0xFFFFFFFE)==0xFFFE ){ + c = 0xFFFD; + } + *pOut = c; + return n; + } +} + +/* +** The nextCharContext structure has been set up. Add all "next" characters +** to the result set. +*/ +static void findNextChars(nextCharContext *p){ + unsigned cPrev = 0; + unsigned char zPrev[8]; + int n, rc; + + for(;;){ + sqlite3_bind_text(p->pStmt, 1, (char*)p->zPrefix, p->nPrefix, + SQLITE_STATIC); + n = writeUtf8(zPrev, cPrev+1); + sqlite3_bind_text(p->pStmt, 2, (char*)zPrev, n, SQLITE_STATIC); + rc = sqlite3_step(p->pStmt); + if( rc==SQLITE_DONE ){ + sqlite3_reset(p->pStmt); + return; + }else if( rc!=SQLITE_ROW ){ + p->otherError = rc; + return; + }else{ + const unsigned char *zOut = sqlite3_column_text(p->pStmt, 0); + unsigned cNext; + n = readUtf8(zOut+p->nPrefix, &cNext); + sqlite3_reset(p->pStmt); + nextCharAppend(p, cNext); + cPrev = cNext; + if( p->mallocFailed ) return; + } + } +} + + +/* +** next_character(A,T,F,W) +** +** Return a string composted of all next possible characters after +** A for elements of T.F. If W is supplied, then it is an SQL expression +** that limits the elements in T.F that are considered. +*/ +static void nextCharFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + nextCharContext c; + const unsigned char *zTable = sqlite3_value_text(argv[1]); + const unsigned char *zField = sqlite3_value_text(argv[2]); + const unsigned char *zWhere; + char *zSql; + int rc; + + memset(&c, 0, sizeof(c)); + c.db = sqlite3_context_db_handle(context); + c.zPrefix = sqlite3_value_text(argv[0]); + c.nPrefix = sqlite3_value_bytes(argv[0]); + if( zTable==0 || zField==0 || c.zPrefix==0 ) return; + if( argc<4 + || (zWhere = sqlite3_value_text(argv[3]))==0 + || zWhere[0]==0 + ){ + zSql = sqlite3_mprintf( + "SELECT \"%w\" FROM \"%w\"" + " WHERE \"%w\">=(?1 || ?2)" + " AND \"%w\"<=(?1 || char(1114111))" /* 1114111 == 0x10ffff */ + " ORDER BY 1 ASC LIMIT 1", + zField, zTable, zField, zField); + }else{ + zSql = sqlite3_mprintf( + "SELECT \"%w\" FROM \"%w\"" + " WHERE \"%w\">=(?1 || ?2)" + " AND \"%w\"<=(?1 || char(1114111))" /* 1114111 == 0x10ffff */ + " AND (%s)" + " ORDER BY 1 ASC LIMIT 1", + zField, zTable, zField, zField, zWhere); + } + if( zSql==0 ){ + sqlite3_result_error_nomem(context); + return; + } + + rc = sqlite3_prepare_v2(c.db, zSql, -1, &c.pStmt, 0); + sqlite3_free(zSql); + if( rc ){ + sqlite3_result_error(context, sqlite3_errmsg(c.db), -1); + return; + } + findNextChars(&c); + if( c.mallocFailed ){ + sqlite3_result_error_nomem(context); + }else{ + unsigned char *pRes; + pRes = sqlite3_malloc( c.nUsed*4 + 1 ); + if( pRes==0 ){ + sqlite3_result_error_nomem(context); + }else{ + int i; + int n = 0; + for(i=0; i -#include -#include -#include -#define ALWAYS(X) 1 -#define NEVER(X) 0 -typedef unsigned char u8; -typedef unsigned short u16; -#include +#ifndef SQLITE_AMALGAMATION +# include +# include +# include +# include +# define ALWAYS(X) 1 +# define NEVER(X) 0 + typedef unsigned char u8; + typedef unsigned short u16; +# include +#endif /* ** Character classes for ASCII characters: diff --git a/main.mk b/main.mk index 90571b0456..30add69e29 100644 --- a/main.mk +++ b/main.mk @@ -274,6 +274,7 @@ TESTSRC += \ $(TOP)/ext/misc/closure.c \ $(TOP)/ext/misc/fuzzer.c \ $(TOP)/ext/misc/ieee754.c \ + $(TOP)/ext/misc/nextchar.c \ $(TOP)/ext/misc/regexp.c \ $(TOP)/ext/misc/spellfix.c \ $(TOP)/ext/misc/wholenumber.c diff --git a/manifest b/manifest index 113cfb711f..a3d9885853 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Fix\sthe\stool/build-shell.sh\sscript\sto\sremove\sreferences\sto\sfiles\sthat\sare\nnow\sloadable\sextensions. -D 2013-04-25T17:27:08.281 +C Added\sthe\snextchar.c\sextension.\s\sMinor\schanges\sto\sthe\sspellfix.c\sextension\nso\sthat\sit\scan\sbe\sappended\sto\san\samalgamation\sand\scompiled\swithout\sduplicating\nsymbols. +D 2013-04-25T19:31:33.149 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f -F Makefile.in 38a45083d4df568eefd31d27d67381850f35a016 +F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 -F Makefile.msc d03cde9073d5752d6ddcaa6807a8bdda277e48f2 +F Makefile.msc 8f4ee0dab220a5276d5da61149dfd6cd5d1dd5b8 F Makefile.vxworks db21ed42a01d5740e656b16f92cb5d8d5e5dd315 F README cd04a36fbc7ea56932a4052d7d0b7f09f27c33d6 F VERSION 05c7bd63b96f31cfdef5c766ed91307ac121f5aa @@ -88,8 +88,9 @@ F ext/misc/closure.c fec0c8537c69843e0b7631d500a14c0527962cd6 F ext/misc/editdist3.wiki 06100a0c558921a563cbc40e0d0151902b1eef6d F ext/misc/fuzzer.c fb64a15af978ae73fa9075b9b1dfbe82b8defc6f F ext/misc/ieee754.c 2565ce373d842977efe0922dc50b8a41b3289556 +F ext/misc/nextchar.c 1131e2b36116ffc6fe6b2e3464bfdace27978b1e F ext/misc/regexp.c c25c65fe775f5d9801fb8573e36ebe73f2c0c2e0 -F ext/misc/spellfix.c 8bb699116e36cc5e68d7ddf1810b638a3090c744 +F ext/misc/spellfix.c e323eebb877d735bc64404c16a6d758ab17a0b7a F ext/misc/spellfix1.wiki dd1830444c14cf0f54dd680cc044df2ace2e9d09 F ext/misc/wholenumber.c ce362368b9381ea48cbd951ade8df867eeeab014 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 @@ -113,7 +114,7 @@ F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 -F main.mk 9535835509ac58a1902ba85ed77bcce9840d62d4 +F main.mk 1b25be82452366abc27cc9ab2acf3244a773d5a1 F mkdll.sh 7d09b23c05d56532e9d44a50868eb4b12ff4f74a F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f F mkextw.sh 4123480947681d9b434a5e7b1ee08135abe409ac @@ -200,7 +201,7 @@ F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c 2ecec9937e69bc17560ad886da35195daa7261b8 -F src/test1.c e9562428421bf0cdbf11bf5f60ab0cdabee45885 +F src/test1.c 2b0ec224983403312a4d1db8546e1e1c45694251 F src/test2.c 29e7154112f7448d64204e8d31179cf497ecf425 F src/test3.c 96aed72a8e1d542fed127e3e8350ae357712fa82 F src/test4.c cea2c55110241e4674e66d476d29c914627999f5 @@ -756,7 +757,7 @@ F test/speed3.test d32043614c08c53eafdc80f33191d5bd9b920523 F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa F test/speed4p.test 0e51908951677de5a969b723e03a27a1c45db38b -F test/spellfix.test a85915ab25af7fcfb0d99cb1951e6ef15e26202c +F test/spellfix.test bea537caf587df30d430c2c6a8fe9f64b8712834 F test/sqllimits1.test b1aae27cc98eceb845e7f7adf918561256e31298 F test/stat.test be8d477306006ec696bc86757cfb34bec79447ce F test/stmt.test 25d64e3dbf9a3ce89558667d7f39d966fe2a71b9 @@ -1060,7 +1061,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 381564e91bbf619f99a48b0b7a94ac586cb9ee79 -R dfc5d9e4a45d231ac8d29e1b5d10bc7e +P aabeea98f53edde68f484f1794ae70789dac3889 +R 77b136ceb7e3bfd54c75649b938a3e12 U drh -Z 8c5379cfdd83f28d6a14d88089c63be6 +Z fb4502a56cfa201ec6c890343c386675 diff --git a/manifest.uuid b/manifest.uuid index 6111651bb6..7c24934462 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -aabeea98f53edde68f484f1794ae70789dac3889 \ No newline at end of file +56b9a417f5451631f11c5206d625f11472ee65f9 \ No newline at end of file diff --git a/src/test1.c b/src/test1.c index 3cbe2bd543..34370d48e4 100644 --- a/src/test1.c +++ b/src/test1.c @@ -6047,9 +6047,9 @@ static int optimization_control( typedef struct sqlite3_api_routines sqlite3_api_routines; /* -** load_static_extension DB NAME +** load_static_extension DB NAME ... ** -** Load an extension that is statically linked. +** Load one or more statically linked extensions. */ static int tclLoadStaticExtensionCmd( void * clientData, @@ -6061,6 +6061,7 @@ static int tclLoadStaticExtensionCmd( extern int sqlite3_closure_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_fuzzer_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_ieee_init(sqlite3*,char**,const sqlite3_api_routines*); + extern int sqlite3_nextchar_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_regexp_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_spellfix_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_wholenumber_init(sqlite3*,char**,const sqlite3_api_routines*); @@ -6072,33 +6073,36 @@ static int tclLoadStaticExtensionCmd( { "closure", sqlite3_closure_init }, { "fuzzer", sqlite3_fuzzer_init }, { "ieee754", sqlite3_ieee_init }, + { "nextchar", sqlite3_nextchar_init }, { "regexp", sqlite3_regexp_init }, { "spellfix", sqlite3_spellfix_init }, { "wholenumber", sqlite3_wholenumber_init }, }; sqlite3 *db; const char *zName; - int i, rc; + int i, j, rc; char *zErrMsg = 0; - if( objc!=3 ){ - Tcl_WrongNumArgs(interp, 1, objv, "DB NAME"); + if( objc<3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB NAME ..."); return TCL_ERROR; } if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; - zName = Tcl_GetString(objv[2]); - for(i=0; i=ArraySize(aExtension) ){ - Tcl_AppendResult(interp, "no such extension: ", zName, (char*)0); - return TCL_ERROR; - } - rc = aExtension[i].pInit(db, &zErrMsg, 0); - if( rc!=SQLITE_OK || zErrMsg ){ - Tcl_AppendResult(interp, "initialization of ", zName, " failed: ", zErrMsg, - (char*)0); - sqlite3_free(zErrMsg); - return TCL_ERROR; + for(j=2; j=ArraySize(aExtension) ){ + Tcl_AppendResult(interp, "no such extension: ", zName, (char*)0); + return TCL_ERROR; + } + rc = aExtension[i].pInit(db, &zErrMsg, 0); + if( rc!=SQLITE_OK || zErrMsg ){ + Tcl_AppendResult(interp, "initialization of ", zName, " failed: ", zErrMsg, + (char*)0); + sqlite3_free(zErrMsg); + return TCL_ERROR; + } } return TCL_OK; } diff --git a/test/spellfix.test b/test/spellfix.test index dbe8bd60e9..dfa487a1b0 100644 --- a/test/spellfix.test +++ b/test/spellfix.test @@ -16,7 +16,7 @@ set testprefix spellfix ifcapable !vtab { finish_test ; return } -load_static_extension db spellfix +load_static_extension db spellfix nextchar set vocab { rabbi rabbit rabbits rabble rabid rabies raccoon raccoons race raced racer @@ -84,6 +84,26 @@ foreach {tn word res} { } $res } +# Tests of the next_char function. +# +do_test 1.10 { + db eval { + CREATE TABLE vocab(w TEXT PRIMARY KEY); + INSERT INTO vocab SELECT word FROM t1; + } +} {} +do_execsql_test 1.11 { + SELECT next_char('re','vocab','w'); +} {a} +do_execsql_test 1.12 { + SELECT next_char('r','vocab','w'); +} {ae} +do_execsql_test 1.13 { + SELECT next_char('','vocab','w'); +} {r} +do_test 1.14 { + catchsql {SELECT next_char('','xyzzy','a')} +} {1 {no such table: xyzzy}} do_execsql_test 2.1 { CREATE VIRTUAL TABLE t2 USING spellfix1; From aa29c86e49377af479d697e4e046d55a3cf09265 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 25 Apr 2013 20:34:02 +0000 Subject: [PATCH 12/32] Rebalance FTS expressions after parsing to limit recursion during evaluation. Avoid recursion when deleting FTS expression trees. Enforce a limit on the depth of an expression tree. FossilOrigin-Name: f968d43f80cc2f236e7d09ba1e8278343e2b6976 --- ext/fts3/fts3_expr.c | 323 ++++++++++++++++++++++++++++++++++++----- manifest | 20 ++- manifest.uuid | 2 +- test/fts3expr3.test | 202 ++++++++++++++++++++++++++ test/permutations.test | 1 + 5 files changed, 504 insertions(+), 44 deletions(-) create mode 100644 test/fts3expr3.test diff --git a/ext/fts3/fts3_expr.c b/ext/fts3/fts3_expr.c index 04f38483a3..c767d7e71e 100644 --- a/ext/fts3/fts3_expr.c +++ b/ext/fts3/fts3_expr.c @@ -640,8 +640,10 @@ static int fts3ExprParse( } pNot->eType = FTSQUERY_NOT; pNot->pRight = p; + p->pParent = pNot; if( pNotBranch ){ pNot->pLeft = pNotBranch; + pNotBranch->pParent = pNot; } pNotBranch = pNot; p = pPrev; @@ -729,6 +731,7 @@ static int fts3ExprParse( pIter = pIter->pLeft; } pIter->pLeft = pRet; + pRet->pParent = pIter; pRet = pNotBranch; } } @@ -745,6 +748,223 @@ exprparse_out: return rc; } +/* +** Return SQLITE_ERROR if the maximum depth of the expression tree passed +** as the only argument is more than nMaxDepth. +*/ +static int fts3ExprCheckDepth(Fts3Expr *p, int nMaxDepth){ + int rc = SQLITE_OK; + if( p ){ + if( nMaxDepth==0 ){ + rc = SQLITE_ERROR; + }else{ + rc = fts3ExprCheckDepth(p->pLeft, nMaxDepth-1); + if( rc==SQLITE_OK ){ + rc = fts3ExprCheckDepth(p->pRight, nMaxDepth-1); + } + } + } + return rc; +} + +/* +** This function attempts to transform the expression tree at (*pp) to +** an equivalent but more balanced form. The tree is modified in place. +** If successful, SQLITE_OK is returned and (*pp) set to point to the +** new root expression node. +** +** nMaxDepth is the maximum allowable depth of the balanced sub-tree. +** +** Otherwise, if an error occurs, an SQLite error code is returned and +** expression (*pp) freed. +*/ +static int fts3ExprBalance(Fts3Expr **pp, int nMaxDepth){ + int rc = SQLITE_OK; /* Return code */ + Fts3Expr *pRoot = *pp; /* Initial root node */ + Fts3Expr *pFree = 0; /* List of free nodes. Linked by pParent. */ + int eType = pRoot->eType; /* Type of node in this tree */ + + if( nMaxDepth==0 ){ + rc = SQLITE_ERROR; + } + + if( rc==SQLITE_OK && (eType==FTSQUERY_AND || eType==FTSQUERY_OR) ){ + Fts3Expr **apLeaf; + apLeaf = (Fts3Expr **)sqlite3_malloc(sizeof(Fts3Expr *) * nMaxDepth); + if( 0==apLeaf ){ + rc = SQLITE_NOMEM; + }else{ + memset(apLeaf, 0, sizeof(Fts3Expr *) * nMaxDepth); + } + + if( rc==SQLITE_OK ){ + int i; + Fts3Expr *p; + + /* Set $p to point to the left-most leaf in the tree of eType nodes. */ + for(p=pRoot; p->eType==eType; p=p->pLeft){ + assert( p->pParent==0 || p->pParent->pLeft==p ); + assert( p->pLeft && p->pRight ); + } + + /* This loop runs once for each leaf in the tree of eType nodes. */ + while( 1 ){ + int iLvl; + Fts3Expr *pParent = p->pParent; /* Current parent of p */ + + assert( pParent==0 || pParent->pLeft==p ); + p->pParent = 0; + if( pParent ){ + pParent->pLeft = 0; + }else{ + pRoot = 0; + } + rc = fts3ExprBalance(&p, nMaxDepth-1); + if( rc!=SQLITE_OK ) break; + + for(iLvl=0; p && iLvlpLeft = apLeaf[iLvl]; + pFree->pRight = p; + pFree->pLeft->pParent = pFree; + pFree->pRight->pParent = pFree; + + p = pFree; + pFree = pFree->pParent; + p->pParent = 0; + apLeaf[iLvl] = 0; + } + } + if( p ){ + sqlite3Fts3ExprFree(p); + rc = SQLITE_ERROR; + break; + } + + /* If that was the last leaf node, break out of the loop */ + if( pParent==0 ) break; + + /* Set $p to point to the next leaf in the tree of eType nodes */ + for(p=pParent->pRight; p->eType==eType; p=p->pLeft); + + /* Remove pParent from the original tree. */ + assert( pParent->pParent==0 || pParent->pParent->pLeft==pParent ); + pParent->pRight->pParent = pParent->pParent; + if( pParent->pParent ){ + pParent->pParent->pLeft = pParent->pRight; + }else{ + assert( pParent==pRoot ); + pRoot = pParent->pRight; + } + + /* Link pParent into the free node list. It will be used as an + ** internal node of the new tree. */ + pParent->pParent = pFree; + pFree = pParent; + } + + if( rc==SQLITE_OK ){ + p = 0; + for(i=0; ipParent = 0; + }else{ + pFree->pRight = p; + pFree->pLeft = apLeaf[i]; + pFree->pLeft->pParent = pFree; + pFree->pRight->pParent = pFree; + + p = pFree; + pFree = pFree->pParent; + p->pParent = 0; + } + } + } + pRoot = p; + }else{ + /* An error occurred. Delete the contents of the apLeaf[] array + ** and pFree list. Everything else is cleaned up by the call to + ** sqlite3Fts3ExprFree(pRoot) below. */ + Fts3Expr *pDel; + for(i=0; ipParent; + sqlite3_free(pDel); + } + } + + assert( pFree==0 ); + sqlite3_free( apLeaf ); + } + } + + if( rc!=SQLITE_OK ){ + sqlite3Fts3ExprFree(pRoot); + pRoot = 0; + } + *pp = pRoot; + return rc; +} + +/* +** This function is similar to sqlite3Fts3ExprParse(), with the following +** differences: +** +** 1. It does not do expression rebalancing. +** 2. It does not check that the expression does not exceed the +** maximum allowable depth. +** 3. Even if it fails, *ppExpr may still be set to point to an +** expression tree. It should be deleted using sqlite3Fts3ExprFree() +** in this case. +*/ +static int fts3ExprParseUnbalanced( + sqlite3_tokenizer *pTokenizer, /* Tokenizer module */ + int iLangid, /* Language id for tokenizer */ + char **azCol, /* Array of column names for fts3 table */ + int bFts4, /* True to allow FTS4-only syntax */ + int nCol, /* Number of entries in azCol[] */ + int iDefaultCol, /* Default column to query */ + const char *z, int n, /* Text of MATCH query */ + Fts3Expr **ppExpr /* OUT: Parsed query structure */ +){ + static const int MAX_EXPR_DEPTH = 12; + int nParsed; + int rc; + ParseContext sParse; + + memset(&sParse, 0, sizeof(ParseContext)); + sParse.pTokenizer = pTokenizer; + sParse.iLangid = iLangid; + sParse.azCol = (const char **)azCol; + sParse.nCol = nCol; + sParse.iDefaultCol = iDefaultCol; + sParse.bFts4 = bFts4; + if( z==0 ){ + *ppExpr = 0; + return SQLITE_OK; + } + if( n<0 ){ + n = (int)strlen(z); + } + rc = fts3ExprParse(&sParse, z, n, ppExpr, &nParsed); + assert( rc==SQLITE_OK || *ppExpr==0 ); + + /* Check for mismatched parenthesis */ + if( rc==SQLITE_OK && sParse.nNest ){ + rc = SQLITE_ERROR; + } + + return rc; +} + /* ** Parameters z and n contain a pointer to and length of a buffer containing ** an fts3 query expression, respectively. This function attempts to parse the @@ -779,29 +999,20 @@ int sqlite3Fts3ExprParse( const char *z, int n, /* Text of MATCH query */ Fts3Expr **ppExpr /* OUT: Parsed query structure */ ){ - int nParsed; - int rc; - ParseContext sParse; - - memset(&sParse, 0, sizeof(ParseContext)); - sParse.pTokenizer = pTokenizer; - sParse.iLangid = iLangid; - sParse.azCol = (const char **)azCol; - sParse.nCol = nCol; - sParse.iDefaultCol = iDefaultCol; - sParse.bFts4 = bFts4; - if( z==0 ){ - *ppExpr = 0; - return SQLITE_OK; + static const int MAX_EXPR_DEPTH = 12; + int rc = fts3ExprParseUnbalanced( + pTokenizer, iLangid, azCol, bFts4, nCol, iDefaultCol, z, n, ppExpr + ); + + /* Rebalance the expression. And check that its depth does not exceed + ** MAX_EXPR_DEPTH. */ + if( rc==SQLITE_OK && *ppExpr ){ + rc = fts3ExprBalance(ppExpr, MAX_EXPR_DEPTH); + if( rc==SQLITE_OK ){ + rc = fts3ExprCheckDepth(*ppExpr, MAX_EXPR_DEPTH); + } } - if( n<0 ){ - n = (int)strlen(z); - } - rc = fts3ExprParse(&sParse, z, n, ppExpr, &nParsed); - - /* Check for mismatched parenthesis */ - if( rc==SQLITE_OK && sParse.nNest ){ - rc = SQLITE_ERROR; + if( rc!=SQLITE_OK ){ sqlite3Fts3ExprFree(*ppExpr); *ppExpr = 0; } @@ -810,16 +1021,40 @@ int sqlite3Fts3ExprParse( } /* -** Free a parsed fts3 query expression allocated by sqlite3Fts3ExprParse(). +** Free a single node of an expression tree. */ -void sqlite3Fts3ExprFree(Fts3Expr *p){ - if( p ){ - assert( p->eType==FTSQUERY_PHRASE || p->pPhrase==0 ); - sqlite3Fts3ExprFree(p->pLeft); - sqlite3Fts3ExprFree(p->pRight); - sqlite3Fts3EvalPhraseCleanup(p->pPhrase); - sqlite3_free(p->aMI); - sqlite3_free(p); +static void fts3FreeExprNode(Fts3Expr *p){ + assert( p->eType==FTSQUERY_PHRASE || p->pPhrase==0 ); + sqlite3Fts3EvalPhraseCleanup(p->pPhrase); + sqlite3_free(p->aMI); + sqlite3_free(p); +} + +/* +** Free a parsed fts3 query expression allocated by sqlite3Fts3ExprParse(). +** +** This function would be simpler if it recursively called itself. But +** that would mean passing a sufficiently large expression to ExprParse() +** could cause a stack overflow. +*/ +void sqlite3Fts3ExprFree(Fts3Expr *pDel){ + Fts3Expr *p; + assert( pDel==0 || pDel->pParent==0 ); + for(p=pDel; p && (p->pLeft||p->pRight); p=(p->pLeft ? p->pLeft : p->pRight)){ + assert( p->pParent==0 || p==p->pParent->pRight || p==p->pParent->pLeft ); + } + while( p ){ + Fts3Expr *pParent = p->pParent; + fts3FreeExprNode(p); + if( pParent && p==pParent->pLeft && pParent->pRight ){ + p = pParent->pRight; + while( p && (p->pLeft || p->pRight) ){ + assert( p==p->pParent->pRight || p==p->pParent->pLeft ); + p = (p->pLeft ? p->pLeft : p->pRight); + } + }else{ + p = pParent; + } } } @@ -871,6 +1106,9 @@ static int queryTestTokenizer( ** the returned expression text and then freed using sqlite3_free(). */ static char *exprToString(Fts3Expr *pExpr, char *zBuf){ + if( pExpr==0 ){ + return sqlite3_mprintf(""); + } switch( pExpr->eType ){ case FTSQUERY_PHRASE: { Fts3Phrase *pPhrase = pExpr->pPhrase; @@ -978,10 +1216,19 @@ static void fts3ExprTest( azCol[ii] = (char *)sqlite3_value_text(argv[ii+2]); } - rc = sqlite3Fts3ExprParse( - pTokenizer, 0, azCol, 0, nCol, nCol, zExpr, nExpr, &pExpr - ); + if( sqlite3_user_data(context) ){ + rc = sqlite3Fts3ExprParse( + pTokenizer, 0, azCol, 0, nCol, nCol, zExpr, nExpr, &pExpr + ); + assert( rc==SQLITE_OK || pExpr==0 ); + }else{ + rc = fts3ExprParseUnbalanced( + pTokenizer, 0, azCol, 0, nCol, nCol, zExpr, nExpr, &pExpr + ); + } + if( rc!=SQLITE_OK && rc!=SQLITE_NOMEM ){ + sqlite3Fts3ExprFree(pExpr); sqlite3_result_error(context, "Error parsing expression", -1); }else if( rc==SQLITE_NOMEM || !(zBuf = exprToString(pExpr, 0)) ){ sqlite3_result_error_nomem(context); @@ -1004,9 +1251,15 @@ exprtest_out: ** with database connection db. */ int sqlite3Fts3ExprInitTestInterface(sqlite3* db){ - return sqlite3_create_function( + int rc = sqlite3_create_function( db, "fts3_exprtest", -1, SQLITE_UTF8, 0, fts3ExprTest, 0, 0 ); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "fts3_exprtest_rebalance", + -1, SQLITE_UTF8, (void *)1, fts3ExprTest, 0, 0 + ); + } + return rc; } #endif diff --git a/manifest b/manifest index f702eb82f2..3441db13bd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarnings. -D 2013-04-22T23:38:50.007 +C Rebalance\sFTS\sexpressions\safter\sparsing\sto\slimit\srecursion\sduring\sevaluation.\sAvoid\srecursion\swhen\sdeleting\sFTS\sexpression\strees.\sEnforce\sa\slimit\son\sthe\sdepth\sof\san\sexpression\stree. +D 2013-04-25T20:34:02.667 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 87591ea5bf7d6ed521ad42d5bc69c124debe11a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -59,7 +59,7 @@ F ext/fts3/fts3.c 784aadfb4c2a217c3eb1feaecac924989f29728f F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h 352c8a83ee4c6a14ced1759a39dd890ab947cbe0 F ext/fts3/fts3_aux.c b02632f6dd0e375ce97870206d914ea6d8df5ccd -F ext/fts3/fts3_expr.c 6cb4410f87676ae633bd7923bbc78526cb839c4d +F ext/fts3/fts3_expr.c a01c0a2e00d0e848f7cdda3b4114fe03b560f59b F ext/fts3/fts3_hash.c 8dd2d06b66c72c628c2732555a32bc0943114914 F ext/fts3/fts3_hash.h 39cf6874dc239d6b4e30479b1975fe5b22a3caaf F ext/fts3/fts3_icu.c e319e108661147bcca8dd511cd562f33a1ba81b5 @@ -494,6 +494,7 @@ F test/fts3drop.test 1b906e293d6773812587b3dc458cb9e8f3f0c297 F test/fts3e.test 1f6c6ac9cc8b772ca256e6b22aaeed50c9350851 F test/fts3expr.test 5e745b2b6348499d9ef8d59015de3182072c564c F test/fts3expr2.test 18da930352e5693eaa163a3eacf96233b7290d1a +F test/fts3expr3.test aed4e71aa40ac1dcba860d1c43e7e23aa812eb1c F test/fts3fault.test cb72dccb0a3b9f730f16c5240f3fcb9303eb1660 F test/fts3fault2.test 3198eef2804deea7cac8403e771d9cbcb752d887 F test/fts3first.test dbdedd20914c8d539aa3206c9b34a23775644641 @@ -670,7 +671,7 @@ F test/pageropt.test 6b8f6a123a5572c195ad4ae40f2987007923bbd6 F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16 F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 -F test/permutations.test d2918e9ee6d0008292e5a70e90eed23b0440ca88 +F test/permutations.test 3d0bab9c49c1ec08b868059e30a3e1956f2162e2 F test/pragma.test 60d29cd3d8098a2c20bf4c072810f99e3bf2757a F test/pragma2.test 3a55f82b954242c642f8342b17dffc8b47472947 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 @@ -1054,7 +1055,10 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 49cfa14fceeef2d55b449eb927c283ce6f650c07 -R 83c630a254d9cc209b10e1a83d85002b -U drh -Z a7a8588b07b6fce1e57eced049fe41f1 +P 1a1cf5aa86734c832d845e07780262a178188d56 +R 821db2ca4e9de1c38e42323dffba1c12 +T *branch * fts3-expr-rebalance +T *sym-fts3-expr-rebalance * +T -sym-trunk * +U dan +Z 130034041e106884ea2e83109b05e05e diff --git a/manifest.uuid b/manifest.uuid index fa38fa5b2c..7824179050 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1a1cf5aa86734c832d845e07780262a178188d56 \ No newline at end of file +f968d43f80cc2f236e7d09ba1e8278343e2b6976 \ No newline at end of file diff --git a/test/fts3expr3.test b/test/fts3expr3.test new file mode 100644 index 0000000000..51a6adf58e --- /dev/null +++ b/test/fts3expr3.test @@ -0,0 +1,202 @@ +# 2009 January 1 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the part of the FTS3 expression +# parser that rebalances large expressions. +# +# $Id: fts3expr2.test,v 1.2 2009/06/05 17:09:12 drh Exp $ +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/malloc_common.tcl +set ::testprefix fts3expr3 + +# If SQLITE_ENABLE_FTS3 is defined, omit this file. +ifcapable !fts3 { + finish_test + return +} + +set sqlite_fts3_enable_parentheses 1 + +proc strip_phrase_data {L} { + if {[lindex $L 0] eq "PHRASE"} { + return [list P [lrange $L 3 end]] + } + return [list \ + [lindex $L 0] \ + [strip_phrase_data [lindex $L 1]] \ + [strip_phrase_data [lindex $L 2]] \ + ] +} +proc test_fts3expr2 {expr} { + strip_phrase_data [ + db one {SELECT fts3_exprtest_rebalance('simple', $expr, 'a', 'b', 'c')} + ] +} + +proc balanced_exprtree_structure {nEntry} { + set L [list] + for {set i 1} {$i <= $nEntry} {incr i} { + lappend L xxx + } + while {[llength $L] > 1} { + set N [list] + if {[llength $L] % 2} { + foreach {a b} [lrange $L 0 end-1] { lappend N [list AND $a $b] } + lappend N [lindex $L end] + } else { + foreach {a b} $L { lappend N [list AND $a $b] } + } + set L $N + } + return [lindex $L 0] +} + +proc balanced_and_tree {nEntry} { + set query [balanced_exprtree_structure $nEntry] + if {$query == "xxx"} { + return "P 1" + } + for {set i 1} {$i <= $nEntry} {incr i} { + regsub xxx $query "{P $i}" query + } + return $query +} + +proc random_tree_structure {nEntry bParen op} { + set query xxx + for {set i 1} {$i < $nEntry} {incr i} { + set x1 [expr int(rand()*4.0)] + set x2 [expr int(rand()*2.0)] + if {$x1==0 && $bParen} { + set query "($query)" + } + if {$x2} { + set query "xxx $op $query" + } else { + set query "$query $op xxx" + } + } + return $query +} + +proc random_and_query {nEntry {bParen 0}} { + set query [random_tree_structure $nEntry $bParen AND] + for {set i 1} {$i <= $nEntry} {incr i} { + regsub xxx $query $i query + } + return $query +} + +proc random_or_query {nEntry} { + set query [random_tree_structure $nEntry 1 OR] + for {set i 1} {$i <= $nEntry} {incr i} { + regsub xxx $query $i query + } + return $query +} + +proc random_andor_query {nEntry} { + set query [random_tree_structure $nEntry 1 AND] + for {set i 1} {$i <= $nEntry} {incr i} { + regsub xxx $query "([random_or_query $nEntry])" query + } + return $query +} + +proc balanced_andor_tree {nEntry} { + set tree [balanced_exprtree_structure $nEntry] + set node "{[balanced_and_tree $nEntry]}" + regsub -all AND $node OR node + regsub -all xxx $tree $node tree + return $tree +} + +# Test that queries like "1 AND 2 AND 3 AND 4..." are transformed to +# balanced trees by FTS. +# +for {set i 1} {$i < 100} {incr i} { + do_test 1.$i { + test_fts3expr2 [random_and_query $i] + } [balanced_and_tree $i] +} + +# Same again, except with parenthesis inserted at arbitrary points. +# +for {set i 1} {$i < 100} {incr i} { + do_test 2.$i { + test_fts3expr2 [random_and_query $i 1] + } [balanced_and_tree $i] +} + +# Now attempt to balance two AND trees joined by an OR. +# +for {set i 1} {$i < 100} {incr i} { + do_test 3.$i { + test_fts3expr2 "[random_and_query $i 1] OR [random_and_query $i 1]" + } [list OR [balanced_and_tree $i] [balanced_and_tree $i]] +} + +# Try trees of AND nodes with leaves that are themselves trees of OR nodes. +# +for {set i 2} {$i < 32} {incr i} { + do_test 3.$i { + test_fts3expr2 [random_andor_query $i] + } [balanced_andor_tree $i] +} + +# These exceed the depth limit. +# +for {set i 33} {$i < 40} {incr i} { + do_test 3.$i { + list [catch {test_fts3expr2 [random_andor_query $i]} msg] $msg + } {1 {Error parsing expression}} +} + +# This also exceeds the depth limit. +# +do_test 4.1 { + set q "1" + for {set i 2} {$i < 5000} {incr i} { + append q " AND $i" + } + list [catch {test_fts3expr2 $q} msg] $msg +} {1 {Error parsing expression}} + +proc create_toggle_tree {nDepth} { + if {$nDepth == 0} { return xxx } + set nNew [expr $nDepth-1] + if {$nDepth % 2} { + return "([create_toggle_tree $nNew]) OR ([create_toggle_tree $nNew])" + } + return "([create_toggle_tree $nNew]) AND ([create_toggle_tree $nNew])" +} + +do_test 4.2 { + list [catch {test_fts3expr2 [create_toggle_tree 17]} msg] $msg +} {1 {Error parsing expression}} + +set query [random_andor_query 12] +set result [balanced_andor_tree 12] +do_faultsim_test fts3expr3-fault-1 -faults oom-* -body { + test_fts3expr2 $::query +} -test { + faultsim_test_result [list 0 $::result] +} + +set sqlite_fts3_enable_parentheses 0 +finish_test + + + + diff --git a/test/permutations.test b/test/permutations.test index 9334fc49e6..3a3be27aff 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -186,6 +186,7 @@ test_suite "fts3" -prefix "" -description { fts3ak.test fts3al.test fts3am.test fts3an.test fts3ao.test fts3atoken.test fts3b.test fts3c.test fts3cov.test fts3d.test fts3defer.test fts3defer2.test fts3e.test fts3expr.test fts3expr2.test + fts3expr3.test fts3near.test fts3query.test fts3shared.test fts3snippet.test fts3sort.test fts3fault.test fts3malloc.test fts3matchinfo.test From 2bba8c249a154a3c8e38032169acfe29a3a5bf86 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 26 Apr 2013 12:08:29 +0000 Subject: [PATCH 13/32] Reduce the default SQLITE_MAX_MMAP_SIZE slightly so that it fits in a signed 32-bit integer. FossilOrigin-Name: 460752b8575320163d2659bb7ff24aff41e2bb66 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/sqliteInt.h | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index a3d9885853..41113c30a7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Added\sthe\snextchar.c\sextension.\s\sMinor\schanges\sto\sthe\sspellfix.c\sextension\nso\sthat\sit\scan\sbe\sappended\sto\san\samalgamation\sand\scompiled\swithout\sduplicating\nsymbols. -D 2013-04-25T19:31:33.149 +C Reduce\sthe\sdefault\sSQLITE_MAX_MMAP_SIZE\sslightly\sso\sthat\sit\sfits\sin\sa\nsigned\s32-bit\sinteger. +D 2013-04-26T12:08:29.821 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -196,7 +196,7 @@ F src/shell.c 5d527e5d08f05ec2c43ff194ea44bf62b974f4c9 F src/sqlite.h.in ec279b782bea05db63b8b29481f9642b406004af F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h d936f797812c28b81b26ed18345baf8db28a21a5 -F src/sqliteInt.h 3585ea1bb8776baa9f2675e9ef3d9170e7aeda29 +F src/sqliteInt.h de835c584032769461c123a564381f9808542c0e F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -1061,7 +1061,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P aabeea98f53edde68f484f1794ae70789dac3889 -R 77b136ceb7e3bfd54c75649b938a3e12 +P 56b9a417f5451631f11c5206d625f11472ee65f9 +R 83175c7cd0d9ce60e1daa5a05331eb65 U drh -Z fb4502a56cfa201ec6c890343c386675 +Z 44151e6d508d016989d25b422911b94b diff --git a/manifest.uuid b/manifest.uuid index 7c24934462..8936a5bdc4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -56b9a417f5451631f11c5206d625f11472ee65f9 \ No newline at end of file +460752b8575320163d2659bb7ff24aff41e2bb66 \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 876629a8c8..b041b3fb6e 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -563,7 +563,7 @@ extern const int sqlite3one; || defined(_WIN32) \ || (defined(__APPLE__) && defined(__MACH__)) \ || defined(__sun) -# define SQLITE_MAX_MMAP_SIZE 2147483648 +# define SQLITE_MAX_MMAP_SIZE 0x7fff0000 /* 2147418112 */ # else # define SQLITE_MAX_MMAP_SIZE 0 # endif From 6f77140f78e7c871cdbdff52639cb9a850159f21 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 26 Apr 2013 13:14:19 +0000 Subject: [PATCH 14/32] Fix harmless compiler warnings in the FTS expression parser. FossilOrigin-Name: 3c78af8c535e16518f18733325f4cd1df7ec8282 --- ext/fts3/fts3_expr.c | 3 +-- manifest | 22 +++++++++++----------- manifest.uuid | 2 +- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/ext/fts3/fts3_expr.c b/ext/fts3/fts3_expr.c index c767d7e71e..b28ab68e4b 100644 --- a/ext/fts3/fts3_expr.c +++ b/ext/fts3/fts3_expr.c @@ -895,7 +895,7 @@ static int fts3ExprBalance(Fts3Expr **pp, int nMaxDepth){ for(i=0; ipParent; sqlite3_free(pDel); } @@ -935,7 +935,6 @@ static int fts3ExprParseUnbalanced( const char *z, int n, /* Text of MATCH query */ Fts3Expr **ppExpr /* OUT: Parsed query structure */ ){ - static const int MAX_EXPR_DEPTH = 12; int nParsed; int rc; ParseContext sParse; diff --git a/manifest b/manifest index 31e42e1348..e150e60d93 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\slatest\strunk\schanges. -D 2013-04-26T06:58:06.233 +C Fix\sharmless\scompiler\swarnings\sin\sthe\sFTS\sexpression\sparser. +D 2013-04-26T13:14:19.396 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -59,7 +59,7 @@ F ext/fts3/fts3.c 784aadfb4c2a217c3eb1feaecac924989f29728f F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h 352c8a83ee4c6a14ced1759a39dd890ab947cbe0 F ext/fts3/fts3_aux.c b02632f6dd0e375ce97870206d914ea6d8df5ccd -F ext/fts3/fts3_expr.c a01c0a2e00d0e848f7cdda3b4114fe03b560f59b +F ext/fts3/fts3_expr.c 4021d21aadebb50499c482f6b7b718e4b93113e6 F ext/fts3/fts3_hash.c 8dd2d06b66c72c628c2732555a32bc0943114914 F ext/fts3/fts3_hash.h 39cf6874dc239d6b4e30479b1975fe5b22a3caaf F ext/fts3/fts3_icu.c e319e108661147bcca8dd511cd562f33a1ba81b5 @@ -86,13 +86,13 @@ F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37 F ext/misc/amatch.c 3369b2b544066e620d986f0085d039c77d1ef17f F ext/misc/closure.c fec0c8537c69843e0b7631d500a14c0527962cd6 F ext/misc/editdist3.wiki 06100a0c558921a563cbc40e0d0151902b1eef6d -F ext/misc/fuzzer.c fb64a15af978ae73fa9075b9b1dfbe82b8defc6f w src/test_fuzzer.c +F ext/misc/fuzzer.c fb64a15af978ae73fa9075b9b1dfbe82b8defc6f F ext/misc/ieee754.c 2565ce373d842977efe0922dc50b8a41b3289556 F ext/misc/nextchar.c 1131e2b36116ffc6fe6b2e3464bfdace27978b1e -F ext/misc/regexp.c c25c65fe775f5d9801fb8573e36ebe73f2c0c2e0 w src/test_regexp.c -F ext/misc/spellfix.c e323eebb877d735bc64404c16a6d758ab17a0b7a w src/test_spellfix.c +F ext/misc/regexp.c c25c65fe775f5d9801fb8573e36ebe73f2c0c2e0 +F ext/misc/spellfix.c e323eebb877d735bc64404c16a6d758ab17a0b7a F ext/misc/spellfix1.wiki dd1830444c14cf0f54dd680cc044df2ace2e9d09 -F ext/misc/wholenumber.c ce362368b9381ea48cbd951ade8df867eeeab014 w src/test_wholenumber.c +F ext/misc/wholenumber.c ce362368b9381ea48cbd951ade8df867eeeab014 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 757abea591d4ff67c0ff4e8f9776aeda86b18c14 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e @@ -1062,7 +1062,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P f968d43f80cc2f236e7d09ba1e8278343e2b6976 56b9a417f5451631f11c5206d625f11472ee65f9 -R 70b7c4a8fc1b35fd998a4752a3b57b39 -U dan -Z 98b8c0af1d54867f961d6716e238405d +P 4d08e74d34e82f3be588049c9576a5c1008435e7 +R a62e327052722b023f8b56bfc3226b16 +U drh +Z 0ddc594c1b1b9290fe656a6228646bb0 diff --git a/manifest.uuid b/manifest.uuid index 6ecc568665..157b4bd510 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4d08e74d34e82f3be588049c9576a5c1008435e7 \ No newline at end of file +3c78af8c535e16518f18733325f4cd1df7ec8282 \ No newline at end of file From 9ea88b2b484b0d80ccc0e194e70a03c545f3fdf5 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 26 Apr 2013 15:55:57 +0000 Subject: [PATCH 15/32] Update documentation with new hyperlinks. No changes to code. FossilOrigin-Name: 640eb54ad6aac9bc7109cba167389a9bcec3f21e --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/sqlite.h.in | 12 +++++++++--- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 914f1bde6e..59fe24e66e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Rebalance\sFTS\sexpressions\safter\sparsing\sto\slimit\srecursion\sduring\sevaluation.\sAvoid\srecursion\swhen\sdeleting\sFTS\sexpression\strees.\sEnforce\sa\slimit\s(currently\s12)\son\sthe\sdepth\sof\san\sexpression\stree. -D 2013-04-26T14:13:15.156 +C Update\sdocumentation\swith\snew\shyperlinks.\s\sNo\schanges\sto\scode. +D 2013-04-26T15:55:57.095 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -193,7 +193,7 @@ F src/resolve.c 83cc2d942ee216bc56956c6e6fadb691c1727fa1 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 6bfbe11e2fef81c5e18d30513ab6c69f171667eb F src/shell.c 5d527e5d08f05ec2c43ff194ea44bf62b974f4c9 -F src/sqlite.h.in ec279b782bea05db63b8b29481f9642b406004af +F src/sqlite.h.in f5efcffff16619129bcb92ac352a77fa758f86c1 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h d936f797812c28b81b26ed18345baf8db28a21a5 F src/sqliteInt.h de835c584032769461c123a564381f9808542c0e @@ -1062,7 +1062,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 460752b8575320163d2659bb7ff24aff41e2bb66 2648966f17bc1b783ef6d3b2368c613f6e02945e -R 3adacbaf5ab67ae1f94478e13972158e -U dan -Z 40f0df5151b2a7d6fa4a01e133941564 +P 49d23ef61f9ce2ffe13237b51a0e01b0b46ba96b +R 9dfdc1bbf9c2e5197fa469091ec011ea +U drh +Z 247eb74d6133c2eb692e1fce731d0d35 diff --git a/manifest.uuid b/manifest.uuid index 64a0465053..de2ac0cb31 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -49d23ef61f9ce2ffe13237b51a0e01b0b46ba96b \ No newline at end of file +640eb54ad6aac9bc7109cba167389a9bcec3f21e \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 3265b3196b..dd5f4d9bae 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -1587,7 +1587,9 @@ struct sqlite3_mem_methods { ** page cache implementation into that object.)^ ** ** [[SQLITE_CONFIG_LOG]]
SQLITE_CONFIG_LOG
-**
^The SQLITE_CONFIG_LOG option takes two arguments: a pointer to a +**
The SQLITE_CONFIG_LOG option is used to configure the SQLite +** global [error log]. +** [^The SQLITE_CONFIG_LOG option takes two arguments: a pointer to a ** function with a call signature of void(*)(void*,int,const char*), ** and a pointer to void. ^If the function pointer is not NULL, it is ** invoked by [sqlite3_log()] to process each logging event. ^If the @@ -2522,6 +2524,9 @@ int sqlite3_set_authorizer( ** as each triggered subprogram is entered. The callbacks for triggers ** contain a UTF-8 SQL comment that identifies the trigger.)^ ** +** The [SQLITE_TRACE_SIZE_LIMIT] compile-time option can be used to limit +** the length of [bound parameter] expansion in the output of sqlite3_trace(). +** ** ^The callback function registered by sqlite3_profile() is invoked ** as each SQL statement finishes. ^The profile callback contains ** the original statement text and an estimate of wall-clock time @@ -3060,7 +3065,8 @@ int sqlite3_limit(sqlite3*, int id, int newVal); **
  • ** ^If the database schema changes, instead of returning [SQLITE_SCHEMA] as it ** always used to do, [sqlite3_step()] will automatically recompile the SQL -** statement and try to run it again. +** statement and try to run it again. As many as [SQLITE_MAX_SCHEMA_RETRY] +** retries will occur before sqlite3_step() gives up and returns an error. **
    • ** **
  • @@ -6872,7 +6878,7 @@ int sqlite3_strglob(const char *zGlob, const char *zStr); /* ** CAPI3REF: Error Logging Interface ** -** ^The [sqlite3_log()] interface writes a message into the error log +** ^The [sqlite3_log()] interface writes a message into the [error log] ** established by the [SQLITE_CONFIG_LOG] option to [sqlite3_config()]. ** ^If logging is enabled, the zFormat string and subsequent arguments are ** used with [sqlite3_snprintf()] to generate the final output string. From 47a2b4a0b0067bcae3960bb07e616589d509def2 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 26 Apr 2013 16:09:29 +0000 Subject: [PATCH 16/32] Avoid using posix_fallocate() in WAL mode, as it is not supported by all file-systems. FossilOrigin-Name: 9c7523dabf4aee609287732ce787c9b9a9087e7f --- manifest | 15 ++++--- manifest.uuid | 2 +- src/os_unix.c | 115 +++++++++++++++++++++++++++++--------------------- 3 files changed, 78 insertions(+), 54 deletions(-) diff --git a/manifest b/manifest index 914f1bde6e..72d6bee5a4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Rebalance\sFTS\sexpressions\safter\sparsing\sto\slimit\srecursion\sduring\sevaluation.\sAvoid\srecursion\swhen\sdeleting\sFTS\sexpression\strees.\sEnforce\sa\slimit\s(currently\s12)\son\sthe\sdepth\sof\san\sexpression\stree. -D 2013-04-26T14:13:15.156 +C Avoid\susing\sposix_fallocate()\sin\sWAL\smode,\sas\sit\sis\snot\ssupported\sby\sall\sfile-systems. +D 2013-04-26T16:09:29.947 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -177,7 +177,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c b4ad71336fd96f97776f75587cd9e8218288f5be F src/os.h 4a46270a64e9193af4a0aaa3bc2c66dc07c29b3f F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 5a9ac4a566fb566a2ff9b9e3a9d723075d9d26a7 +F src/os_unix.c f86cd628ffb9ccf3adcc5c15b02c00a64eaf598e F src/os_win.c 673b3e3d1fa3040d8d95a7f1f5e0e553aed56cfb F src/pager.c 6c3a8a5d665498b0344395a2c9f82d5abc4cc771 F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 @@ -1062,7 +1062,10 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 460752b8575320163d2659bb7ff24aff41e2bb66 2648966f17bc1b783ef6d3b2368c613f6e02945e -R 3adacbaf5ab67ae1f94478e13972158e +P 49d23ef61f9ce2ffe13237b51a0e01b0b46ba96b +R c26b6b9eda44a997204fb8564cbd02a4 +T *branch * avoid-fallocate +T *sym-avoid-fallocate * +T -sym-trunk * U dan -Z 40f0df5151b2a7d6fa4a01e133941564 +Z 8ca670519b4ace84695b56a8d0f2a643 diff --git a/manifest.uuid b/manifest.uuid index 64a0465053..a357f425fc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -49d23ef61f9ce2ffe13237b51a0e01b0b46ba96b \ No newline at end of file +9c7523dabf4aee609287732ce787c9b9a9087e7f \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 7448afe4c7..1200ff352b 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -3193,6 +3193,51 @@ static int unixRead( } } +/* +** Attempt to seek the file-descriptor passed as the first argument to +** absolute offset iOff, then attempt to write nBuf bytes of data from +** pBuf to it. If an error occurs, return -1 and set *piErrno. Otherwise, +** return the actual number of bytes written (which may be less than +** nBuf). +*/ +static int seekAndWriteFd( + int fd, /* File descriptor to write to */ + i64 iOff, /* File offset to begin writing at */ + const void *pBuf, /* Copy data from this buffer to the file */ + int nBuf, /* Size of buffer pBuf in bytes */ + int *piErrno /* OUT: Error number if error occurs */ +){ + int rc = 0; /* Value returned by system call */ + + assert( nBuf==(nBuf&0x1ffff) ); + nBuf &= 0x1ffff; + TIMER_START; + +#if defined(USE_PREAD) + do{ rc = osPwrite(fd, pBuf, nBuf, iOff); }while( rc<0 && errno==EINTR ); +#elif defined(USE_PREAD64) + do{ rc = osPwrite64(fd, pBuf, nBuf, iOff);}while( rc<0 && errno==EINTR); +#else + do{ + i64 iSeek = lseek(fd, iOff, SEEK_SET); + SimulateIOError( iSeek-- ); + + if( iSeek!=iOff ){ + if( piErrno ) *piErrno = (iSeek==-1 ? errno : 0); + return -1; + } + rc = osWrite(fd, pBuf, nBuf); + }while( rc<0 && errno==EINTR ); +#endif + + TIMER_END; + OSTRACE(("WRITE %-3d %5d %7lld %llu\n", fd, rc, iOff, TIMER_ELAPSED)); + + if( rc<0 && piErrno ) *piErrno = errno; + return rc; +} + + /* ** Seek to the offset in id->offset then read cnt bytes into pBuf. ** Return the number of bytes actually read. Update the offset. @@ -3201,39 +3246,7 @@ static int unixRead( ** is set before returning. */ static int seekAndWrite(unixFile *id, i64 offset, const void *pBuf, int cnt){ - int got; -#if (!defined(USE_PREAD) && !defined(USE_PREAD64)) - i64 newOffset; -#endif - assert( cnt==(cnt&0x1ffff) ); - cnt &= 0x1ffff; - TIMER_START; -#if defined(USE_PREAD) - do{ got = osPwrite(id->h, pBuf, cnt, offset); }while( got<0 && errno==EINTR ); -#elif defined(USE_PREAD64) - do{ got = osPwrite64(id->h, pBuf, cnt, offset);}while( got<0 && errno==EINTR); -#else - do{ - newOffset = lseek(id->h, offset, SEEK_SET); - SimulateIOError( newOffset-- ); - if( newOffset!=offset ){ - if( newOffset == -1 ){ - ((unixFile*)id)->lastErrno = errno; - }else{ - ((unixFile*)id)->lastErrno = 0; - } - return -1; - } - got = osWrite(id->h, pBuf, cnt); - }while( got<0 && errno==EINTR ); -#endif - TIMER_END; - if( got<0 ){ - ((unixFile*)id)->lastErrno = errno; - } - - OSTRACE(("WRITE %-3d %5d %7lld %llu\n", id->h, got, offset, TIMER_ELAPSED)); - return got; + return seekAndWriteFd(id->h, offset, pBuf, cnt, &id->lastErrno); } @@ -4322,24 +4335,32 @@ static int unixShmMap( if( sStat.st_sizeh, sStat.st_size, nByte)!=0 ){ - rc = unixLogError(SQLITE_IOERR_SHMSIZE, "fallocate", - pShmNode->zFilename); + if( !bExtend ){ goto shmpage_out; } -#else - if( robust_ftruncate(pShmNode->h, nByte) ){ - rc = unixLogError(SQLITE_IOERR_SHMSIZE, "ftruncate", - pShmNode->zFilename); - goto shmpage_out; + + /* Alternatively, if bExtend is true, extend the file. Do this by + ** writing a single byte to the end of each (OS) page being + ** allocated or extended. Technically, we need only write to the + ** last page in order to extend the file. But writing to all new + ** pages forces the OS to allocate them immediately, which reduces + ** the chances of SIGBUS while accessing the mapped region later on. + */ + else{ + static const int pgsz = 4096; + int iPg; + + /* Write to the last byte of each newly allocated or extended page */ + assert( (nByte % pgsz)==0 ); + for(iPg=(sStat.st_size/pgsz); iPg<(nByte/pgsz); iPg++){ + if( seekAndWriteFd(pShmNode->h, iPg*pgsz + pgsz-1, "", 1, 0)!=1 ){ + const char *zFile = pShmNode->zFilename; + rc = unixLogError(SQLITE_IOERR_SHMSIZE, "write", zFile); + goto shmpage_out; + } + } } -#endif } } From 8e4714b3033abfe8e48ffac1845166de42d4ddc3 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 26 Apr 2013 18:36:58 +0000 Subject: [PATCH 17/32] Avoid unnecessarily reseting the pager cache after committing a transaction that takes advantage of the SQLITE_IOCAP_ATOMIC related optimization. FossilOrigin-Name: c47144e98c0a0f9e09780c945de10c57b6a495ea --- manifest | 14 +++++------ manifest.uuid | 2 +- src/pager.c | 5 ++++ test/io.test | 66 +++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 79 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index de4051d566..c5c2b1f688 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\susing\sposix_fallocate()\sin\sWAL\smode,\sas\sit\sis\snot\ssupported\sby\sall\sfile-systems. -D 2013-04-26T17:00:52.665 +C Avoid\sunnecessarily\sreseting\sthe\spager\scache\safter\scommitting\sa\stransaction\sthat\stakes\sadvantage\sof\sthe\sSQLITE_IOCAP_ATOMIC\srelated\soptimization. +D 2013-04-26T18:36:58.055 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -179,7 +179,7 @@ F src/os.h 4a46270a64e9193af4a0aaa3bc2c66dc07c29b3f F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c f86cd628ffb9ccf3adcc5c15b02c00a64eaf598e F src/os_win.c 673b3e3d1fa3040d8d95a7f1f5e0e553aed56cfb -F src/pager.c 6c3a8a5d665498b0344395a2c9f82d5abc4cc771 +F src/pager.c 4a9160d268977e56ae2df90182050ab30fca715d F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 @@ -571,7 +571,7 @@ F test/instr.test a34e1d46a9eefb098a7167ef0e730a4a3d82fba0 F test/intarray.test 066b7d7ac38d25bf96f87f1b017bfc687551cdd4 F test/interrupt.test dfe9a67a94b0b2d8f70545ba1a6cca10780d71cc F test/intpkey.test 7af30f6ae852d8d1c2b70e4bf1551946742e92d8 -F test/io.test a4be25a446a99a0604ceecc039ee7363c56e4185 +F test/io.test b90105d295225a712e2e9cb685876101712058b5 F test/ioerr.test 40bb2cfcab63fb6aa7424cd97812a84bc16b5fb8 F test/ioerr2.test 9d71166f8466eda510f1af6137bdabaa82b5408d F test/ioerr3.test d3cec5e1a11ad6d27527d0d38573fbff14c71bdd @@ -1062,7 +1062,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 640eb54ad6aac9bc7109cba167389a9bcec3f21e 9c7523dabf4aee609287732ce787c9b9a9087e7f -R b4ea9ecffc738493e7005d865dad4b9a +P 1bbb4be1a25947f75b2b0c6f368199016b6f7de3 +R 50c4e22fb69fce6bfb1bbec0cac61356 U dan -Z 8914d43d4bd2278afd40d03ae3982d74 +Z 042ff5429ace151fb0c673232d14fee9 diff --git a/manifest.uuid b/manifest.uuid index 86ca24f5f4..58806060f6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1bbb4be1a25947f75b2b0c6f368199016b6f7de3 \ No newline at end of file +c47144e98c0a0f9e09780c945de10c57b6a495ea \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 1c8b5df651..dffcf602d7 100644 --- a/src/pager.c +++ b/src/pager.c @@ -5927,6 +5927,11 @@ static int pager_incr_changecounter(Pager *pPager, int isDirectMode){ pPager->aStat[PAGER_STAT_WRITE]++; } if( rc==SQLITE_OK ){ + /* Update the pager's copy of the change-counter. Otherwise, the + ** next time a read transaction is opened the cache will be + ** flushed (as the change-counter values will not match). */ + const void *pCopy = (const void *)&((const char *)zBuf)[24]; + memcpy(&pPager->dbFileVers, pCopy, sizeof(pPager->dbFileVers)); pPager->changeCountDone = 1; } }else{ diff --git a/test/io.test b/test/io.test index bf4d157275..f9583e0e33 100644 --- a/test/io.test +++ b/test/io.test @@ -38,6 +38,10 @@ sqlite3 db test.db -vfs devsym # # io-5.* - Test that the default page size is selected and used # correctly. +# +# io-6.* - Test that the pager-cache is not being flushed unnecessarily +# after a transaction that uses the special atomic-write path +# is committed. # set ::nWrite 0 @@ -565,5 +569,67 @@ foreach {char sectorsize pgsize} { } $pgsize } +#---------------------------------------------------------------------- +# +do_test io-6.1 { + db close + sqlite3_simulate_device -char atomic + forcedelete test.db + sqlite3 db test.db -vfs devsym + execsql { + PRAGMA mmap_size = 0; + PRAGMA page_size = 1024; + CREATE TABLE t1(x); + CREATE TABLE t2(x); + CREATE TABLE t3(x); + CREATE INDEX i3 ON t3(x); + INSERT INTO t3 VALUES(randomblob(100)); + INSERT INTO t3 SELECT randomblob(100) FROM t3; + INSERT INTO t3 SELECT randomblob(100) FROM t3; + INSERT INTO t3 SELECT randomblob(100) FROM t3; + INSERT INTO t3 SELECT randomblob(100) FROM t3; + INSERT INTO t3 SELECT randomblob(100) FROM t3; + INSERT INTO t3 SELECT randomblob(100) FROM t3; + INSERT INTO t3 SELECT randomblob(100) FROM t3; + INSERT INTO t3 SELECT randomblob(100) FROM t3; + INSERT INTO t3 SELECT randomblob(100) FROM t3; + INSERT INTO t3 SELECT randomblob(100) FROM t3; + INSERT INTO t3 SELECT randomblob(100) FROM t3; + } + + db_save_and_close +} {} + +foreach {tn sql} { + 1 { BEGIN; + INSERT INTO t1 VALUES('123'); + INSERT INTO t2 VALUES('456'); + COMMIT; + } + 2 { BEGIN; + INSERT INTO t1 VALUES('123'); + COMMIT; + } +} { + db_restore + sqlite3 db test.db -vfs devsym + execsql { + PRAGMA mmap_size = 0; + SELECT x FROM t3 ORDER BY rowid; + SELECT x FROM t3 ORDER BY x; + } + do_execsql_test 6.2.$tn.1 { PRAGMA integrity_check } {ok} + do_execsql_test 6.2.$tn.2 $sql + + # Corrupt the database file on disk. This should not matter for the + # purposes of the following "PRAGMA integrity_check", as the entire + # database should be cached in the pager-cache. If corruption is + # reported, it indicates that executing $sql caused the pager cache + # to be flushed. Which is a bug. + hexio_write test.db [expr 1024 * 5] [string repeat 00 2048] + do_execsql_test 6.2.$tn.3 { PRAGMA integrity_check } {ok} +} + sqlite3_simulate_device -char {} -sectorsize 0 finish_test + From a13090fee950b1b34abdb532b18315b30758abe1 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 26 Apr 2013 19:33:34 +0000 Subject: [PATCH 18/32] Fix a formatting typo in a comment. No changes to code. FossilOrigin-Name: 7a97226ffe174349e7113340f5354c4e44bd9738 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/sqlite.h.in | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index c5c2b1f688..0536079cee 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sunnecessarily\sreseting\sthe\spager\scache\safter\scommitting\sa\stransaction\sthat\stakes\sadvantage\sof\sthe\sSQLITE_IOCAP_ATOMIC\srelated\soptimization. -D 2013-04-26T18:36:58.055 +C Fix\sa\sformatting\stypo\sin\sa\scomment.\s\sNo\schanges\sto\scode. +D 2013-04-26T19:33:34.383 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -193,7 +193,7 @@ F src/resolve.c 83cc2d942ee216bc56956c6e6fadb691c1727fa1 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 6bfbe11e2fef81c5e18d30513ab6c69f171667eb F src/shell.c 5d527e5d08f05ec2c43ff194ea44bf62b974f4c9 -F src/sqlite.h.in f5efcffff16619129bcb92ac352a77fa758f86c1 +F src/sqlite.h.in e6c241170da854abf6f15ec7e6c565348b690169 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h d936f797812c28b81b26ed18345baf8db28a21a5 F src/sqliteInt.h de835c584032769461c123a564381f9808542c0e @@ -1062,7 +1062,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 1bbb4be1a25947f75b2b0c6f368199016b6f7de3 -R 50c4e22fb69fce6bfb1bbec0cac61356 -U dan -Z 042ff5429ace151fb0c673232d14fee9 +P c47144e98c0a0f9e09780c945de10c57b6a495ea +R 9aa1f84a471b601353c6669ae1266379 +U drh +Z 18431ca135b1f0070cd379da7401fa7c diff --git a/manifest.uuid b/manifest.uuid index 58806060f6..89ec5fbc80 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c47144e98c0a0f9e09780c945de10c57b6a495ea \ No newline at end of file +7a97226ffe174349e7113340f5354c4e44bd9738 \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index dd5f4d9bae..5eb74a8756 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -1589,7 +1589,7 @@ struct sqlite3_mem_methods { ** [[SQLITE_CONFIG_LOG]]
    SQLITE_CONFIG_LOG
    **
    The SQLITE_CONFIG_LOG option is used to configure the SQLite ** global [error log]. -** [^The SQLITE_CONFIG_LOG option takes two arguments: a pointer to a +** (^The SQLITE_CONFIG_LOG option takes two arguments: a pointer to a ** function with a call signature of void(*)(void*,int,const char*), ** and a pointer to void. ^If the function pointer is not NULL, it is ** invoked by [sqlite3_log()] to process each logging event. ^If the From c5797545de1b6bd30acc640443069a612e1b6194 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 27 Apr 2013 12:13:29 +0000 Subject: [PATCH 19/32] Untested fix for building on VxWorks. FossilOrigin-Name: f14d55cf358b0392d3b8cd61dc85f43a610a8edf --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_unix.c | 3 +-- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 0536079cee..b47b06dfb4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sformatting\stypo\sin\sa\scomment.\s\sNo\schanges\sto\scode. -D 2013-04-26T19:33:34.383 +C Untested\sfix\sfor\sbuilding\son\sVxWorks. +D 2013-04-27T12:13:29.526 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -177,7 +177,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c b4ad71336fd96f97776f75587cd9e8218288f5be F src/os.h 4a46270a64e9193af4a0aaa3bc2c66dc07c29b3f F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c f86cd628ffb9ccf3adcc5c15b02c00a64eaf598e +F src/os_unix.c 5a214c5431fd005dbb3b8bbaaa306433e8e9b48f F src/os_win.c 673b3e3d1fa3040d8d95a7f1f5e0e553aed56cfb F src/pager.c 4a9160d268977e56ae2df90182050ab30fca715d F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 @@ -1062,7 +1062,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P c47144e98c0a0f9e09780c945de10c57b6a495ea -R 9aa1f84a471b601353c6669ae1266379 +P 7a97226ffe174349e7113340f5354c4e44bd9738 +R 45f16684eb908e3d81d1d0d938ab9d11 U drh -Z 18431ca135b1f0070cd379da7401fa7c +Z 3089ee3240913ca0c76196e0917eb228 diff --git a/manifest.uuid b/manifest.uuid index 89ec5fbc80..8005ea1784 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7a97226ffe174349e7113340f5354c4e44bd9738 \ No newline at end of file +f14d55cf358b0392d3b8cd61dc85f43a610a8edf \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 1200ff352b..0074412410 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -5287,9 +5287,8 @@ static int fillInUnixFile( if( h>=0 ) robust_close(pNew, h, __LINE__); h = -1; osUnlink(zFilename); - isDelete = 0; + pNew->ctrlFlags |= UNIXFILE_DELETE; } - if( isDelete ) pNew->ctrlFlags |= UNIXFILE_DELETE; #endif if( rc!=SQLITE_OK ){ if( h>=0 ) robust_close(pNew, h, __LINE__); From 015db9c85995e1661079afd11fb7c45a95ceee7e Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 27 Apr 2013 18:06:40 +0000 Subject: [PATCH 20/32] Remove spellfix virtual table documentation from the source tree. Reference the separate documentation on the website instead. FossilOrigin-Name: adcf78909ff9064b6e3c4dd15ccd3245c8cf270b --- ext/misc/editdist3.wiki | 114 ---------- ext/misc/spellfix.c | 2 +- ext/misc/spellfix1.wiki | 464 ---------------------------------------- manifest | 14 +- manifest.uuid | 2 +- 5 files changed, 8 insertions(+), 588 deletions(-) delete mode 100644 ext/misc/editdist3.wiki delete mode 100644 ext/misc/spellfix1.wiki diff --git a/ext/misc/editdist3.wiki b/ext/misc/editdist3.wiki deleted file mode 100644 index 494922c5e9..0000000000 --- a/ext/misc/editdist3.wiki +++ /dev/null @@ -1,114 +0,0 @@ -The editdist3 algorithm - -The editdist3 algorithm is a function that computes the minimum edit distance -(a.k.a. the Levenshtein distance) between two input strings. Features of -editdist3 include: - - * It works with unicode (UTF8) text. - - * A table of insertion, deletion, and substitution costs can be - provided by the application. - - * Multi-character insertsions, deletions, and substitutions can be - enumerated in the cost table. - -

    The COST table

    - -To program the costs of editdist3, create a table such as the following: - -
    -CREATE TABLE editcost(
-  iLang INT,   -- The language ID
-  cFrom TEXT,  -- Convert text from this
-  cTo   TEXT,  -- Convert text into this
-  iCost INT    -- The cost of doing the conversionnn
-);
-
    - -The cost table can be named anything you want - it does not have to be called -"editcost". And the table can contain additional columns. However, it the -table must contain the four columns show above, with exactly the names shown. - -The iLang column is a non-negative integer that identifies a set of costs -appropriate for a particular language. The editdist3 function will only use -a single iLang value for any given edit-distance computation. The default -value is 0. It is recommended that applications that only need to use a -single langauge always use iLang==0 for all entries. - -The iCost column is the numeric cost of transforming cFrom into cTo. This -value should be a non-negative integer, and should probably be less than 100. -The default single-character insertion and deletion costs are 100 and the -default single-character to single-character substitution cost is 150. A -cost of 10000 or more is considered "infinite" and causes the rule to be -ignored. - -The cFrom and cTo columns show edit transformation strings. Either or both -columns may contain more than one character. Or either column (but not both) -may hold an empty string. When cFrom is empty, that is the cost of inserting -cTo. When cTo is empty, that is the cost of deleting cFrom. - -In the spellfix1 algorithm, cFrom is the text as the user entered it and -cTo is the correctly spelled text as it exists in the database. The goal -of the editdist3 algorithm is to determine how close the user-entered text is -to the dictionary text. - -There are three special-case entries in the cost table: - - - - - - -
    cFromcToMeaning
    '''?'The default insertion cost
    '?'''The default deletion cost
    '?''?'The default substitution cost
    - -If any of the special-case entries shows above are omitted, then the -value of 100 is used for insertion and deletion and 150 is used for -substitution. To disable the default insertion, deletion, and/or substitution -set their respective cost to 10000 or more. - -Other entries in the cost table specific transforms for particular characters. -The cost of specific transforms should be less than the default costs, or else -the default costs will take precedence and the specific transforms will never -be used. - -Some example, cost table entries: - -
    -INSERT INTO editcost(iLang, cFrom, cTo, iCost)
-VALUES(0, 'a', 'ä', 5);
-
    - -The rule above says that the letter "a" in user input can be matched against -the letter "ä" in the dictionary with a penalty of 5. - -
    -INSERT INTO editcost(iLang, cFrom, cTo, iCost)
-VALUES(0, 'ss', 'ß', 8);
-
    - -The number of characters in cFrom and cTo do not need to be the same. The -rule above says that "ss" on user input will match "ß" with a penalty of 8. - -

    Experimenting with the editcost3() function

    - -The [./spellfix1.wiki | spellfix1 virtual table] -uses editdist3 if the "edit_cost_table=TABLE" option -is specified as an argument when the spellfix1 virtual table is created. -But editdist3 can also be tested directly using the built-in "editdist3()" -SQL function. The editdist3() SQL function has 3 forms: - - 1. editdist3('TABLENAME'); - 2. editdist3('string1', 'string2'); - 3. editdist3('string1', 'string2', langid); - -The first form loads the edit distance coefficients from a table called -'TABLENAME'. Any prior coefficients are discarded. So when experimenting -with weights and the weight table changes, simply rerun the single-argument -form of editdist3() to reload revised coefficients. Note that the -edit distance -weights used by the editdist3() SQL function are independent from the -weights used by the spellfix1 virtual table. - -The second and third forms return the computed edit distance between strings -'string1' and "string2'. In the second form, an language id of 0 is used. -The language id is specified in the third form. diff --git a/ext/misc/spellfix.c b/ext/misc/spellfix.c index e0a7d1f374..c368b34e8f 100644 --- a/ext/misc/spellfix.c +++ b/ext/misc/spellfix.c @@ -12,7 +12,7 @@ ** ** This module implements the spellfix1 VIRTUAL TABLE that can be used ** to search a large vocabulary for close matches. See separate -** documentation files (spellfix1.wiki and editdist3.wiki) for details. +** documentation (http://www.sqlite.org/spellfix1.html) for details. */ #include "sqlite3ext.h" SQLITE_EXTENSION_INIT1 diff --git a/ext/misc/spellfix1.wiki b/ext/misc/spellfix1.wiki deleted file mode 100644 index 288c55dd2e..0000000000 --- a/ext/misc/spellfix1.wiki +++ /dev/null @@ -1,464 +0,0 @@ -The Spellfix1 Virtual Table - -This spellfix1 virtual table is used to search -a large vocabulary for close matches. For example, spellfix1 -can be used to suggest corrections to misspelled words. Or, -it could be used with FTS4 to do full-text search using potentially -misspelled words. - -Create an instance of the spellfix1 virtual table like this: - -
    -CREATE VIRTUAL TABLE demo USING spellfix1;
-
    - -The "spellfix1" term is the name of this module and must be entered as -shown. The "demo" term is the -name of the virtual table you will be creating and can be altered -to suit the needs of your application. The virtual table is initially -empty. In order for the virtual table to be useful, you will need to -populate it with your vocabulary. Suppose you -have a list of words in a table named "big_vocabulary". Then do this: - -
    -INSERT INTO demo(word) SELECT word FROM big_vocabulary;
-
    - -If you intend to use this virtual table in cooperation with an FTS4 -table (for spelling correctly of search terms) then you might extract -the vocabulary using an fts3aux table: - -
    -INSERT INTO demo(word) SELECT term FROM search_aux WHERE col='*';
-
    - -You can also provide the virtual table with a "rank" for each word. -The "rank" is an estimate of how common the word is. Larger numbers -mean the word is more common. If you omit the rank when populating -the table, then a rank of 1 is assumed. But if you have rank -information, you can supply it and the virtual table will show a -slight preference for selecting more commonly used terms. To -populate the rank from an fts4aux table "search_aux" do something -like this: - -
    -INSERT INTO demo(word,rank)
-   SELECT term, documents FROM search_aux WHERE col='*';
-
    - -To query the virtual table, include a MATCH operator in the WHERE -clause. For example: - -
    -SELECT word FROM demo WHERE word MATCH 'kennasaw';
-
    - -Using a dataset of American place names (derived from -[http://geonames.usgs.gov/domestic/download_data.htm]) the query above -returns 20 results beginning with: - -
    -kennesaw
-kenosha
-kenesaw
-kenaga
-keanak
-
    - -If you append the character '*' to the end of the pattern, then -a prefix search is performed. For example: - -
    -SELECT word FROM demo WHERE word MATCH 'kennes*';
-
    - -Yields 20 results beginning with: - -
    -kennesaw
-kennestone
-kenneson
-kenneys
-keanes
-keenes
-
    - -

    Search Refinements

    - -By default, the spellfix1 table returns no more than 20 results. -(It might return less than 20 if there were fewer good matches.) -You can change the upper bound on the number of returned rows by -adding a "top=N" term to the WHERE clause of your query, where N -is the new maximum. For example, to see the 5 best matches: - -
    -SELECT word FROM demo WHERE word MATCH 'kennes*' AND top=5;
-
    - -Each entry in the spellfix1 virtual table is associated with a -a particular language, identified by the integer "langid" column. -The default langid is 0 and if no other actions are taken, the -entire vocabulary is a part of the 0 language. But if your application -needs to operate in multiple languages, then you can specify different -vocabulary items for each language by specifying the langid field -when populating the table. For example: - -
    -INSERT INTO demo(word,langid) SELECT word, 0 FROM en_vocabulary;
-INSERT INTO demo(word,langid) SELECT word, 1 FROM de_vocabulary;
-INSERT INTO demo(word,langid) SELECT word, 2 FROM fr_vocabulary;
-INSERT INTO demo(word,langid) SELECT word, 3 FROM ru_vocabulary;
-INSERT INTO demo(word,langid) SELECT word, 4 FROM cn_vocabulary;
-
    - -After the virtual table has been populated with items from multiple -languages, specify the language of interest using a "langid=N" term -in the WHERE clause of the query: - -
    -SELECT word FROM demo WHERE word MATCH 'hildes*' AND langid=1;
-
    - -Note that if you do not include the "langid=N" term in the WHERE clause, -the search will be against language 0 (English in the example above.) -All spellfix1 searches are against a single language id. There is no -way to search all languages at once. - - -

    Virtual Table Details

    - -The virtual table actually has a unique rowid with seven columns plus five -extra hidden columns. The columns are as follows: - -
    -
    rowid
    -A unique integer number associated with each -vocabulary item in the table. This can be used -as a foreign key on other tables in the database. - -
    word
    -The text of the word that matches the pattern. -Both word and pattern can contains unicode characters -and can be mixed case. - -
    rank
    -This is the rank of the word, as specified in the -original INSERT statement. - - -
    distance
    -This is an edit distance or Levensthein distance going -from the pattern to the word. - -
    langid
    -This is the language-id of the word. All queries are -against a single language-id, which defaults to 0. -For any given query this value is the same on all rows. - -
    score
    -The score is a combination of rank and distance. The -idea is that a lower score is better. The virtual table -attempts to find words with the lowest score and -by default (unless overridden by ORDER BY) returns -results in order of increasing score. - -
    matchlen
    -In a prefix search, the matchlen is the number of characters in -the string that match against the prefix. For a non-prefix search, -this is the same as length(word). - -
    phonehash
    -This column shows the phonetic hash prefix that was used to restrict -the search. For any given query, this column should be the same for -every row. This information is available for diagnostic purposes and -is not normally considered useful in real applications. - -
    top
    -(HIDDEN) For any query, this value is the same on all -rows. It is an integer which is the maximum number of -rows that will be output. The actually number of rows -output might be less than this number, but it will never -be greater. The default value for top is 20, but that -can be changed for each query by including a term of -the form "top=N" in the WHERE clause of the query. - -
    scope
    -(HIDDEN) For any query, this value is the same on all -rows. The scope is a measure of how widely the virtual -table looks for matching words. Smaller values of -scope cause a broader search. The scope is normally -choosen automatically and is capped at 4. Applications -can change the scope by including a term of the form -"scope=N" in the WHERE clause of the query. Increasing -the scope will make the query run faster, but will reduce -the possible corrections. - -
    srchcnt
    -(HIDDEN) For any query, this value is the same on all -rows. This value is an integer which is the number of -of words examined using the edit-distance algorithm to -find the top matches that are ultimately displayed. This -value is for diagnostic use only. - -
    soundslike
    -(HIDDEN) When inserting vocabulary entries, this field -can be set to an spelling that matches what the word -sounds like. See the DEALING WITH UNUSUAL AND DIFFICULT -SPELLINGS section below for details. - -
    command
    -(HIDDEN) The value of the "command" column is always NULL. However, -applications can insert special strings into the "command" column in order -to provoke certain behaviors in the spellfix1 virtual table. -For example, inserting the string 'reset' into the "command" column -will cause the virtual table will reread its edit distance weights -(if there are any). -
    - -

    Algorithm

    - -The spellfix1 virtual table creates a single -shadow table named "%_vocab" (where the % is replaced by the name of -the virtual table; Ex: "demo_vocab" for the "demo" virtual table). -the shadow table contains the following columns: - -
    -
    id
    -The unique id (INTEGER PRIMARY KEY) - -
    rank
    -The rank of word. - -
    langid
    -The language id for this entry. - -
    word
    -The original UTF8 text of the vocabulary word - -
    k1
    -The word transliterated into lower-case ASCII. -There is a standard table of mappings from non-ASCII -characters into ASCII. Examples: "æ" -> "ae", -"þ" -> "th", "ß" -> "ss", "á" -> "a", ... The -accessory function spellfix1_translit(X) will do -the non-ASCII to ASCII mapping. The built-in lower(X) -function will convert to lower-case. Thus: -k1 = lower(spellfix1_translit(word)). - -
    k2
    -This field holds a phonetic code derived from k1. Letters -that have similar sounds are mapped into the same symbol. -For example, all vowels and vowel clusters become the -single symbol "A". And the letters "p", "b", "f", and -"v" all become "B". All nasal sounds are represented -as "N". And so forth. The mapping is base on -ideas found in Soundex, Metaphone, and other -long-standing phonetic matching systems. This key can -be generated by the function spellfix1_phonehash(X). -Hence: k2 = spellfix1_phonehash(k1) -
    - -There is also a function for computing the Wagner edit distance or the -Levenshtein distance between a pattern and a word. This function -is exposed as spellfix1_editdist(X,Y). The edit distance function -returns the "cost" of converting X into Y. Some transformations -cost more than others. Changing one vowel into a different vowel, -for example is relatively cheap, as is doubling a constant, or -omitting the second character of a double-constant. Other transformations -or more expensive. The idea is that the edit distance function returns -a low cost of words that are similar and a higher cost for words -that are futher apart. In this implementation, the maximum cost -of any single-character edit (delete, insert, or substitute) is 100, -with lower costs for some edits (such as transforming vowels). - -The "score" for a comparison is the edit distance between the pattern -and the word, adjusted down by the base-2 logorithm of the word rank. -For example, a match with distance 100 but rank 1000 would have a -score of 122 (= 100 - log2(1000) + 32) where as a match with distance -100 with a rank of 1 would have a score of 131 (100 - log2(1) + 32). -(NB: The constant 32 is added to each score to keep it from going -negative in case the edit distance is zero.) In this way, frequently -used words get a slightly lower cost which tends to move them toward -the top of the list of alternative spellings. - -A straightforward implementation of a spelling corrector would be -to compare the search term against every word in the vocabulary -and select the 20 with the lowest scores. However, there will -typically be hundreds of thousands or millions of words in the -vocabulary, and so this approach is not fast enough. - -Suppose the term that is being spell-corrected is X. To limit -the search space, X is converted to a k2-like key using the -equivalent of: - -
    -   key = spellfix1_phonehash(lower(spellfix1_translit(X)))
-
    - -This key is then limited to "scope" characters. The default scope -value is 4, but an alternative scope can be specified using the -"scope=N" term in the WHERE clause. After the key has been truncated, -the edit distance is run against every term in the vocabulary that -has a k2 value that begins with the abbreviated key. - -For example, suppose the input word is "Paskagula". The phonetic -key is "BACACALA" which is then truncated to 4 characters "BACA". -The edit distance is then run on the 4980 entries (out of -272,597 entries total) of the vocabulary whose k2 values begin with -BACA, yielding "Pascagoula" as the best match. - -Only terms of the vocabulary with a matching langid are searched. -Hence, the same table can contain entries from multiple languages -and only the requested language will be used. The default langid -is 0. - -

    Configurable Edit Distance

    - -The built-in Wagner edit-distance function with fixed weights can be -replaced by the [./editdist3.wiki | editdist3()] edit-distance function -with application-defined weights and support for unicode, by specifying -the "edit_cost_table=TABLENAME" parameter to the spellfix1 module -when the virtual table is created. -For example: - -
    -CREATE VIRTUAL TABLE demo2 USING spellfix1(edit_cost_table=APPCOST);
-
    - -In the example above, the APPCOST table would be interrogated to find -the edit distance coefficients. It is the presence of the "edit_cost_table=" -parameter to the spellfix1 module name that causes editdist3() to be used -in place of the built-in edit distance function. - -The edit distance coefficients are normally read from the APPCOST table -once and there after stored in memory. Hence, run-time changes to the -APPCOST table will not normally effect the edit distance results. -However, inserting the special string 'reset' into the "command" column of the -virtual table causes the edit distance coefficients to be reread the -APPCOST table. Hence, applications should run a SQL statement similar -to the following when changes to the APPCOST table occur: - -
    -INSERT INTO demo2(command) VALUES('reset'); -
    - -The tables used for edit distance costs can be changed using a command -like the following: - -
    -INSERT INTO demo2(command) VALUES('edit_cost_table=APPCOST2'); -
    - -In the example above, any prior edit distance costs would be discarded and -all future queries would use the costs found in the APPCOST2 table. If the -name of the table specified by the "edit_cost_table" command is "NULL", then -theh built-in Wagner edit-distance function will be used instead of the -editdist3() function in all future queries. - -

    Dealing With Unusual And Difficult Spellings

    - -The algorithm above works quite well for most cases, but there are -exceptions. These exceptions can be dealt with by making additional -entries in the virtual table using the "soundslike" column. - -For example, many words of Greek origin begin with letters "ps" where -the "p" is silent. Ex: psalm, pseudonym, psoriasis, psyche. In -another example, many Scottish surnames can be spelled with an -initial "Mac" or "Mc". Thus, "MacKay" and "McKay" are both pronounced -the same. - -Accommodation can be made for words that are not spelled as they -sound by making additional entries into the virtual table for the -same word, but adding an alternative spelling in the "soundslike" -column. For example, the canonical entry for "psalm" would be this: - -
    -  INSERT INTO demo(word) VALUES('psalm');
-
    - -To enhance the ability to correct the spelling of "salm" into -"psalm", make an addition entry like this: - -
    -  INSERT INTO demo(word,soundslike) VALUES('psalm','salm');
-
    - -It is ok to make multiple entries for the same word as long as -each entry has a different soundslike value. Note that if no -soundslike value is specified, the soundslike defaults to the word -itself. - -Listed below are some cases where it might make sense to add additional -soundslike entries. The specific entries will depend on the application -and the target language. - - * Silent "p" in words beginning with "ps": psalm, psyche - - * Silent "p" in words beginning with "pn": pneumonia, pneumatic - - * Silent "p" in words beginning with "pt": pterodactyl, ptolemaic - - * Silent "d" in words beginning with "dj": djinn, Djikarta - - * Silent "k" in words beginning with "kn": knight, Knuthson - - * Silent "g" in words beginning with "gn": gnarly, gnome, gnat - - * "Mac" versus "Mc" beginning Scottish surnames - - * "Tch" sounds in Slavic words: Tchaikovsky vs. Chaykovsky - - * The letter "j" pronounced like "h" in Spanish: LaJolla - - * Words beginning with "wr" versus "r": write vs. rite - - * Miscellanous problem words such as "debt", "tsetse", - "Nguyen", "Van Nuyes". - -

    Auxiliary Functions

    - -The source code module that implements the spellfix1 virtual table also -implements several SQL functions that might be useful to applications -that employ spellfix1 or for testing or diagnostic work while developing -applications that use spellfix1. The following auxiliary functions are -available: - -
    -
    editdist3(P,W)
    editdist2(P,W,L)
    editdist3(T)
    -These routines provide direct access to the version of the Wagner -edit-distance function that allows for application-defined weights -on edit operations. The first two forms of this function compare -pattern P against word W and return the edit distance. In the first -function, the langid is assumed to be 0 and in the second, the -langid is given by the L parameter. The third form of this function -reloads edit distance coefficience from the table named by T. - -
    spellfix1_editdist(P,W)
    -This routine provides access to the built-in Wagner edit-distance -function that uses default, fixed costs. The value returned is -the edit distance needed to transform W into P. - -
    spellfix1_phonehash(X)
    -This routine constructs a phonetic hash of the pure ascii input word X -and returns that hash. This routine is used internally by spellfix1 in -order to transform the K1 column of the shadow table into the K2 -column. - -
    spellfix1_scriptcode(X)
    -Given an input string X, this routine attempts to determin the dominant -script of that input and returns the ISO-15924 numeric code for that -script. The current implementation understands the following scripts: -
      -
    • 215 - Latin -
    • 220 - Cyrillic -
    • 200 - Greek -
    -Additional language codes might be added in future releases. - -
    spellfix1_translit(X)
    -This routine transliterates unicode text into pure ascii, returning -the pure ascii representation of the input text X. This is the function -that is used internally to transform vocabulary words into the K1 -column of the shadow table. - -
    diff --git a/manifest b/manifest index b47b06dfb4..16c565ef04 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Untested\sfix\sfor\sbuilding\son\sVxWorks. -D 2013-04-27T12:13:29.526 +C Remove\sspellfix\svirtual\stable\sdocumentation\sfrom\sthe\ssource\stree.\nReference\sthe\sseparate\sdocumentation\son\sthe\swebsite\sinstead. +D 2013-04-27T18:06:40.561 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -85,13 +85,11 @@ F ext/icu/icu.c eb9ae1d79046bd7871aa97ee6da51eb770134b5a F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37 F ext/misc/amatch.c 3369b2b544066e620d986f0085d039c77d1ef17f F ext/misc/closure.c fec0c8537c69843e0b7631d500a14c0527962cd6 -F ext/misc/editdist3.wiki 06100a0c558921a563cbc40e0d0151902b1eef6d F ext/misc/fuzzer.c fb64a15af978ae73fa9075b9b1dfbe82b8defc6f F ext/misc/ieee754.c 2565ce373d842977efe0922dc50b8a41b3289556 F ext/misc/nextchar.c 1131e2b36116ffc6fe6b2e3464bfdace27978b1e F ext/misc/regexp.c c25c65fe775f5d9801fb8573e36ebe73f2c0c2e0 -F ext/misc/spellfix.c e323eebb877d735bc64404c16a6d758ab17a0b7a -F ext/misc/spellfix1.wiki dd1830444c14cf0f54dd680cc044df2ace2e9d09 +F ext/misc/spellfix.c f9d24a2b2617cee143b7841b453e4e1fd8f189cc F ext/misc/wholenumber.c ce362368b9381ea48cbd951ade8df867eeeab014 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 757abea591d4ff67c0ff4e8f9776aeda86b18c14 @@ -1062,7 +1060,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 7a97226ffe174349e7113340f5354c4e44bd9738 -R 45f16684eb908e3d81d1d0d938ab9d11 +P f14d55cf358b0392d3b8cd61dc85f43a610a8edf +R 1d8237b86dffdd5327eb650dcb291120 U drh -Z 3089ee3240913ca0c76196e0917eb228 +Z 369cfa66c298e6ca3c71b08895115463 diff --git a/manifest.uuid b/manifest.uuid index 8005ea1784..edb252e518 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f14d55cf358b0392d3b8cd61dc85f43a610a8edf \ No newline at end of file +adcf78909ff9064b6e3c4dd15ccd3245c8cf270b \ No newline at end of file From 4eb9b721515b27a9197fcae481cd2961ab71edcd Mon Sep 17 00:00:00 2001 From: mistachkin Date: Mon, 29 Apr 2013 07:01:23 +0000 Subject: [PATCH 21/32] Update 'fuzzerfault' test for its new module loading command. Fix several test names in 'io.test' and make sure the database gets closed between tests. FossilOrigin-Name: e81e9ca11db09424dd310bbc91686a5daa618cb1 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- test/fuzzerfault.test | 8 ++++---- test/io.test | 2 ++ 4 files changed, 15 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 16c565ef04..3db186dfa5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sspellfix\svirtual\stable\sdocumentation\sfrom\sthe\ssource\stree.\nReference\sthe\sseparate\sdocumentation\son\sthe\swebsite\sinstead. -D 2013-04-27T18:06:40.561 +C Update\s'fuzzerfault'\stest\sfor\sits\snew\smodule\sloading\scommand.\s\sFix\sseveral\stest\snames\sin\s'io.test'\sand\smake\ssure\sthe\sdatabase\sgets\sclosed\sbetween\stests. +D 2013-04-29T07:01:23.073 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -534,7 +534,7 @@ F test/fuzz3.test aec64345184d1662bd30e6a17851ff659d596dc5 F test/fuzz_common.tcl a87dfbb88c2a6b08a38e9a070dabd129e617b45b F test/fuzz_malloc.test 328f70aaca63adf29b4c6f06505ed0cf57ca7c26 F test/fuzzer1.test 41bd5aa6ae0cf18d06342a4476e3cad98604ae48 -F test/fuzzerfault.test f96773292b82ce40f7e6614f04ab2166c4ed3c28 +F test/fuzzerfault.test 8792cd77fd5bce765b05d0c8e01b9edcf8af8536 F test/hook.test 45cb22b940c3cc0af616ba7430f666e245711a48 F test/icu.test 70df4faca133254c042d02ae342c0a141f2663f4 F test/in.test 5941096407d8c133b9eff15bd3e666624b6cbde3 @@ -569,7 +569,7 @@ F test/instr.test a34e1d46a9eefb098a7167ef0e730a4a3d82fba0 F test/intarray.test 066b7d7ac38d25bf96f87f1b017bfc687551cdd4 F test/interrupt.test dfe9a67a94b0b2d8f70545ba1a6cca10780d71cc F test/intpkey.test 7af30f6ae852d8d1c2b70e4bf1551946742e92d8 -F test/io.test b90105d295225a712e2e9cb685876101712058b5 +F test/io.test 0147ed5fdbce3286d6128f5f7e3b76ee8352d652 F test/ioerr.test 40bb2cfcab63fb6aa7424cd97812a84bc16b5fb8 F test/ioerr2.test 9d71166f8466eda510f1af6137bdabaa82b5408d F test/ioerr3.test d3cec5e1a11ad6d27527d0d38573fbff14c71bdd @@ -1060,7 +1060,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P f14d55cf358b0392d3b8cd61dc85f43a610a8edf -R 1d8237b86dffdd5327eb650dcb291120 -U drh -Z 369cfa66c298e6ca3c71b08895115463 +P adcf78909ff9064b6e3c4dd15ccd3245c8cf270b +R 263b9a6fce3f45179cbdd8291a5dbd19 +U mistachkin +Z c1a76ab097fbf09032c65307a2000f74 diff --git a/manifest.uuid b/manifest.uuid index edb252e518..d77da23add 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -adcf78909ff9064b6e3c4dd15ccd3245c8cf270b \ No newline at end of file +e81e9ca11db09424dd310bbc91686a5daa618cb1 \ No newline at end of file diff --git a/test/fuzzerfault.test b/test/fuzzerfault.test index 50450ede7c..6449612a66 100644 --- a/test/fuzzerfault.test +++ b/test/fuzzerfault.test @@ -30,7 +30,7 @@ do_test 1-pre1 { } {} do_faultsim_test 1 -prep { faultsim_restore_and_reopen - register_fuzzer_module db + load_static_extension db fuzzer } -body { execsql { CREATE VIRTUAL TABLE x1 USING fuzzer(x1_rules); @@ -43,7 +43,7 @@ do_faultsim_test 1 -prep { do_test 2-pre1 { faultsim_delete_and_reopen - register_fuzzer_module db + load_static_extension db fuzzer execsql { CREATE TABLE x2_rules(ruleset, cFrom, cTo, cost); INSERT INTO x2_rules VALUES(0, 'a', 'x', 1); @@ -56,7 +56,7 @@ do_test 2-pre1 { do_faultsim_test 2 -prep { faultsim_restore_and_reopen - register_fuzzer_module db + load_static_extension db fuzzer } -body { execsql { SELECT count(*) FROM x2 WHERE word MATCH 'abc'; @@ -78,7 +78,7 @@ do_test 3-pre1 { do_faultsim_test 3 -prep { faultsim_restore_and_reopen - register_fuzzer_module db + load_static_extension db fuzzer } -body { execsql { CREATE VIRTUAL TABLE x1 USING fuzzer(x1_rules); diff --git a/test/io.test b/test/io.test index f9583e0e33..43289567f1 100644 --- a/test/io.test +++ b/test/io.test @@ -16,6 +16,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +set ::testprefix io db close sqlite3_simulate_device @@ -628,6 +629,7 @@ foreach {tn sql} { # to be flushed. Which is a bug. hexio_write test.db [expr 1024 * 5] [string repeat 00 2048] do_execsql_test 6.2.$tn.3 { PRAGMA integrity_check } {ok} + db close } sqlite3_simulate_device -char {} -sectorsize 0 From f054396b38ae93ffb831896368b8226e29ae75d0 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 29 Apr 2013 09:17:42 +0000 Subject: [PATCH 22/32] Fix mmap1.test so that it passes on windows as well as unix. FossilOrigin-Name: 52417eac3ecaec2dbbde170334358f5ddbd32501 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/mmap1.test | 18 ++++++++++++------ 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 3db186dfa5..2c3620bed6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\s'fuzzerfault'\stest\sfor\sits\snew\smodule\sloading\scommand.\s\sFix\sseveral\stest\snames\sin\s'io.test'\sand\smake\ssure\sthe\sdatabase\sgets\sclosed\sbetween\stests. -D 2013-04-29T07:01:23.073 +C Fix\smmap1.test\sso\sthat\sit\spasses\son\swindows\sas\swell\sas\sunix. +D 2013-04-29T09:17:42.276 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -646,7 +646,7 @@ F test/misc5.test 528468b26d03303b1f047146e5eefc941b9069f5 F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test dd82ec9250b89178b96cd28b2aca70639d21e5b3 F test/misuse.test ba4fb5d1a6101d1c171ea38b3c613d0661c83054 -F test/mmap1.test 0b5802cf64acaa509ea889b3c708196cc6eb9d31 +F test/mmap1.test 8696aa1b0bd88961c2f16af2a3f7a69d701cea50 F test/mmap2.test a5ba639f90b5fc487400a49e158e14e465943e98 F test/multiplex.test e08cc7177bd6d85990ee1d71100bb6c684c02256 F test/multiplex2.test 580ca5817c7edbe4cc68fa150609c9473393003a @@ -1060,7 +1060,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P adcf78909ff9064b6e3c4dd15ccd3245c8cf270b -R 263b9a6fce3f45179cbdd8291a5dbd19 -U mistachkin -Z c1a76ab097fbf09032c65307a2000f74 +P e81e9ca11db09424dd310bbc91686a5daa618cb1 +R 0b5ca68b3c8f8712e6440b0981a1650e +U dan +Z 3af177977b6ccb1ab6aa9597c01ddcc6 diff --git a/manifest.uuid b/manifest.uuid index d77da23add..b4bcb2b27e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e81e9ca11db09424dd310bbc91686a5daa618cb1 \ No newline at end of file +52417eac3ecaec2dbbde170334358f5ddbd32501 \ No newline at end of file diff --git a/test/mmap1.test b/test/mmap1.test index 0d0e3fd7a4..09b6d95927 100644 --- a/test/mmap1.test +++ b/test/mmap1.test @@ -40,14 +40,20 @@ proc register_rblob_code {dbname seed} { }] } +# For cases 1.1 and 1.4, the number of pages read using xRead() is 4 on +# unix and 9 on windows. The difference is that windows only ever maps +# an integer number of OS pages (i.e. creates mappings that are a multiple +# of 4KB in size). Whereas on unix any sized mapping may be created. +# foreach {t mmap_size nRead c2init} { - 1.1 { PRAGMA mmap_size = 67108864 } 4 {PRAGMA mmap_size = 0} - 1.2 { PRAGMA mmap_size = 53248 } 150 {PRAGMA mmap_size = 0} - 1.3 { PRAGMA mmap_size = 0 } 344 {PRAGMA mmap_size = 0} - 1.4 { PRAGMA mmap_size = 67108864 } 4 {PRAGMA mmap_size = 67108864 } - 1.5 { PRAGMA mmap_size = 53248 } 150 {PRAGMA mmap_size = 67108864 } - 1.6 { PRAGMA mmap_size = 0 } 344 {PRAGMA mmap_size = 67108864 } + 1.1 { PRAGMA mmap_size = 67108864 } /[49]/ {PRAGMA mmap_size = 0} + 1.2 { PRAGMA mmap_size = 53248 } 150 {PRAGMA mmap_size = 0} + 1.3 { PRAGMA mmap_size = 0 } 344 {PRAGMA mmap_size = 0} + 1.4 { PRAGMA mmap_size = 67108864 } /[49]/ {PRAGMA mmap_size = 67108864 } + 1.5 { PRAGMA mmap_size = 53248 } 150 {PRAGMA mmap_size = 67108864 } + 1.6 { PRAGMA mmap_size = 0 } 344 {PRAGMA mmap_size = 67108864 } } { + do_multiclient_test tn { sql1 {PRAGMA page_size=1024} sql1 $mmap_size From 181f4f789d503b014bc1ec662fe0ef3383b03094 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 29 Apr 2013 17:12:06 +0000 Subject: [PATCH 23/32] Fix an off-by-one in the code for limiting the depth of FTS expression trees. FossilOrigin-Name: 72ac73189c3577740a77d2ea2fc7118391c0703f --- ext/fts3/fts3_expr.c | 2 +- manifest | 14 +++++++------- manifest.uuid | 2 +- test/fts3expr3.test | 14 +++++++++++--- 4 files changed, 20 insertions(+), 12 deletions(-) diff --git a/ext/fts3/fts3_expr.c b/ext/fts3/fts3_expr.c index b28ab68e4b..2cb0f94bcb 100644 --- a/ext/fts3/fts3_expr.c +++ b/ext/fts3/fts3_expr.c @@ -755,7 +755,7 @@ exprparse_out: static int fts3ExprCheckDepth(Fts3Expr *p, int nMaxDepth){ int rc = SQLITE_OK; if( p ){ - if( nMaxDepth==0 ){ + if( nMaxDepth<0 ){ rc = SQLITE_ERROR; }else{ rc = fts3ExprCheckDepth(p->pLeft, nMaxDepth-1); diff --git a/manifest b/manifest index 2c3620bed6..7a2fc313d4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\smmap1.test\sso\sthat\sit\spasses\son\swindows\sas\swell\sas\sunix. -D 2013-04-29T09:17:42.276 +C Fix\san\soff-by-one\sin\sthe\scode\sfor\slimiting\sthe\sdepth\sof\sFTS\sexpression\strees. +D 2013-04-29T17:12:06.416 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -59,7 +59,7 @@ F ext/fts3/fts3.c 784aadfb4c2a217c3eb1feaecac924989f29728f F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h 352c8a83ee4c6a14ced1759a39dd890ab947cbe0 F ext/fts3/fts3_aux.c b02632f6dd0e375ce97870206d914ea6d8df5ccd -F ext/fts3/fts3_expr.c 4021d21aadebb50499c482f6b7b718e4b93113e6 +F ext/fts3/fts3_expr.c 2449c3f31b87c3376b66999cdd10d23be0b0bd42 F ext/fts3/fts3_hash.c 8dd2d06b66c72c628c2732555a32bc0943114914 F ext/fts3/fts3_hash.h 39cf6874dc239d6b4e30479b1975fe5b22a3caaf F ext/fts3/fts3_icu.c e319e108661147bcca8dd511cd562f33a1ba81b5 @@ -499,7 +499,7 @@ F test/fts3drop.test 1b906e293d6773812587b3dc458cb9e8f3f0c297 F test/fts3e.test 1f6c6ac9cc8b772ca256e6b22aaeed50c9350851 F test/fts3expr.test 5e745b2b6348499d9ef8d59015de3182072c564c F test/fts3expr2.test 18da930352e5693eaa163a3eacf96233b7290d1a -F test/fts3expr3.test aed4e71aa40ac1dcba860d1c43e7e23aa812eb1c +F test/fts3expr3.test 1bfb762b53a794f990f3dffaae8bbea5736422f7 F test/fts3fault.test cb72dccb0a3b9f730f16c5240f3fcb9303eb1660 F test/fts3fault2.test 3198eef2804deea7cac8403e771d9cbcb752d887 F test/fts3first.test dbdedd20914c8d539aa3206c9b34a23775644641 @@ -1060,7 +1060,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P e81e9ca11db09424dd310bbc91686a5daa618cb1 -R 0b5ca68b3c8f8712e6440b0981a1650e +P 52417eac3ecaec2dbbde170334358f5ddbd32501 +R 9ce2a78bcb94198e11ae7a2e16b98558 U dan -Z 3af177977b6ccb1ab6aa9597c01ddcc6 +Z 54156ad010a8fff228c493eff10ae743 diff --git a/manifest.uuid b/manifest.uuid index b4bcb2b27e..39bbe3f113 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -52417eac3ecaec2dbbde170334358f5ddbd32501 \ No newline at end of file +72ac73189c3577740a77d2ea2fc7118391c0703f \ No newline at end of file diff --git a/test/fts3expr3.test b/test/fts3expr3.test index 51a6adf58e..e3cc2619c6 100644 --- a/test/fts3expr3.test +++ b/test/fts3expr3.test @@ -149,7 +149,7 @@ for {set i 1} {$i < 100} {incr i} { # Try trees of AND nodes with leaves that are themselves trees of OR nodes. # -for {set i 2} {$i < 32} {incr i} { +for {set i 2} {$i < 64} {incr i 4} { do_test 3.$i { test_fts3expr2 [random_andor_query $i] } [balanced_andor_tree $i] @@ -157,7 +157,7 @@ for {set i 2} {$i < 32} {incr i} { # These exceed the depth limit. # -for {set i 33} {$i < 40} {incr i} { +for {set i 65} {$i < 70} {incr i} { do_test 3.$i { list [catch {test_fts3expr2 [random_andor_query $i]} msg] $msg } {1 {Error parsing expression}} @@ -165,13 +165,21 @@ for {set i 33} {$i < 40} {incr i} { # This also exceeds the depth limit. # -do_test 4.1 { + +do_test 4.1.1 { set q "1" for {set i 2} {$i < 5000} {incr i} { append q " AND $i" } list [catch {test_fts3expr2 $q} msg] $msg } {1 {Error parsing expression}} +do_test 4.1.2 { + set q "1" + for {set i 2} {$i < 4000} {incr i} { + append q " AND $i" + } + catch {test_fts3expr2 $q} +} {0} proc create_toggle_tree {nDepth} { if {$nDepth == 0} { return xxx } From 3a01b600d3d17c098e474707eb0afda005b0fd96 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 29 Apr 2013 18:07:37 +0000 Subject: [PATCH 24/32] Improve the error message issued when an FTS query exceeds the maximum allowable tree depth. FossilOrigin-Name: f480b1fe6012f36c59cd0525efdc6df74143ccd0 --- ext/fts3/fts3.c | 8 +++----- ext/fts3/fts3Int.h | 2 +- ext/fts3/fts3_expr.c | 20 ++++++++++++++++---- manifest | 16 ++++++++-------- manifest.uuid | 2 +- 5 files changed, 29 insertions(+), 19 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 0c7e07740d..34d1e2acbd 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -2975,14 +2975,12 @@ static int fts3FilterMethod( pCsr->iLangid = 0; if( nVal==2 ) pCsr->iLangid = sqlite3_value_int(apVal[1]); + assert( p->base.zErrMsg==0 ); rc = sqlite3Fts3ExprParse(p->pTokenizer, pCsr->iLangid, - p->azColumn, p->bFts4, p->nColumn, iCol, zQuery, -1, &pCsr->pExpr + p->azColumn, p->bFts4, p->nColumn, iCol, zQuery, -1, &pCsr->pExpr, + &p->base.zErrMsg ); if( rc!=SQLITE_OK ){ - if( rc==SQLITE_ERROR ){ - static const char *zErr = "malformed MATCH expression: [%s]"; - p->base.zErrMsg = sqlite3_mprintf(zErr, zQuery); - } return rc; } diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index a11c18a3b7..1fbb9c7587 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -524,7 +524,7 @@ void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *, const char *); /* fts3_expr.c */ int sqlite3Fts3ExprParse(sqlite3_tokenizer *, int, - char **, int, int, int, const char *, int, Fts3Expr ** + char **, int, int, int, const char *, int, Fts3Expr **, char ** ); void sqlite3Fts3ExprFree(Fts3Expr *); #ifdef SQLITE_TEST diff --git a/ext/fts3/fts3_expr.c b/ext/fts3/fts3_expr.c index 2cb0f94bcb..3ea3f23c96 100644 --- a/ext/fts3/fts3_expr.c +++ b/ext/fts3/fts3_expr.c @@ -756,7 +756,7 @@ static int fts3ExprCheckDepth(Fts3Expr *p, int nMaxDepth){ int rc = SQLITE_OK; if( p ){ if( nMaxDepth<0 ){ - rc = SQLITE_ERROR; + rc = SQLITE_TOOBIG; }else{ rc = fts3ExprCheckDepth(p->pLeft, nMaxDepth-1); if( rc==SQLITE_OK ){ @@ -841,7 +841,7 @@ static int fts3ExprBalance(Fts3Expr **pp, int nMaxDepth){ } if( p ){ sqlite3Fts3ExprFree(p); - rc = SQLITE_ERROR; + rc = SQLITE_TOOBIG; break; } @@ -996,7 +996,8 @@ int sqlite3Fts3ExprParse( int nCol, /* Number of entries in azCol[] */ int iDefaultCol, /* Default column to query */ const char *z, int n, /* Text of MATCH query */ - Fts3Expr **ppExpr /* OUT: Parsed query structure */ + Fts3Expr **ppExpr, /* OUT: Parsed query structure */ + char **pzErr /* OUT: Error message (sqlite3_malloc) */ ){ static const int MAX_EXPR_DEPTH = 12; int rc = fts3ExprParseUnbalanced( @@ -1011,9 +1012,18 @@ int sqlite3Fts3ExprParse( rc = fts3ExprCheckDepth(*ppExpr, MAX_EXPR_DEPTH); } } + if( rc!=SQLITE_OK ){ sqlite3Fts3ExprFree(*ppExpr); *ppExpr = 0; + if( rc==SQLITE_TOOBIG ){ + *pzErr = sqlite3_mprintf( + "FTS expression tree is too large (maximum depth %d)", MAX_EXPR_DEPTH + ); + rc = SQLITE_ERROR; + }else if( rc==SQLITE_ERROR ){ + *pzErr = sqlite3_mprintf("malformed MATCH expression: [%s]", z); + } } return rc; @@ -1216,10 +1226,12 @@ static void fts3ExprTest( } if( sqlite3_user_data(context) ){ + char *zDummy = 0; rc = sqlite3Fts3ExprParse( - pTokenizer, 0, azCol, 0, nCol, nCol, zExpr, nExpr, &pExpr + pTokenizer, 0, azCol, 0, nCol, nCol, zExpr, nExpr, &pExpr, &zDummy ); assert( rc==SQLITE_OK || pExpr==0 ); + sqlite3_free(zDummy); }else{ rc = fts3ExprParseUnbalanced( pTokenizer, 0, azCol, 0, nCol, nCol, zExpr, nExpr, &pExpr diff --git a/manifest b/manifest index 7a2fc313d4..f3fe09157b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\soff-by-one\sin\sthe\scode\sfor\slimiting\sthe\sdepth\sof\sFTS\sexpression\strees. -D 2013-04-29T17:12:06.416 +C Improve\sthe\serror\smessage\sissued\swhen\san\sFTS\squery\sexceeds\sthe\smaximum\sallowable\stree\sdepth. +D 2013-04-29T18:07:37.241 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -55,11 +55,11 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers e0a8b81383ea60d0334d274fadf305ea14a8c314 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 784aadfb4c2a217c3eb1feaecac924989f29728f +F ext/fts3/fts3.c 5c3d44d16701cc4bc81ebf0bb9d5bff136d42de0 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h 352c8a83ee4c6a14ced1759a39dd890ab947cbe0 +F ext/fts3/fts3Int.h 23ea0a2bb7258d2539376ed60220cce28ba25765 F ext/fts3/fts3_aux.c b02632f6dd0e375ce97870206d914ea6d8df5ccd -F ext/fts3/fts3_expr.c 2449c3f31b87c3376b66999cdd10d23be0b0bd42 +F ext/fts3/fts3_expr.c 44b4a3c4983ddbf1958c4a40468efb4ff2e0549a F ext/fts3/fts3_hash.c 8dd2d06b66c72c628c2732555a32bc0943114914 F ext/fts3/fts3_hash.h 39cf6874dc239d6b4e30479b1975fe5b22a3caaf F ext/fts3/fts3_icu.c e319e108661147bcca8dd511cd562f33a1ba81b5 @@ -1060,7 +1060,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 52417eac3ecaec2dbbde170334358f5ddbd32501 -R 9ce2a78bcb94198e11ae7a2e16b98558 +P 72ac73189c3577740a77d2ea2fc7118391c0703f +R 845f7187fab8dee39261b18defdba512 U dan -Z 54156ad010a8fff228c493eff10ae743 +Z 0f3ad9cc69769334523e8cc24ad50e47 diff --git a/manifest.uuid b/manifest.uuid index 39bbe3f113..eb0b27c97e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -72ac73189c3577740a77d2ea2fc7118391c0703f \ No newline at end of file +f480b1fe6012f36c59cd0525efdc6df74143ccd0 \ No newline at end of file From 3c449c6b6b04e49b74a12e9eb47f5799b06bfaee Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 30 Apr 2013 14:06:57 +0000 Subject: [PATCH 25/32] Make sure extra parentheses around subqueries in the FROM clause are harmless. Ticket [28c6e830f239ea5]. FossilOrigin-Name: 1c79569226db3d5a73e65a35877635ea8b478866 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/parse.y | 2 ++ test/where8.test | 16 +++++++++++++++- 4 files changed, 26 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index f3fe09157b..46a1307582 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improve\sthe\serror\smessage\sissued\swhen\san\sFTS\squery\sexceeds\sthe\smaximum\sallowable\stree\sdepth. -D 2013-04-29T18:07:37.241 +C Make\ssure\sextra\sparentheses\saround\ssubqueries\sin\sthe\sFROM\sclause\sare\sharmless.\nTicket\s[28c6e830f239ea5]. +D 2013-04-30T14:06:57.299 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -179,7 +179,7 @@ F src/os_unix.c 5a214c5431fd005dbb3b8bbaaa306433e8e9b48f F src/os_win.c 673b3e3d1fa3040d8d95a7f1f5e0e553aed56cfb F src/pager.c 4a9160d268977e56ae2df90182050ab30fca715d F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 -F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 +F src/parse.y 9708365594eea519cdc8504dee425c0a41c79502 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 9fd22671c270b35131ef480bbc00392b8b5f8ab9 @@ -1003,7 +1003,7 @@ F test/where4.test e9b9e2f2f98f00379e6031db6a6fca29bae782a2 F test/where5.test fdf66f96d29a064b63eb543e28da4dfdccd81ad2 F test/where6.test 5da5a98cec820d488e82708301b96cb8c18a258b F test/where7.test 5c566388f0cc318b0032ce860f4ac5548e3c265a -F test/where8.test 02619a9bfc6df7b19979a02852bac09c3c99477a +F test/where8.test d9f889e62dccddb9d790b0c84dfc7861e03a162c F test/where8m.test da346596e19d54f0aba35ebade032a7c47d79739 F test/where9.test 0157862ccf0cfdf1a4622cdf697e5e2f09a8de44 F test/whereA.test 24c234263c8fe358f079d5e57d884fb569d2da0a @@ -1060,7 +1060,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 72ac73189c3577740a77d2ea2fc7118391c0703f -R 845f7187fab8dee39261b18defdba512 -U dan -Z 0f3ad9cc69769334523e8cc24ad50e47 +P f480b1fe6012f36c59cd0525efdc6df74143ccd0 +R 741140f0e346ccf75acdb75c67b0b716 +U drh +Z 1f58c2b83ff706000e8e5b26ed1b3eec diff --git a/manifest.uuid b/manifest.uuid index eb0b27c97e..b2d3fed836 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f480b1fe6012f36c59cd0525efdc6df74143ccd0 \ No newline at end of file +1c79569226db3d5a73e65a35877635ea8b478866 \ No newline at end of file diff --git a/src/parse.y b/src/parse.y index abc0e7dc08..8310b26989 100644 --- a/src/parse.y +++ b/src/parse.y @@ -520,7 +520,9 @@ seltablist(A) ::= stl_prefix(X) nm(Y) dbnm(D) as(Z) indexed_opt(I) struct SrcList_item *pOld = F->a; pNew->zName = pOld->zName; pNew->zDatabase = pOld->zDatabase; + pNew->pSelect = pOld->pSelect; pOld->zName = pOld->zDatabase = 0; + pOld->pSelect = 0; } sqlite3SrcListDelete(pParse->db, F); }else{ diff --git a/test/where8.test b/test/where8.test index 3bf1790132..ae2d04a75d 100644 --- a/test/where8.test +++ b/test/where8.test @@ -296,13 +296,27 @@ do_test where8-3.21 { SELECT a, d FROM t1, (t2) WHERE (a=d OR b=e) AND a<5 ORDER BY a } } {1 1 2 2 3 3 4 2 4 4 0 0} +do_test where8-3.21.1 { + execsql_status { + SELECT a, d FROM t1, ((t2)) AS t3 WHERE (a=d OR b=e) AND a<5 ORDER BY a + } +} {1 1 2 2 3 3 4 2 4 4 0 0} +do_test where8-3.21.2 { + execsql_status { + SELECT a, d FROM t1, ((SELECT * FROM t2)) AS t3 WHERE (a=d OR b=e) AND a<5 ORDER BY a + } +} {1 1 2 2 3 3 4 2 4 4 0 0} do_test where8-3.22 { execsql_status { SELECT a, d FROM ((((((t1))), (((t2)))))) WHERE (a=d OR b=e) AND a<5 ORDER BY a } } {1 1 2 2 3 3 4 2 4 4 0 0} - +do_test where8-3.23 { + execsql_status { + SELECT * FROM ((SELECT * FROM t2)) AS t3; + } +} {1 {} I 2 four IV 3 {} IX 4 sixteen XVI 5 {} XXV 6 thirtysix XXXVI 7 fortynine XLIX 8 sixtyeight LXIV 9 eightyone LXXXIX 10 {} C 9 0} #----------------------------------------------------------------------- # The following tests - where8-4.* - verify that adding or removing From 9a1eccb621ad48bc606019a1fb18220069af38af Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 30 Apr 2013 14:25:32 +0000 Subject: [PATCH 26/32] Update the documentation to explain that when the 3rd parameter to sqlite3_bind_text() and friends is NULL the result is to bind a NULL SQL value. Ticket [19b44e35753ba] FossilOrigin-Name: bd92de0e8d922b96513c5d431493800dda7e7562 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/sqlite.h.in | 3 +++ 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 46a1307582..08e82e4973 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\ssure\sextra\sparentheses\saround\ssubqueries\sin\sthe\sFROM\sclause\sare\sharmless.\nTicket\s[28c6e830f239ea5]. -D 2013-04-30T14:06:57.299 +C Update\sthe\sdocumentation\sto\sexplain\sthat\swhen\sthe\s3rd\sparameter\sto\nsqlite3_bind_text()\sand\sfriends\sis\sNULL\sthe\sresult\sis\sto\sbind\sa\sNULL\nSQL\svalue.\s\sTicket\s[19b44e35753ba] +D 2013-04-30T14:25:32.967 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -191,7 +191,7 @@ F src/resolve.c 83cc2d942ee216bc56956c6e6fadb691c1727fa1 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 6bfbe11e2fef81c5e18d30513ab6c69f171667eb F src/shell.c 5d527e5d08f05ec2c43ff194ea44bf62b974f4c9 -F src/sqlite.h.in e6c241170da854abf6f15ec7e6c565348b690169 +F src/sqlite.h.in 5a5a22a9b192d81a9e5dee00274e3a0484c4afb1 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h d936f797812c28b81b26ed18345baf8db28a21a5 F src/sqliteInt.h de835c584032769461c123a564381f9808542c0e @@ -1060,7 +1060,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P f480b1fe6012f36c59cd0525efdc6df74143ccd0 -R 741140f0e346ccf75acdb75c67b0b716 +P 1c79569226db3d5a73e65a35877635ea8b478866 +R d47a972814ae8bb4214b1f841a420de6 U drh -Z 1f58c2b83ff706000e8e5b26ed1b3eec +Z 88df79255fe6b8c77c4c16bff1d52cd7 diff --git a/manifest.uuid b/manifest.uuid index b2d3fed836..c280f104a7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1c79569226db3d5a73e65a35877635ea8b478866 \ No newline at end of file +bd92de0e8d922b96513c5d431493800dda7e7562 \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 5eb74a8756..6608823175 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -3270,6 +3270,9 @@ typedef struct sqlite3_context sqlite3_context; ** parameter [SQLITE_LIMIT_VARIABLE_NUMBER] (default value: 999). ** ** ^The third argument is the value to bind to the parameter. +** ^If the third parameter to sqlite3_bind_text() or sqlite3_bind_text16() +** or sqlite3_bind_blob() is a NULL pointer then the fourth parameter +** is ignored and the end result is the same as sqlite3_bind_null(). ** ** ^(In those routines that have a fourth argument, its value is the ** number of bytes in the parameter. To be clear: the value is the From 5c10f3b38e3930ec9937f8867d12fe431e3de2bc Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 1 May 2013 17:22:38 +0000 Subject: [PATCH 27/32] Avoid redundant constraint checking due to transitive constraints. FossilOrigin-Name: 329478cbed06f93652de50abdb31a6b41af02b9e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 08e82e4973..7f57ed1891 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sthe\sdocumentation\sto\sexplain\sthat\swhen\sthe\s3rd\sparameter\sto\nsqlite3_bind_text()\sand\sfriends\sis\sNULL\sthe\sresult\sis\sto\sbind\sa\sNULL\nSQL\svalue.\s\sTicket\s[19b44e35753ba] -D 2013-04-30T14:25:32.967 +C Avoid\sredundant\sconstraint\schecking\sdue\sto\stransitive\sconstraints. +D 2013-05-01T17:22:38.014 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -263,7 +263,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h a4d3da523d55a226a0b28e9058ef88d0a8051887 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c d54e63087b52c309550aa2defdb20ef27add9f9a +F src/where.c e36bbfbf565c7a2ce49c5cd3101ca7310cd379af F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1060,7 +1060,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 1c79569226db3d5a73e65a35877635ea8b478866 -R d47a972814ae8bb4214b1f841a420de6 +P bd92de0e8d922b96513c5d431493800dda7e7562 +R 98be73d126ba545d3f3d37ca997eaf1d U drh -Z 88df79255fe6b8c77c4c16bff1d52cd7 +Z df56d0b87948c58bd48a65ece4cb954b diff --git a/manifest.uuid b/manifest.uuid index c280f104a7..d118513ae8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bd92de0e8d922b96513c5d431493800dda7e7562 \ No newline at end of file +329478cbed06f93652de50abdb31a6b41af02b9e \ No newline at end of file diff --git a/src/where.c b/src/where.c index d70205205c..62f9af6760 100644 --- a/src/where.c +++ b/src/where.c @@ -4883,6 +4883,7 @@ static Bitmask codeOneLoopStart( assert( (pTerm->prereqRight & newNotReady)!=0 ); pAlt = findTerm(pWC, iCur, pTerm->u.leftColumn, notReady, WO_EQ|WO_IN, 0); if( pAlt==0 ) continue; + if( pAlt->wtFlags & (TERM_CODED) ) continue; VdbeNoopComment((v, "begin transitive constraint")); sEq = *pAlt->pExpr; sEq.pLeft = pE->pLeft; From a309552e2cc3aa3de14c7fd5a6110375d4efb43d Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 1 May 2013 17:58:35 +0000 Subject: [PATCH 28/32] Do not use a transitive constraint to an IN operator where the RHS is a constant if there exists a direct == operator to another table in an outer loop. FossilOrigin-Name: faedaeace9c7ed9a8aaf96700caee09db0c0c061 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 7f57ed1891..a5bd0b436b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sredundant\sconstraint\schecking\sdue\sto\stransitive\sconstraints. -D 2013-05-01T17:22:38.014 +C Do\snot\suse\sa\stransitive\sconstraint\sto\san\sIN\soperator\swhere\sthe\sRHS\sis\sa\nconstant\sif\sthere\sexists\sa\sdirect\s==\soperator\sto\sanother\stable\sin\san\souter\nloop. +D 2013-05-01T17:58:35.871 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -263,7 +263,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h a4d3da523d55a226a0b28e9058ef88d0a8051887 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c e36bbfbf565c7a2ce49c5cd3101ca7310cd379af +F src/where.c 12d4200eb6ae991cad02367c391db076ac1af1b0 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1060,7 +1060,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P bd92de0e8d922b96513c5d431493800dda7e7562 -R 98be73d126ba545d3f3d37ca997eaf1d +P 329478cbed06f93652de50abdb31a6b41af02b9e +R cc457983507eee39e1b071292dee5b26 U drh -Z df56d0b87948c58bd48a65ece4cb954b +Z 944723cfd5dfa8d16765eaab2fd82886 diff --git a/manifest.uuid b/manifest.uuid index d118513ae8..f88e8ae7a8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -329478cbed06f93652de50abdb31a6b41af02b9e \ No newline at end of file +faedaeace9c7ed9a8aaf96700caee09db0c0c061 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 62f9af6760..2de894c3e7 100644 --- a/src/where.c +++ b/src/where.c @@ -705,7 +705,7 @@ static WhereTerm *findTerm( continue; } } - if( pTerm->prereqRight==0 ){ + if( pTerm->prereqRight==0 && (pTerm->eOperator&WO_EQ)!=0 ){ pResult = pTerm; goto findTerm_success; }else if( pResult==0 ){ From 4ee09b4bcc491037416597306a53b8508e4dbec3 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 1 May 2013 19:49:27 +0000 Subject: [PATCH 29/32] Allocate 4 bytes of unused header space for an "Application ID". Add the "PRAGMA application_id" command to set and query this identifier. Add the "magic.txt" file to show how the posix file command might use this application id. FossilOrigin-Name: 28c9e7fdee2471a3026ee05ff591194d5f398131 --- magic.txt | 28 ++++++++++++++++++++++++++++ manifest | 22 +++++++++++++--------- manifest.uuid | 2 +- src/btree.h | 1 + src/pragma.c | 9 +++++++++ test/pragma.test | 10 ++++++++++ tool/showdb.c | 2 +- 7 files changed, 63 insertions(+), 11 deletions(-) create mode 100644 magic.txt diff --git a/magic.txt b/magic.txt new file mode 100644 index 0000000000..22bdae5712 --- /dev/null +++ b/magic.txt @@ -0,0 +1,28 @@ +# This file contains suggested magic(5) text for the unix file(1) +# utility for recognizing SQLite3 databases. +# +# When SQLite is used as an application file format, it is desirable to +# have file(1) recognize the database file as being with the specific +# application. You can set the application_id for a database file +# using: +# +# PRAGMA application_id = INTEGER; +# +# INTEGER can be any signed 32-bit integer. That integer is written as +# a 4-byte big-endian integer into offset 68 of the database header. In +# the tests below, this integer is sometimes rendered as a string. For +# example, instead of "belong =1598444364" we write "string =_FSL" and +# instead of "belong =1598903374" we write "string =_MTN". +# +# The Monotone application used "PRAGMA user_version=1598903374;" to set +# its identifier long before "PRAGMA application_id" became available. +# The user_version is very similar to application_id except that it is +# stored at offset 60 instead of offset 68. The user of +# "PRAGMA application_id" is preferred now. The rules using offset 60 +# for Monotone are for historical compatibility only. +# +0 string =SQLite\ format\ 3 +>68 string =_FSL Fossil repository +>68 belong =0 +>>60 string =_MTN Monotone source repository +>>60 string !_MTN SQLite 3.x database diff --git a/manifest b/manifest index a5bd0b436b..706371cb24 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\suse\sa\stransitive\sconstraint\sto\san\sIN\soperator\swhere\sthe\sRHS\sis\sa\nconstant\sif\sthere\sexists\sa\sdirect\s==\soperator\sto\sanother\stable\sin\san\souter\nloop. -D 2013-05-01T17:58:35.871 +C Allocate\s4\sbytes\sof\sunused\sheader\sspace\sfor\san\s"Application\sID".\s\sAdd\nthe\s"PRAGMA\sapplication_id"\scommand\sto\sset\sand\squery\sthis\sidentifier.\nAdd\sthe\s"magic.txt"\sfile\sto\sshow\show\sthe\sposix\sfile\scommand\smight\suse\nthis\sapplication\sid. +D 2013-05-01T19:49:27.740 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -112,6 +112,7 @@ F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 +F magic.txt 4255db3a17f0da4b3f561a904585a6605fe6a6b8 F main.mk 1b25be82452366abc27cc9ab2acf3244a773d5a1 F mkdll.sh 7d09b23c05d56532e9d44a50868eb4b12ff4f74a F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f @@ -137,7 +138,7 @@ F src/backup.c b266767351ae2d847716c56fcb2a1fea7c761c03 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c 480a6d255cc4f066029daf23dd54acf152cd0e13 -F src/btree.h d9490cd37aaeb530a41b07f06e1262950b1be916 +F src/btree.h 6fa8a3ff2483d0bb64a9f0105a8cedeac9e00cca F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 F src/build.c 083da8466fd7e481cb8bd5264398f537507f6176 F src/callback.c d7e46f40c3cf53c43550b7da7a1d0479910b62cc @@ -183,7 +184,7 @@ F src/parse.y 9708365594eea519cdc8504dee425c0a41c79502 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 9fd22671c270b35131ef480bbc00392b8b5f8ab9 -F src/pragma.c 3eacf001cbf4becbd494f8d82d08fdf1648cf8cb +F src/pragma.c 8779308bc1ea1901c4bc94dfe9a83d436f73f52c F src/prepare.c 743e484233c51109666d402f470523553b41797c F src/printf.c 4a9f882f1c1787a8b494a2987765acf9d97ac21f F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 @@ -677,7 +678,7 @@ F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16 F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 F test/permutations.test 3d0bab9c49c1ec08b868059e30a3e1956f2162e2 -F test/pragma.test 60d29cd3d8098a2c20bf4c072810f99e3bf2757a +F test/pragma.test 5e7de6c32a5d764f09437d2025f07e4917b9e178 F test/pragma2.test 3a55f82b954242c642f8342b17dffc8b47472947 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 F test/progress.test 5b075c3c790c7b2a61419bc199db87aaf48b8301 @@ -1040,7 +1041,7 @@ F tool/omittest.tcl 4665982e95a6e5c1bd806cf7bc3dea95be422d77 F tool/opcodeDoc.awk b3a2a3d5d3075b8bd90b7afe24283efdd586659c F tool/restore_jrnl.tcl 6957a34f8f1f0f8285e07536225ec3b292a9024a F tool/rollback-test.c 9fc98427d1e23e84429d7e6d07d9094fbdec65a5 -F tool/showdb.c acd24ea035a3bd2ffe266f1ef8a161812c29b2f0 +F tool/showdb.c 525ecc443578647703051308ad50a93de6ba2c4b F tool/showjournal.c b62cecaab86a4053d944c276bb5232e4d17ece02 F tool/showwal.c 3f7f7da5ec0cba51b1449a75f700493377da57b5 F tool/soak1.tcl 8d407956e1a45b485a8e072470a3e629a27037fe @@ -1060,7 +1061,10 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 329478cbed06f93652de50abdb31a6b41af02b9e -R cc457983507eee39e1b071292dee5b26 +P faedaeace9c7ed9a8aaf96700caee09db0c0c061 +R 4f1b06c15d2f5747c86dc6db3cebcf91 +T *branch * application-id +T *sym-application-id * +T -sym-trunk * U drh -Z 944723cfd5dfa8d16765eaab2fd82886 +Z 76bc7425f1509081256079e29083992d diff --git a/manifest.uuid b/manifest.uuid index f88e8ae7a8..1b206e6fa8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -faedaeace9c7ed9a8aaf96700caee09db0c0c061 \ No newline at end of file +28c9e7fdee2471a3026ee05ff591194d5f398131 \ No newline at end of file diff --git a/src/btree.h b/src/btree.h index 23f6ad7068..ace0f8cd21 100644 --- a/src/btree.h +++ b/src/btree.h @@ -140,6 +140,7 @@ int sqlite3BtreeNewDb(Btree *p); #define BTREE_TEXT_ENCODING 5 #define BTREE_USER_VERSION 6 #define BTREE_INCR_VACUUM 7 +#define BTREE_APPLICATION_ID 8 /* ** Values that may be OR'd together to form the second argument of an diff --git a/src/pragma.c b/src/pragma.c index 65efbd8b21..3056a7d8e2 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1567,6 +1567,11 @@ void sqlite3Pragma( ** PRAGMA [database.]user_version ** PRAGMA [database.]user_version = ** + ** PRAGMA [database.]freelist_count = + ** + ** PRAGMA [database.]application_id + ** PRAGMA [database.]application_id = + ** ** The pragma's schema_version and user_version are used to set or get ** the value of the schema-version and user-version, respectively. Both ** the schema-version and the user-version are 32-bit signed integers @@ -1588,10 +1593,14 @@ void sqlite3Pragma( if( sqlite3StrICmp(zLeft, "schema_version")==0 || sqlite3StrICmp(zLeft, "user_version")==0 || sqlite3StrICmp(zLeft, "freelist_count")==0 + || sqlite3StrICmp(zLeft, "application_id")==0 ){ int iCookie; /* Cookie index. 1 for schema-cookie, 6 for user-cookie. */ sqlite3VdbeUsesBtree(v, iDb); switch( zLeft[0] ){ + case 'a': case 'A': + iCookie = BTREE_APPLICATION_ID; + break; case 'f': case 'F': iCookie = BTREE_FREE_PAGE_COUNT; break; diff --git a/test/pragma.test b/test/pragma.test index 3eb624328e..a6d198eb69 100644 --- a/test/pragma.test +++ b/test/pragma.test @@ -936,6 +936,16 @@ proc check_temp_store {} { return "unknown" } +# Application_ID +# +do_test pragma-8.3.1 { + execsql { + PRAGMA application_id; + } +} {0} +do_test pragma-8.3.2 { + execsql {PRAGMA Application_ID(12345); PRAGMA application_id;} +} {12345} # Test temp_store and temp_store_directory pragmas # diff --git a/tool/showdb.c b/tool/showdb.c index dbd79e9586..27424e0920 100644 --- a/tool/showdb.c +++ b/tool/showdb.c @@ -176,7 +176,7 @@ static void print_db_header(void){ print_decode_line(aData, 56, 4, "Text encoding"); print_decode_line(aData, 60, 4, "User version"); print_decode_line(aData, 64, 4, "Incremental-vacuum mode"); - print_decode_line(aData, 68, 4, "meta[7]"); + print_decode_line(aData, 68, 4, "Application ID"); print_decode_line(aData, 72, 4, "meta[8]"); print_decode_line(aData, 76, 4, "meta[9]"); print_decode_line(aData, 80, 4, "meta[10]"); From b8a67ec827b5a5987f447d68d63648bd2045b3cd Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 1 May 2013 20:36:23 +0000 Subject: [PATCH 30/32] Preserve the application-ID across VACUUM. Updates to the magic number file. FossilOrigin-Name: 4a190bea18e156b6fa4dc9f21c3ad32409049603 --- magic.txt | 11 ++++++----- manifest | 17 +++++++---------- manifest.uuid | 2 +- src/vacuum.c | 1 + 4 files changed, 15 insertions(+), 16 deletions(-) diff --git a/magic.txt b/magic.txt index 22bdae5712..90e2dc50a0 100644 --- a/magic.txt +++ b/magic.txt @@ -17,12 +17,13 @@ # The Monotone application used "PRAGMA user_version=1598903374;" to set # its identifier long before "PRAGMA application_id" became available. # The user_version is very similar to application_id except that it is -# stored at offset 60 instead of offset 68. The user of +# stored at offset 68 instead of offset 60. The user of # "PRAGMA application_id" is preferred now. The rules using offset 60 # for Monotone are for historical compatibility only. # 0 string =SQLite\ format\ 3 ->68 string =_FSL Fossil repository ->68 belong =0 ->>60 string =_MTN Monotone source repository ->>60 string !_MTN SQLite 3.x database +>68 belong =0x0f055111 Fossil repository - +>68 belong =0x0f055112 Fossil checkout - +>68 belong =0x0f055113 Fossil global configuration - +>60 belong =0x5f4d544e Monotone source repository - +>0 string =SQLite SQLite3 database diff --git a/manifest b/manifest index 706371cb24..06e860720a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Allocate\s4\sbytes\sof\sunused\sheader\sspace\sfor\san\s"Application\sID".\s\sAdd\nthe\s"PRAGMA\sapplication_id"\scommand\sto\sset\sand\squery\sthis\sidentifier.\nAdd\sthe\s"magic.txt"\sfile\sto\sshow\show\sthe\sposix\sfile\scommand\smight\suse\nthis\sapplication\sid. -D 2013-05-01T19:49:27.740 +C Preserve\sthe\sapplication-ID\sacross\sVACUUM.\s\sUpdates\sto\sthe\smagic\snumber\nfile. +D 2013-05-01T20:36:23.594 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -112,7 +112,7 @@ F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 -F magic.txt 4255db3a17f0da4b3f561a904585a6605fe6a6b8 +F magic.txt 626cd2e726c22440f94d5342e872011990f90401 F main.mk 1b25be82452366abc27cc9ab2acf3244a773d5a1 F mkdll.sh 7d09b23c05d56532e9d44a50868eb4b12ff4f74a F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f @@ -250,7 +250,7 @@ F src/trigger.c cd95ac64efa60e39faf9b5597443192ff27a22fa F src/update.c a2a5631d618cbe240fc83725fa9e95c56ae0084c F src/utf.c 8d819e2e5104a430fc2005f018db14347c95a38f F src/util.c f566b5138099a2df8533b190d0dcc74b7dfbe0c9 -F src/vacuum.c 2727bdd08847fcb6b2d2da6d14f018910e8645d3 +F src/vacuum.c ddf21cc9577c4cb459d08bee9863a78ec000c5bb F src/vdbe.c 5f0047130f80c7fd0bc41bc51a653b5542c4fbd5 F src/vdbe.h b52887278cb173e66188da84dfab216bea61119d F src/vdbeInt.h c1e830268b75f04a2901dd895b51a637a26c7045 @@ -1061,10 +1061,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P faedaeace9c7ed9a8aaf96700caee09db0c0c061 -R 4f1b06c15d2f5747c86dc6db3cebcf91 -T *branch * application-id -T *sym-application-id * -T -sym-trunk * +P 28c9e7fdee2471a3026ee05ff591194d5f398131 +R eb1c61fca07cfc02ebd6d8ced912d19c U drh -Z 76bc7425f1509081256079e29083992d +Z b916c83ab322021bf56be2467b81fddd diff --git a/manifest.uuid b/manifest.uuid index 1b206e6fa8..f4b80354d1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -28c9e7fdee2471a3026ee05ff591194d5f398131 \ No newline at end of file +4a190bea18e156b6fa4dc9f21c3ad32409049603 \ No newline at end of file diff --git a/src/vacuum.c b/src/vacuum.c index 7ed2478326..4afb2cca64 100644 --- a/src/vacuum.c +++ b/src/vacuum.c @@ -289,6 +289,7 @@ int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){ BTREE_DEFAULT_CACHE_SIZE, 0, /* Preserve the default page cache size */ BTREE_TEXT_ENCODING, 0, /* Preserve the text encoding */ BTREE_USER_VERSION, 0, /* Preserve the user version */ + BTREE_APPLICATION_ID, 0, /* Preserve the application id */ }; assert( 1==sqlite3BtreeIsInTrans(pTemp) ); From 8a28a310281f08c81d0f1c1461ba71f6bf32397c Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 1 May 2013 20:40:46 +0000 Subject: [PATCH 31/32] Fix comments in the magic number file. FossilOrigin-Name: 5a500848d2fa96fc7397e2acb64d5ae6551b5b1e --- magic.txt | 11 ++++------- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 11 insertions(+), 14 deletions(-) diff --git a/magic.txt b/magic.txt index 90e2dc50a0..c99b4e004c 100644 --- a/magic.txt +++ b/magic.txt @@ -9,17 +9,14 @@ # PRAGMA application_id = INTEGER; # # INTEGER can be any signed 32-bit integer. That integer is written as -# a 4-byte big-endian integer into offset 68 of the database header. In -# the tests below, this integer is sometimes rendered as a string. For -# example, instead of "belong =1598444364" we write "string =_FSL" and -# instead of "belong =1598903374" we write "string =_MTN". +# a 4-byte big-endian integer into offset 68 of the database header. # # The Monotone application used "PRAGMA user_version=1598903374;" to set # its identifier long before "PRAGMA application_id" became available. # The user_version is very similar to application_id except that it is -# stored at offset 68 instead of offset 60. The user of -# "PRAGMA application_id" is preferred now. The rules using offset 60 -# for Monotone are for historical compatibility only. +# stored at offset 68 instead of offset 60. The application_id pragma +# is preferred. The rule using offset 60 for Monotone is for historical +# compatibility only. # 0 string =SQLite\ format\ 3 >68 belong =0x0f055111 Fossil repository - diff --git a/manifest b/manifest index 06e860720a..89c31cf5be 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Preserve\sthe\sapplication-ID\sacross\sVACUUM.\s\sUpdates\sto\sthe\smagic\snumber\nfile. -D 2013-05-01T20:36:23.594 +C Fix\scomments\sin\sthe\smagic\snumber\sfile. +D 2013-05-01T20:40:46.379 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -112,7 +112,7 @@ F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 -F magic.txt 626cd2e726c22440f94d5342e872011990f90401 +F magic.txt 291863ca976425e2e7bf3f775eb98ece4dd120f6 F main.mk 1b25be82452366abc27cc9ab2acf3244a773d5a1 F mkdll.sh 7d09b23c05d56532e9d44a50868eb4b12ff4f74a F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f @@ -1061,7 +1061,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 28c9e7fdee2471a3026ee05ff591194d5f398131 -R eb1c61fca07cfc02ebd6d8ced912d19c +P 4a190bea18e156b6fa4dc9f21c3ad32409049603 +R 22c224f485fe7b9b09b8cd93def953de U drh -Z b916c83ab322021bf56be2467b81fddd +Z f2d15752a1c82c60eb5f877e525be03c diff --git a/manifest.uuid b/manifest.uuid index f4b80354d1..4ee88a6f3c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4a190bea18e156b6fa4dc9f21c3ad32409049603 \ No newline at end of file +5a500848d2fa96fc7397e2acb64d5ae6551b5b1e \ No newline at end of file From 32c12fe2bb8e4405db53918204d192760655e51a Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 2 May 2013 17:37:31 +0000 Subject: [PATCH 32/32] Minor fixes for compilation with SQLITE_OMIT_WAL defined. FossilOrigin-Name: b81e87e72b976e7157a53a50abc5422e2a6c4c39 --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/os_unix.c | 2 +- src/pager.c | 5 ++++- src/wal.h | 1 + 5 files changed, 16 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index a5bd0b436b..3146caf6a8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\suse\sa\stransitive\sconstraint\sto\san\sIN\soperator\swhere\sthe\sRHS\sis\sa\nconstant\sif\sthere\sexists\sa\sdirect\s==\soperator\sto\sanother\stable\sin\san\souter\nloop. -D 2013-05-01T17:58:35.871 +C Minor\sfixes\sfor\scompilation\swith\sSQLITE_OMIT_WAL\sdefined. +D 2013-05-02T17:37:31.421 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -175,9 +175,9 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c b4ad71336fd96f97776f75587cd9e8218288f5be F src/os.h 4a46270a64e9193af4a0aaa3bc2c66dc07c29b3f F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 5a214c5431fd005dbb3b8bbaaa306433e8e9b48f +F src/os_unix.c 658b180a09a18214d94547f737dbded71667cdab F src/os_win.c 673b3e3d1fa3040d8d95a7f1f5e0e553aed56cfb -F src/pager.c 4a9160d268977e56ae2df90182050ab30fca715d +F src/pager.c 49e23f9898113ddfe90942bdf1c1ef57955d0921 F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 9708365594eea519cdc8504dee425c0a41c79502 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 @@ -261,7 +261,7 @@ F src/vdbesort.c 4fad64071ae120c25f39dcac572d716b9cadeb7f F src/vdbetrace.c 3ad1b4e92b60c082a02ac563da4a2735cc7d297c F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d -F src/wal.h a4d3da523d55a226a0b28e9058ef88d0a8051887 +F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 F src/where.c 12d4200eb6ae991cad02367c391db076ac1af1b0 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 @@ -1060,7 +1060,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 329478cbed06f93652de50abdb31a6b41af02b9e -R cc457983507eee39e1b071292dee5b26 -U drh -Z 944723cfd5dfa8d16765eaab2fd82886 +P faedaeace9c7ed9a8aaf96700caee09db0c0c061 +R 0cb8383fe74822aeab35f5d514cbdea3 +U dan +Z 177e5be29875bd46bd8a77f94ee092d8 diff --git a/manifest.uuid b/manifest.uuid index f88e8ae7a8..253732acb3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -faedaeace9c7ed9a8aaf96700caee09db0c0c061 \ No newline at end of file +b81e87e72b976e7157a53a50abc5422e2a6c4c39 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 0074412410..6eed674072 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -126,7 +126,7 @@ #include #include #include -#ifndef SQLITE_OMIT_WAL +#if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 #include #endif diff --git a/src/pager.c b/src/pager.c index dffcf602d7..1c6a84fea4 100644 --- a/src/pager.c +++ b/src/pager.c @@ -2871,10 +2871,13 @@ static int readDbPage(PgHdr *pPg, u32 iFrame){ return SQLITE_OK; } +#ifndef SQLITE_OMIT_WAL if( iFrame ){ /* Try to pull the page from the write-ahead log. */ rc = sqlite3WalReadFrame(pPager->pWal, iFrame, pgsz, pPg->pData); - }else{ + }else +#endif + { i64 iOffset = (pgno-1)*(i64)pPager->pageSize; rc = sqlite3OsRead(pPager->fd, pPg->pData, pgsz, iOffset); if( rc==SQLITE_IOERR_SHORT_READ ){ diff --git a/src/wal.h b/src/wal.h index ff7624af63..092546354b 100644 --- a/src/wal.h +++ b/src/wal.h @@ -43,6 +43,7 @@ # define sqlite3WalExclusiveMode(y,z) 0 # define sqlite3WalHeapMemory(z) 0 # define sqlite3WalFramesize(z) 0 +# define sqlite3WalFindFrame(x,y,z) 0 #else #define WAL_SAVEPOINT_NDATA 4