From a514b8eb0cc9c6773171381a1dfe5b456d963a97 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 25 Aug 2015 00:27:06 +0000 Subject: [PATCH 001/100] Changes toward being abld to process indexes on expressions. Not there yet - this check-in is just movement in that direction. Some tests are failing. FossilOrigin-Name: 0ad0f8d77d8f95ca2ffb7745d18219f5e87dc89c --- manifest | 19 +++++++++-------- manifest.uuid | 2 +- src/build.c | 25 ++++++++--------------- src/resolve.c | 54 ++++++++++++++++++++++--------------------------- src/sqliteInt.h | 1 + 5 files changed, 45 insertions(+), 56 deletions(-) diff --git a/manifest b/manifest index 9efff535b2..34496b20c1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\ssome\sredundant\scode:\s\sCall\ssqlite3ResolveExprListNames()\srather\sthan\ncalling\ssqlite3ResolveExprNames()\sin\sa\sloop\s-\sin\stwo\splaces. -D 2015-08-24T20:54:06.120 +C Changes\stoward\sbeing\sabld\sto\sprocess\sindexes\son\sexpressions.\s\sNot\sthere\syet\s-\s\nthis\scheck-in\sis\sjust\smovement\sin\sthat\sdirection.\s\sSome\stests\sare\sfailing. +D 2015-08-25T00:27:06.624 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -282,7 +282,7 @@ F src/btmutex.c 45a968cc85afed9b5e6cf55bf1f42f8d18107f79 F src/btree.c f48b3ef91676c06a90a8832987ecef6b94c931ee F src/btree.h 969adc948e89e449220ff0ff724c94bb2a52e9f1 F src/btreeInt.h 8177c9ab90d772d6d2c6c517e05bed774b7c92c0 -F src/build.c 789e75f3478ac63c0f398a131c49a0802c356c2b +F src/build.c d9d53c7318117e04cfb79bc498b45f4982092f0c F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 F src/complete.c addcd8160b081131005d5bc2d34adf20c1c5c92f F src/ctime.c 5a0b735dc95604766f5dac73973658eef782ee8b @@ -335,14 +335,14 @@ F src/pragma.h 631a91c8b0e6ca8f051a1d8a4a0da4150e04620a F src/prepare.c 82e5db1013846a819f198336fed72c44c974e7b1 F src/printf.c 2bc439ff20a4aad0e0ad50a37a67b5eae7d20edc F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 -F src/resolve.c 66b2740075fdb8baf90155180d33d9850cbcc976 +F src/resolve.c ad9404cfa6f698aa530cca1c86570fa92cb65a12 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c da6d1e7a4f1c8d713ed5415b5ed21d82ef465c0f F src/shell.c b1f91e60918df3a68efad1e3a11696b9a7e23d23 F src/sqlite.h.in 378bebc8fe6a88bade25e5f23b7e6123fdc64b00 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h f700e6a9dd1fdcccc9951ab022b366fb66b9e413 -F src/sqliteInt.h edbcd0c0787541a636a25ab1d1eaf847dbd043f1 +F src/sqliteInt.h 5e2ce12324eb03b75d5b8a37084550eb66dbef78 F src/sqliteLimit.h 216557999cb45f2e3578ed53ebefe228d779cb46 F src/status.c f266ad8a2892d659b74f0f50cb6a88b6e7c12179 F src/table.c 51b46b2a62d1b3a959633d593b89bab5e2c9155e @@ -1379,7 +1379,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P bed42116addabcf3dfdc2e2d51ae183965704988 -R 455d327d4502b1aadc3594ff31ef63f2 +P bdaf66465b6b1bdad10c08d9527b98e7000a41e4 +R 3888db9ab5ffaaee9ce833f48e49ed84 +T *branch * index-expr +T *sym-index-expr * +T -sym-trunk * U drh -Z 983b791d1ba8013f2148572b484fe05d +Z 80441a5a4abd6f0065befff78e2e1eff diff --git a/manifest.uuid b/manifest.uuid index 96012b923f..712ea9a3fd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bdaf66465b6b1bdad10c08d9527b98e7000a41e4 \ No newline at end of file +0ad0f8d77d8f95ca2ffb7745d18219f5e87dc89c \ No newline at end of file diff --git a/src/build.c b/src/build.c index 8a7dda89c1..64bfe1b061 100644 --- a/src/build.c +++ b/src/build.c @@ -640,9 +640,7 @@ void sqlite3DeleteTable(sqlite3 *db, Table *pTable){ sqlite3DbFree(db, pTable->zName); sqlite3DbFree(db, pTable->zColAff); sqlite3SelectDelete(db, pTable->pSelect); -#ifndef SQLITE_OMIT_CHECK sqlite3ExprListDelete(db, pTable->pCheck); -#endif #ifndef SQLITE_OMIT_VIRTUALTABLE sqlite3VtabClear(db, pTable); #endif @@ -2891,7 +2889,6 @@ Index *sqlite3CreateIndex( int iDb; /* Index of the database that is being written */ Token *pName = 0; /* Unqualified name of the index to create */ struct ExprList_item *pListItem; /* For looping over pList */ - const Column *pTabCol; /* A column in the table */ int nExtra = 0; /* Space allocated for zExtra[] */ int nExtraCol; /* Number of extra columns needed */ char *zExtra = 0; /* Extra space after the Index object */ @@ -3115,28 +3112,22 @@ Index *sqlite3CreateIndex( ** break backwards compatibility - it needs to be a warning. */ for(i=0, pListItem=pList->a; inExpr; i++, pListItem++){ - const char *zColName; Expr *pCExpr; int requestedSortOrder; char *zColl; /* Collation sequence name */ + sqlite3ResolveSelfReference(pParse, pTab, NC_IdxExpr, pListItem->pExpr, 0); + if( pParse->nErr ) goto exit_create_index; pCExpr = sqlite3ExprSkipCollate(pListItem->pExpr); - if( pCExpr->op!=TK_ID ){ + if( pCExpr->op!=TK_COLUMN ){ sqlite3ErrorMsg(pParse, "indexes on expressions not yet supported"); continue; } - zColName = pCExpr->u.zToken; - for(j=0, pTabCol=pTab->aCol; jnCol; j++, pTabCol++){ - if( sqlite3StrICmp(zColName, pTabCol->zName)==0 ) break; - } - if( j>=pTab->nCol ){ - sqlite3ErrorMsg(pParse, "table %s has no column named %s", - pTab->zName, zColName); - pParse->checkSchema = 1; - goto exit_create_index; - } + j = pCExpr->iColumn; assert( j<=0x7fff ); + if( j<0 ) j = pTab->iPKey; pIndex->aiColumn[i] = (i16)j; + zColl = 0; if( pListItem->pExpr->op==TK_COLLATE ){ int nColl; zColl = pListItem->pExpr->u.zToken; @@ -3146,10 +3137,10 @@ Index *sqlite3CreateIndex( zColl = zExtra; zExtra += nColl; nExtra -= nColl; - }else{ + }else if( j>=0 ){ zColl = pTab->aCol[j].zColl; - if( !zColl ) zColl = "BINARY"; } + if( !zColl ) zColl = "BINARY"; if( !db->init.busy && !sqlite3LocateCollSeq(pParse, zColl) ){ goto exit_create_index; } diff --git a/src/resolve.c b/src/resolve.c index 4ef8fe051b..979596941d 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -547,36 +547,28 @@ Expr *sqlite3CreateColumnExpr(sqlite3 *db, SrcList *pSrc, int iSrc, int iCol){ } /* -** Report an error that an expression is not valid for a partial index WHERE -** clause. +** Report an error that an expression is not valid for some set of +** pNC->ncFlags values determined by validMask. If */ -static void notValidPartIdxWhere( +static void notValid( Parse *pParse, /* Leave error message here */ NameContext *pNC, /* The name context */ - const char *zMsg /* Type of error */ + const char *zMsg, /* Type of error */ + int validMask, /* Set of contexts for which prohibited */ + int okForInit /* No error if pParse->db->init.busy is true */ ){ - if( (pNC->ncFlags & NC_PartIdx)!=0 ){ - sqlite3ErrorMsg(pParse, "%s prohibited in partial index WHERE clauses", - zMsg); - } -} - + assert( (validMask&~(NC_IsCheck|NC_PartIdx|NC_IdxExpr))==0 ); + if( (pNC->ncFlags & validMask)!=0 + && (pParse->db->init.busy==0 || !okForInit) + ){ + const char *zIn = "partial index WHERE clauses"; + if( pNC->ncFlags & NC_IdxExpr ) zIn = "index expressions"; #ifndef SQLITE_OMIT_CHECK -/* -** Report an error that an expression is not valid for a CHECK constraint. -*/ -static void notValidCheckConstraint( - Parse *pParse, /* Leave error message here */ - NameContext *pNC, /* The name context */ - const char *zMsg /* Type of error */ -){ - if( (pNC->ncFlags & NC_IsCheck)!=0 ){ - sqlite3ErrorMsg(pParse,"%s prohibited in CHECK constraints", zMsg); + else if( pNC->ncFlags & NC_IsCheck ) zIn = "CHECK constraints"; +#endif + sqlite3ErrorMsg(pParse, "%s prohibited in %s", zMsg, zIn); } } -#else -# define notValidCheckConstraint(P,N,M) -#endif /* ** Expression p should encode a floating point value between 1.0 and 0.0. @@ -661,6 +653,8 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ Expr *pRight; /* if( pSrcList==0 ) break; */ + notValid(pParse, pNC, "the \".\" operator", NC_IdxExpr, 0); + notValid(pParse, pNC, "the \".\" operator", NC_PartIdx|NC_IsCheck, 1); pRight = pExpr->pRight; if( pRight->op==TK_ID ){ zDb = 0; @@ -690,7 +684,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ u8 enc = ENC(pParse->db); /* The database encoding */ assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); - notValidPartIdxWhere(pParse, pNC, "functions"); + notValid(pParse, pNC, "functions", NC_PartIdx, 0); zId = pExpr->u.zToken; nId = sqlite3Strlen30(zId); pDef = sqlite3FindFunction(pParse->db, zId, nId, n, enc, 0); @@ -740,6 +734,8 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ #endif if( pDef->funcFlags & SQLITE_FUNC_CONSTANT ){ ExprSetProperty(pExpr,EP_ConstFunc); + }else{ + notValid(pParse, pNC, "non-deterministic functions", NC_IdxExpr, 0); } } if( is_agg && (pNC->ncFlags & NC_AllowAgg)==0 ){ @@ -786,8 +782,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ testcase( pExpr->op==TK_IN ); if( ExprHasProperty(pExpr, EP_xIsSelect) ){ int nRef = pNC->nRef; - notValidCheckConstraint(pParse, pNC, "subqueries"); - notValidPartIdxWhere(pParse, pNC, "subqueries"); + notValid(pParse, pNC, "subqueries", NC_IsCheck|NC_PartIdx|NC_IdxExpr,0); sqlite3WalkSelect(pWalker, pExpr->x.pSelect); assert( pNC->nRef>=nRef ); if( nRef!=pNC->nRef ){ @@ -797,8 +792,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ break; } case TK_VARIABLE: { - notValidCheckConstraint(pParse, pNC, "parameters"); - notValidPartIdxWhere(pParse, pNC, "parameters"); + notValid(pParse, pNC, "parameters", NC_IsCheck|NC_PartIdx|NC_IdxExpr, 0); break; } } @@ -1501,14 +1495,14 @@ void sqlite3ResolveSelectNames( void sqlite3ResolveSelfReference( Parse *pParse, /* Parsing context */ Table *pTab, /* The table being referenced */ - int type, /* NC_IsCheck or NC_PartIdx */ + int type, /* NC_IsCheck or NC_PartIdx or NC_IdxExpr */ Expr *pExpr, /* Expression to resolve. May be NULL. */ ExprList *pList /* Expression list to resolve. May be NUL. */ ){ SrcList sSrc; /* Fake SrcList for pParse->pNewTable */ NameContext sNC; /* Name context for pParse->pNewTable */ - assert( type==NC_IsCheck || type==NC_PartIdx ); + assert( type==NC_IsCheck || type==NC_PartIdx || type==NC_IdxExpr ); memset(&sNC, 0, sizeof(sNC)); memset(&sSrc, 0, sizeof(sSrc)); sSrc.nSrc = 1; diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 4c17904ff8..2358795148 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2391,6 +2391,7 @@ struct NameContext { #define NC_IsCheck 0x0004 /* True if resolving names in a CHECK constraint */ #define NC_InAggFunc 0x0008 /* True if analyzing arguments to an agg func */ #define NC_PartIdx 0x0010 /* True if resolving a partial index WHERE */ +#define NC_IdxExpr 0x0020 /* True if resolving columns of CREATE INDEX */ #define NC_MinMaxAgg 0x1000 /* min/max aggregates seen. See note above */ /* From 1f9ca2c84ca1601fde33eab74d376ac1b1c9d3b0 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 25 Aug 2015 16:57:52 +0000 Subject: [PATCH 002/100] Add code to maintain indexes with expression arguments across DELETE, INSERT, and UPDATE statements. Legacy tests pass, but the new code paths are still largely untested. The query planner currently makes no effort to use expression indexes. FossilOrigin-Name: efaabdb71626bdc03768e87e186c72f6f3da75b2 --- manifest | 37 ++++++++++++++---------------- manifest.uuid | 2 +- src/analyze.c | 7 +++--- src/build.c | 60 ++++++++++++++++++++++++++++++++++--------------- src/delete.c | 15 ++++++++----- src/expr.c | 27 ++++++++++++++++++++-- src/insert.c | 38 ++++++++++++++++++++++++------- src/resolve.c | 2 +- src/shell.c | 2 +- src/sqliteInt.h | 4 +++- src/update.c | 8 +++++-- src/vdbeblob.c | 3 ++- test/index.test | 6 ++--- test/rowid.test | 2 ++ 14 files changed, 146 insertions(+), 67 deletions(-) diff --git a/manifest b/manifest index 34496b20c1..b9e43c21d7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Changes\stoward\sbeing\sabld\sto\sprocess\sindexes\son\sexpressions.\s\sNot\sthere\syet\s-\s\nthis\scheck-in\sis\sjust\smovement\sin\sthat\sdirection.\s\sSome\stests\sare\sfailing. -D 2015-08-25T00:27:06.624 +C Add\scode\sto\smaintain\sindexes\swith\sexpression\sarguments\sacross\sDELETE,\sINSERT,\nand\sUPDATE\sstatements.\s\sLegacy\stests\spass,\sbut\sthe\snew\scode\spaths\sare\sstill\nlargely\suntested.\s\sThe\squery\splanner\scurrently\smakes\sno\seffort\sto\suse\nexpression\sindexes. +D 2015-08-25T16:57:52.554 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -273,7 +273,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c 48e14b8aea28dc58baafe3cfcb8889c086b7744a -F src/analyze.c f89727c36f997bd2bf6c5e546c2f51dc94e6f2a4 +F src/analyze.c 3ec61c5142e5fd6f66faf83de93d86d9810d1728 F src/attach.c e944d0052b577703b9b83aac1638452ff42a8395 F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 4d9134dc988a87838c06056c89c0e8c4700a0452 @@ -282,14 +282,14 @@ F src/btmutex.c 45a968cc85afed9b5e6cf55bf1f42f8d18107f79 F src/btree.c f48b3ef91676c06a90a8832987ecef6b94c931ee F src/btree.h 969adc948e89e449220ff0ff724c94bb2a52e9f1 F src/btreeInt.h 8177c9ab90d772d6d2c6c517e05bed774b7c92c0 -F src/build.c d9d53c7318117e04cfb79bc498b45f4982092f0c +F src/build.c c249a192b2395363e9ec89ec7060312ea6a0b2ca F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 F src/complete.c addcd8160b081131005d5bc2d34adf20c1c5c92f F src/ctime.c 5a0b735dc95604766f5dac73973658eef782ee8b F src/date.c 8ec787fed4929d8ccdf6b1bc360fccc3e1d2ca58 F src/dbstat.c f402e77e25089c6003d0c60b3233b9b3947d599a -F src/delete.c 8857a6f27560718f65d43bdbec86c967ae1f8dfa -F src/expr.c 650ac7c4f659980a3315e2aaa02a0d71e87f14a5 +F src/delete.c 07ba8cc1def3ed9a316c0073217c48ebff59062b +F src/expr.c 4a52fd29145d94c6f2e355ec67489dc8d309f05e F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c c9b63a217d86582c22121699a47f22f524608869 F src/func.c 824bea430d3a2b7dbc62806ad54da8fdb8ed9e3f @@ -297,7 +297,7 @@ F src/global.c 508e4087f7b41d688e4762dcf4d4fe28cfbc87f9 F src/hash.c 4263fbc955f26c2e8cdc0cf214bc42435aa4e4f5 F src/hash.h c8f3c31722cf3277d03713909761e152a5b81094 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 -F src/insert.c ad9ebaafdc4438bb0de58cd7d6bc199fb5b6917a +F src/insert.c 602809df9668a8418ea488a59a0bef787f0aec6f F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 92bafa308607dd985ca389a788cd9e0a2b608712 @@ -335,14 +335,14 @@ F src/pragma.h 631a91c8b0e6ca8f051a1d8a4a0da4150e04620a F src/prepare.c 82e5db1013846a819f198336fed72c44c974e7b1 F src/printf.c 2bc439ff20a4aad0e0ad50a37a67b5eae7d20edc F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 -F src/resolve.c ad9404cfa6f698aa530cca1c86570fa92cb65a12 +F src/resolve.c c4691acf9dca466799a0b07cf7f82d18d557f799 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c da6d1e7a4f1c8d713ed5415b5ed21d82ef465c0f -F src/shell.c b1f91e60918df3a68efad1e3a11696b9a7e23d23 +F src/shell.c 5a08835e85c502978bde35a89d4045833f772876 F src/sqlite.h.in 378bebc8fe6a88bade25e5f23b7e6123fdc64b00 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h f700e6a9dd1fdcccc9951ab022b366fb66b9e413 -F src/sqliteInt.h 5e2ce12324eb03b75d5b8a37084550eb66dbef78 +F src/sqliteInt.h 5fe814323e6ea95685206f8764db29dba680c8e8 F src/sqliteLimit.h 216557999cb45f2e3578ed53ebefe228d779cb46 F src/status.c f266ad8a2892d659b74f0f50cb6a88b6e7c12179 F src/table.c 51b46b2a62d1b3a959633d593b89bab5e2c9155e @@ -396,7 +396,7 @@ F src/threads.c 6bbcc9fe50c917864d48287b4792d46d6e873481 F src/tokenize.c 57cb3720f53f84d811def2069c2b169b6be539a5 F src/treeview.c c15df00728034549ff92d78ae851b44952736d3b F src/trigger.c 322f23aad694e8f31d384dcfa386d52a48d3c52f -F src/update.c 487747b328b7216bb7f6af0695d6937d5c9e605f +F src/update.c 798196d7693277d359846df7587fc15ee58a563f F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c bc9dd64b5db544218b871b66243871c202b2781f F src/vacuum.c 2ddd5cad2a7b9cef7f9e431b8c7771634c6b1701 @@ -405,7 +405,7 @@ F src/vdbe.h 7a75045d879118b9d3af7e8b3c108f2f27c51473 F src/vdbeInt.h 8b54e01ad0463590e7cffabce0bc36da9ee4f816 F src/vdbeapi.c bda74ef4b5103d7b4a4be36f936d3cf2b56a7d6f F src/vdbeaux.c af2d86b2b114a106c94fc656503fc5c89594f5af -F src/vdbeblob.c 4f2e8e075d238392df98c5e03a64342465b03f90 +F src/vdbeblob.c 1d7b97115e7bbac4c318db416d2ca83fc779544a F src/vdbemem.c ae38a0d35ae71cf604381a887c170466ba518090 F src/vdbesort.c f5009e7a35e3065635d8918b9a31f498a499976b F src/vdbetrace.c 8befe829faff6d9e6f6e4dee5a7d3f85cc85f1a0 @@ -772,7 +772,7 @@ F test/incrvacuum.test d2a6ddf5e429720b5fe502766af747915ccf6c32 F test/incrvacuum2.test 676c41428765d58f1da7dbe659ef27726d3d30ac F test/incrvacuum3.test 75256fb1377e7c39ef2de62bfc42bbff67be295a F test/incrvacuum_ioerr.test 6ae2f783424e47a0033304808fe27789cf93e635 -F test/index.test 4d990005a67a36984e4f1a5f1bdccea8d08da4ee +F test/index.test fe3c7a1aad82af92623747e9c3f3aa94ccd51238 F test/index2.test ee83c6b5e3173a3d7137140d945d9a5d4fdfb9d6 F test/index3.test b6ec456cf3b81d9a32123fe7e449bde434db338b F test/index4.test ab92e736d5946840236cd61ac3191f91a7856bf6 @@ -941,7 +941,7 @@ F test/rollback2.test fc14cf6d1a2b250d2735ef16124b971bce152f14 F test/rollbackfault.test 6a004f71087cc399296cffbb5429ea6da655ae65 F test/rowallock.test 3f88ec6819489d0b2341c7a7528ae17c053ab7cc F test/rowhash.test 0bc1d31415e4575d10cacf31e1a66b5cc0f8be81 -F test/rowid.test 742b5741584a8a44fd83e856cc2896688401d645 +F test/rowid.test 09fcded0c96fbc0ed11fb75faa3b0bad32cb011a F test/rtree.test 0c8d9dd458d6824e59683c19ab2ffa9ef946f798 F test/run-wordcount.sh 891e89c4c2d16e629cd45951d4ed899ad12afc09 F test/savepoint.test c671fdbd34cd3bfe1518a777526ada595180cf8d @@ -1379,10 +1379,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P bdaf66465b6b1bdad10c08d9527b98e7000a41e4 -R 3888db9ab5ffaaee9ce833f48e49ed84 -T *branch * index-expr -T *sym-index-expr * -T -sym-trunk * +P 0ad0f8d77d8f95ca2ffb7745d18219f5e87dc89c +R 457a1c92538887efc6aebf70344c3622 U drh -Z 80441a5a4abd6f0065befff78e2e1eff +Z d2f3b8f04887de8a5bca7c4f4441bcbc diff --git a/manifest.uuid b/manifest.uuid index 712ea9a3fd..a43b7e84ea 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0ad0f8d77d8f95ca2ffb7745d18219f5e87dc89c \ No newline at end of file +efaabdb71626bdc03768e87e186c72f6f3da75b2 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 59518cdc3f..3e531bd722 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1186,6 +1186,7 @@ static void analyzeOneTable( regKey = sqlite3GetTempRange(pParse, pPk->nKeyCol); for(j=0; jnKeyCol; j++){ k = sqlite3ColumnOfIndex(pIdx, pPk->aiColumn[j]); + assert( k>=0 && knCol ); sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, k, regKey+j); VdbeComment((v, "%s", pTab->aCol[pPk->aiColumn[j]].zName)); } @@ -1235,12 +1236,10 @@ static void analyzeOneTable( ** be taken */ VdbeCoverageNeverTaken(v); #ifdef SQLITE_ENABLE_STAT3 - sqlite3ExprCodeGetColumnOfTable(v, pTab, iTabCur, - pIdx->aiColumn[0], regSample); + sqlite3ExprCodeLoadIndexColumn(pParse, pIdx, iTabCur, 0, regSample); #else for(i=0; iaiColumn[i]; - sqlite3ExprCodeGetColumnOfTable(v, pTab, iTabCur, iCol, regCol+i); + sqlite3ExprCodeLoadIndexColumn(pParse, pIdx, iTabCur, i, regCol+i); } sqlite3VdbeAddOp3(v, OP_MakeRecord, regCol, nCol, regSample); #endif diff --git a/src/build.c b/src/build.c index 64bfe1b061..fc50fdd822 100644 --- a/src/build.c +++ b/src/build.c @@ -443,6 +443,7 @@ static void freeIndex(sqlite3 *db, Index *p){ sqlite3DeleteIndexSamples(db, p); #endif sqlite3ExprDelete(db, p->pPartIdxWhere); + sqlite3ExprListDelete(db, p->aColExpr); sqlite3DbFree(db, p->zColAff); if( p->isResized ) sqlite3DbFree(db, p->azColl); #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 @@ -3101,32 +3102,47 @@ Index *sqlite3CreateIndex( sortOrderMask = 0; /* Ignore DESC */ } - /* Scan the names of the columns of the table to be indexed and - ** load the column indices into the Index structure. Report an error - ** if any column is not found. + /* Analyze the list of expressions that form the terms of the index and + ** report any errors. In the common case where the expression is exactly + ** a table column, store that column in aiColumn[]. For general expressions, + ** populate pIndex->aColExpr and store -2 in aiColumn[]. ** - ** TODO: Add a test to make sure that the same column is not named - ** more than once within the same index. Only the first instance of - ** the column will ever be used by the optimizer. Note that using the - ** same column more than once cannot be an error because that would - ** break backwards compatibility - it needs to be a warning. + ** TODO: Issue a warning if two or more columns of the index are identical. + ** TODO: Issue a warning if the table primary key is used as part of the + ** index key. */ for(i=0, pListItem=pList->a; inExpr; i++, pListItem++){ - Expr *pCExpr; - int requestedSortOrder; + Expr *pCExpr; /* The i-th index expression */ + int requestedSortOrder; /* ASC or DESC on the i-th expression */ char *zColl; /* Collation sequence name */ sqlite3ResolveSelfReference(pParse, pTab, NC_IdxExpr, pListItem->pExpr, 0); if( pParse->nErr ) goto exit_create_index; pCExpr = sqlite3ExprSkipCollate(pListItem->pExpr); if( pCExpr->op!=TK_COLUMN ){ - sqlite3ErrorMsg(pParse, "indexes on expressions not yet supported"); - continue; + if( pTab==pParse->pNewTable ){ + sqlite3ErrorMsg(pParse, "expressions prohibited in PRIMARY KEY and " + "UNIQUE constraints"); + goto exit_create_index; + } + if( pIndex->aColExpr==0 ){ + pIndex->aColExpr = sqlite3ExprListDup(db, pList, 0); + } + j = -2; + pIndex->aiColumn[i] = -2; + if( sqlite3ExprCanBeNull(pList->a[i].pExpr) ){ + pIndex->uniqNotNull = 1; + } + }else{ + j = pCExpr->iColumn; + assert( j<=0x7fff ); + if( j<0 ){ + j = pTab->iPKey; + }else if( pTab->aCol[j].notNull==0 ){ + pIndex->uniqNotNull = 0; + } + pIndex->aiColumn[i] = (i16)j; } - j = pCExpr->iColumn; - assert( j<=0x7fff ); - if( j<0 ) j = pTab->iPKey; - pIndex->aiColumn[i] = (i16)j; zColl = 0; if( pListItem->pExpr->op==TK_COLLATE ){ int nColl; @@ -3147,11 +3163,16 @@ Index *sqlite3CreateIndex( pIndex->azColl[i] = zColl; requestedSortOrder = pListItem->sortOrder & sortOrderMask; pIndex->aSortOrder[i] = (u8)requestedSortOrder; - if( pTab->aCol[j].notNull==0 ) pIndex->uniqNotNull = 0; } + + /* Append the table key to the end of the index. For WITHOUT ROWID + ** tables (when pPk!=0) this will be the declared PRIMARY KEY. For + ** normal tables (when pPk==0) this will be the rowid. + */ if( pPk ){ for(j=0; jnKeyCol; j++){ int x = pPk->aiColumn[j]; + assert( x>=0 ); if( hasColumn(pIndex->aiColumn, pIndex->nKeyCol, x) ){ pIndex->nColumn--; }else{ @@ -3202,6 +3223,7 @@ Index *sqlite3CreateIndex( for(k=0; knKeyCol; k++){ const char *z1; const char *z2; + assert( pIdx->aiColumn[k]>=0 ); if( pIdx->aiColumn[k]!=pIndex->aiColumn[k] ) break; z1 = pIdx->azColl[k]; z2 = pIndex->azColl[k]; @@ -4093,7 +4115,9 @@ void sqlite3UniqueConstraint( sqlite3StrAccumInit(&errMsg, pParse->db, 0, 0, 200); for(j=0; jnKeyCol; j++){ - char *zCol = pTab->aCol[pIdx->aiColumn[j]].zName; + char *zCol; + assert( pIdx->aiColumn[j]>=0 ); + zCol = pTab->aCol[pIdx->aiColumn[j]].zName; if( j ) sqlite3StrAccumAppend(&errMsg, ", ", 2); sqlite3StrAccumAppendAll(&errMsg, pTab->zName); sqlite3StrAccumAppend(&errMsg, ".", 1); diff --git a/src/delete.c b/src/delete.c index 369cdaf6fe..cff922cbb3 100644 --- a/src/delete.c +++ b/src/delete.c @@ -411,6 +411,7 @@ void sqlite3DeleteFrom( /* Extract the rowid or primary key for the current row */ if( pPk ){ for(i=0; iaiColumn[i]>=(-1) ); sqlite3ExprCodeGetColumnOfTable(v, pTab, iTabCur, pPk->aiColumn[i], iPk+i); } @@ -789,14 +790,13 @@ int sqlite3GenerateIndexKey( ){ Vdbe *v = pParse->pVdbe; int j; - Table *pTab = pIdx->pTable; int regBase; int nCol; if( piPartIdxLabel ){ if( pIdx->pPartIdxWhere ){ *piPartIdxLabel = sqlite3VdbeMakeLabel(v); - pParse->iPartIdxTab = iDataCur; + pParse->iSelfTab = iDataCur; sqlite3ExprCachePush(pParse); sqlite3ExprIfFalseDup(pParse, pIdx->pPartIdxWhere, *piPartIdxLabel, SQLITE_JUMPIFNULL); @@ -808,9 +808,14 @@ int sqlite3GenerateIndexKey( regBase = sqlite3GetTempRange(pParse, nCol); if( pPrior && (regBase!=regPrior || pPrior->pPartIdxWhere) ) pPrior = 0; for(j=0; jaiColumn[j]==pIdx->aiColumn[j] ) continue; - sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, pIdx->aiColumn[j], - regBase+j); + if( pPrior + && pPrior->aiColumn[j]==pIdx->aiColumn[j] + && pPrior->aiColumn[j]>=(-1) + ){ + /* This column was already computed by the previous index */ + continue; + } + sqlite3ExprCodeLoadIndexColumn(pParse, pIdx, iDataCur, j, regBase+j); /* If the column affinity is REAL but the number is an integer, then it ** might be stored in the table as an integer (using a compact ** representation) then converted to REAL by an OP_RealAffinity opcode. diff --git a/src/expr.c b/src/expr.c index 1aebef6b16..34c382a192 100644 --- a/src/expr.c +++ b/src/expr.c @@ -2432,6 +2432,28 @@ static void sqlite3ExprCachePinRegister(Parse *pParse, int iReg){ } } +/* Generate code that will load into register regOut a value that is +** appropriate for the iIdxCol-th column of index pIdx. +*/ +void sqlite3ExprCodeLoadIndexColumn( + Parse *pParse, /* The parsing context */ + Index *pIdx, /* The index whose column is to be loaded */ + int iTabCur, /* Cursor pointing to a table row */ + int iIdxCol, /* The column of the index to be loaded */ + int regOut /* Store the index column value in this register */ +){ + i16 iTabCol = pIdx->aiColumn[iIdxCol]; + if( iTabCol>=(-1) ){ + sqlite3ExprCodeGetColumnOfTable(pParse->pVdbe, pIdx->pTable, iTabCur, + iTabCol, regOut); + return; + } + assert( pIdx->aColExpr ); + assert( pIdx->aColExpr->nExpr>iIdxCol ); + pParse->iSelfTab = iTabCur; + sqlite3ExprCode(pParse, pIdx->aColExpr->a[iIdxCol].pExpr, regOut); +} + /* ** Generate code to extract the value of the iCol-th column of a table. */ @@ -2617,8 +2639,9 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ inReg = pExpr->iColumn + pParse->ckBase; break; }else{ - /* Deleting from a partial index */ - iTab = pParse->iPartIdxTab; + /* Coding an expression that is part of an index where column names + ** in the index refer to the table to which the index belongs */ + iTab = pParse->iSelfTab; } } inReg = sqlite3ExprCodeGetColumn(pParse, pExpr->pTab, diff --git a/src/insert.c b/src/insert.c index 839599438c..ae24991301 100644 --- a/src/insert.c +++ b/src/insert.c @@ -89,7 +89,15 @@ const char *sqlite3IndexAffinityStr(Vdbe *v, Index *pIdx){ } for(n=0; nnColumn; n++){ i16 x = pIdx->aiColumn[n]; - pIdx->zColAff[n] = x<0 ? SQLITE_AFF_INTEGER : pTab->aCol[x].affinity; + if( x>=0 ){ + pIdx->zColAff[n] = pTab->aCol[x].affinity; + }else if( x==(-1) ){ + pIdx->zColAff[n] = SQLITE_AFF_INTEGER; + }else{ + assert( x==(-2) ); + assert( pIdx->aColExpr!=0 ); + pIdx->zColAff[n] = sqlite3ExprAffinity(pIdx->aColExpr->a[n].pExpr); + } } pIdx->zColAff[n] = 0; } @@ -1395,15 +1403,22 @@ void sqlite3GenerateConstraintChecks( for(i=0; inColumn; i++){ int iField = pIdx->aiColumn[i]; int x; - if( iField<0 || iField==pTab->iPKey ){ - if( regRowid==regIdx+i ) continue; /* ROWID already in regIdx+i */ - x = regNewData; - regRowid = pIdx->pPartIdxWhere ? -1 : regIdx+i; + if( iField==(-2) ){ + pParse->ckBase = regNewData+1; + sqlite3ExprCode(pParse, pIdx->aColExpr->a[i].pExpr, regIdx+i); + pParse->ckBase = 0; + VdbeComment((v, "%s column %d", pIdx->zName, i)); }else{ - x = iField + regNewData + 1; + if( iField==(-1) || iField==pTab->iPKey ){ + if( regRowid==regIdx+i ) continue; /* ROWID already in regIdx+i */ + x = regNewData; + regRowid = pIdx->pPartIdxWhere ? -1 : regIdx+i; + }else{ + x = iField + regNewData + 1; + } + sqlite3VdbeAddOp2(v, OP_SCopy, x, regIdx+i); + VdbeComment((v, "%s", iField<0 ? "rowid" : pTab->aCol[iField].zName)); } - sqlite3VdbeAddOp2(v, OP_SCopy, x, regIdx+i); - VdbeComment((v, "%s", iField<0 ? "rowid" : pTab->aCol[iField].zName)); } sqlite3VdbeAddOp3(v, OP_MakeRecord, regIdx, pIdx->nColumn, aRegIdx[ix]); VdbeComment((v, "for %s", pIdx->zName)); @@ -1724,6 +1739,13 @@ static int xferCompatibleIndex(Index *pDest, Index *pSrc){ if( pSrc->aiColumn[i]!=pDest->aiColumn[i] ){ return 0; /* Different columns indexed */ } + if( pSrc->aiColumn[i]==(-2) ){ + assert( pSrc->aColExpr!=0 && pDest->aColExpr!=0 ); + if( sqlite3ExprCompare(pSrc->aColExpr->a[i].pExpr, + pDest->aColExpr->a[i].pExpr, -1)!=0 ){ + return 0; /* Different expressions in the index */ + } + } if( pSrc->aSortOrder[i]!=pDest->aSortOrder[i] ){ return 0; /* Different sort orders */ } diff --git a/src/resolve.c b/src/resolve.c index 979596941d..a85543f09e 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -654,7 +654,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ /* if( pSrcList==0 ) break; */ notValid(pParse, pNC, "the \".\" operator", NC_IdxExpr, 0); - notValid(pParse, pNC, "the \".\" operator", NC_PartIdx|NC_IsCheck, 1); + /*notValid(pParse, pNC, "the \".\" operator", NC_PartIdx|NC_IsCheck, 1);*/ pRight = pExpr->pRight; if( pRight->op==TK_ID ){ zDb = 0; diff --git a/src/shell.c b/src/shell.c index c289998a7b..da7f78e10d 100644 --- a/src/shell.c +++ b/src/shell.c @@ -4252,8 +4252,8 @@ static int process_input(ShellState *p, FILE *in){ fprintf(stderr, "Error: incomplete SQL: %s\n", zSql); errCnt++; } - free(zSql); } + free(zSql); free(zLine); return errCnt>0; } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 2358795148..73bf83d3de 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1871,6 +1871,7 @@ struct Index { u8 *aSortOrder; /* for each column: True==DESC, False==ASC */ char **azColl; /* Array of collation sequence names for index */ Expr *pPartIdxWhere; /* WHERE clause for partial indices */ + ExprList *aColExpr; /* Column expressions */ int tnum; /* DB Page containing root of this index */ LogEst szIdxRow; /* Estimated average row size in bytes */ u16 nKeyCol; /* Number of columns forming the key */ @@ -2661,7 +2662,7 @@ struct Parse { int nOpAlloc; /* Number of slots allocated for Vdbe.aOp[] */ int iFixedOp; /* Never back out opcodes iFixedOp-1 or earlier */ int ckBase; /* Base register of data during check constraints */ - int iPartIdxTab; /* Table corresponding to a partial index */ + int iSelfTab; /* Table of an index whose exprs are being coded */ int iCacheLevel; /* ColCache valid when aColCache[].iLevel<=iCacheLevel */ int iCacheCnt; /* Counter used to generate aColCache[].lru values */ int nLabel; /* Number of labels used */ @@ -3362,6 +3363,7 @@ int sqlite3WhereIsSorted(WhereInfo*); int sqlite3WhereContinueLabel(WhereInfo*); int sqlite3WhereBreakLabel(WhereInfo*); int sqlite3WhereOkOnePass(WhereInfo*, int*); +void sqlite3ExprCodeLoadIndexColumn(Parse*, Index*, int, int, int); int sqlite3ExprCodeGetColumn(Parse*, Table*, int, int, int, u8); void sqlite3ExprCodeGetColumnOfTable(Vdbe*, Table*, int, int, int); void sqlite3ExprCodeMove(Parse*, int, int, int); diff --git a/src/update.c b/src/update.c index f8347448a1..cc9b690779 100644 --- a/src/update.c +++ b/src/update.c @@ -272,7 +272,9 @@ void sqlite3Update( /* There is one entry in the aRegIdx[] array for each index on the table ** being updated. Fill in aRegIdx[] with a register number that will hold - ** the key for accessing each index. + ** the key for accessing each index. + ** + ** FIXME: Be smarter about omitting indexes that use expressions. */ for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ int reg; @@ -281,7 +283,8 @@ void sqlite3Update( }else{ reg = 0; for(i=0; inKeyCol; i++){ - if( aXRef[pIdx->aiColumn[i]]>=0 ){ + i16 iIdxCol = pIdx->aiColumn[i]; + if( iIdxCol<0 || aXRef[iIdxCol]>=0 ){ reg = ++pParse->nMem; break; } @@ -381,6 +384,7 @@ void sqlite3Update( if( pWInfo==0 ) goto update_cleanup; okOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass); for(i=0; iaiColumn[i]>=(-1) ); sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, pPk->aiColumn[i], iPk+i); } diff --git a/src/vdbeblob.c b/src/vdbeblob.c index ea01f5ce80..2cdc3edb00 100644 --- a/src/vdbeblob.c +++ b/src/vdbeblob.c @@ -247,7 +247,8 @@ int sqlite3_blob_open( for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ int j; for(j=0; jnKeyCol; j++){ - if( pIdx->aiColumn[j]==iCol ){ + /* FIXME: Be smarter about indexes that use expressions */ + if( pIdx->aiColumn[j]==iCol || pIdx->aiColumn[j]==(-2) ){ zFault = "indexed"; } } diff --git a/test/index.test b/test/index.test index 59c0ea6f90..712f42c3a4 100644 --- a/test/index.test +++ b/test/index.test @@ -56,11 +56,11 @@ do_test index-2.1 { # Try adding an index on a column of a table where the table # exists but the column does not. # -do_test index-2.1 { +do_test index-2.1b { execsql {CREATE TABLE test1(f1 int, f2 int, f3 int)} set v [catch {execsql {CREATE INDEX index1 ON test1(f4)}} msg] lappend v $msg -} {1 {table test1 has no column named f4}} +} {1 {no such column: f4}} # Try an index with some columns that match and others that do now. # @@ -68,7 +68,7 @@ do_test index-2.2 { set v [catch {execsql {CREATE INDEX index1 ON test1(f1, f2, f4, f3)}} msg] execsql {DROP TABLE test1} lappend v $msg -} {1 {table test1 has no column named f4}} +} {1 {no such column: f4}} # Try creating a bunch of indices on the same table # diff --git a/test/rowid.test b/test/rowid.test index b00b5287fd..21696f6839 100644 --- a/test/rowid.test +++ b/test/rowid.test @@ -144,6 +144,7 @@ do_test rowid-2.8 { execsql {SELECT x FROM t1 ORDER BY x} } {1 3 5 7 9} +if 0 { # we can now.... # We cannot index by ROWID # do_test rowid-2.9 { @@ -162,6 +163,7 @@ do_test rowid-2.12 { set v [catch {execsql {CREATE INDEX idxt1 ON t1(x, rowid)}} msg] lappend v $msg } {1 {table t1 has no column named rowid}} +} # Columns defined in the CREATE statement override the buildin ROWID # column names. From bb52308f3d53949407c13e7cc624b1983b0d4034 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 27 Aug 2015 15:58:51 +0000 Subject: [PATCH 003/100] Adjustments to the WHERE term scanning, to better handle scanning terms of an index. FossilOrigin-Name: 5611130a595e7f0b6d5f21d76f2755e9c09c7810 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 53 ++++++++++++++++++++++++++++++++++----------------- 3 files changed, 43 insertions(+), 24 deletions(-) diff --git a/manifest b/manifest index cc1cce44df..8a7ec7eaf1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\senhancements\sfrom\strunk. -D 2015-08-26T18:04:23.442 +C Adjustments\sto\sthe\sWHERE\sterm\sscanning,\sto\sbetter\shandle\sscanning\sterms\sof\nan\sindex. +D 2015-08-27T15:58:51.073 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -414,7 +414,7 @@ F src/vxworks.h c18586c8edc1bddbc15c004fa16aeb1e1342b4fb F src/wal.c 6fb6b68969e4692593c2552c4e7bff5882de2cb8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 2e14d17f592d176b6dc879c33fbdec4fbccaa2ba -F src/where.c fb8546b8053433bfb39f6c39fdc99c111a6f97c5 +F src/where.c 0709bbc33117a21db57a031b1452a35de14bf86b F src/whereInt.h 880a8599226ac1c00203490d934f3ed79b292572 F src/wherecode.c 3d9113cc307ffeed58db41fe9f2d807c94787ab5 F src/whereexpr.c 1a308d1ee5144890d21ea9cf70d49bc96a83432b @@ -1380,7 +1380,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P e8b02902c48f3668dcff7b1767347ac7ef3c971e c2f3bbad778504681b39ab9399a1eb3c1a35ab3f -R 81693495b2084fce0fa08431d5ef28b1 +P ec6ddb3d481d005c304a26c948c9c808586750e9 +R e5ffdfd8fde6c8b7686cb0a9ca3923f7 U drh -Z bec0cbafd7b79403eeb50b2f9a3289c8 +Z 4ee413196597973385c2eb895ecb052b diff --git a/manifest.uuid b/manifest.uuid index c6150c8ad4..5b29dbd27d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ec6ddb3d481d005c304a26c948c9c808586750e9 \ No newline at end of file +5611130a595e7f0b6d5f21d76f2755e9c09c7810 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 9789b0848d..f3454c0fd0 100644 --- a/src/where.c +++ b/src/where.c @@ -273,11 +273,12 @@ static WhereTerm *whereScanInit( /* memset(pScan, 0, sizeof(*pScan)); */ pScan->pOrigWC = pWC; pScan->pWC = pWC; + if( pIdx ){ + j = iColumn; + iColumn = pIdx->aiColumn[j]; + } if( pIdx && iColumn>=0 ){ pScan->idxaff = pIdx->pTable->aCol[iColumn].affinity; - for(j=0; pIdx->aiColumn[j]!=iColumn; j++){ - if( NEVER(j>pIdx->nColumn) ) return 0; - } pScan->zCollName = pIdx->azColl[j]; }else{ pScan->idxaff = 0; @@ -298,6 +299,9 @@ static WhereTerm *whereScanInit( ** the WO_xx operator codes specified by the op parameter. ** Return a pointer to the term. Return 0 if not found. ** +** If pIdx!=0 then search for terms matching the iColumn-th column of pIdx +** rather than the iColumn-th column of table iCur. +** ** The term returned might by Y= if there is another constraint in ** the WHERE clause that specifies that X=Y. Any such constraints will be ** identified by the WO_EQUIV bit in the pTerm->eOperator field. The @@ -375,6 +379,24 @@ static int findIndexCol( return -1; } +/* +** Return TRUE if the iCol-th column of index pIdx is NOT NULL +*/ +static int indexColumnNotNull(Index *pIdx, int iCol){ + int j; + assert( pIdx!=0 ); + assert( iCol>=0 && iColnColumn ); + j = pIdx->aiColumn[iCol]; + if( j>=0 ){ + return pIdx->pTable->aCol[j].notNull; + }else if( j==(-1) ){ + return 1; + }else{ + assert( j==(-2) ); + return !sqlite3ExprCanBeNull(pIdx->aColExpr->a[iCol].pExpr); + } +} + /* ** Return true if the DISTINCT expression-list passed as the third argument ** is redundant. @@ -425,12 +447,9 @@ static int isDistinctRedundant( for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ if( !IsUniqueIndex(pIdx) ) continue; for(i=0; inKeyCol; i++){ - i16 iCol = pIdx->aiColumn[i]; - if( 0==sqlite3WhereFindTerm(pWC, iBase, iCol, ~(Bitmask)0, WO_EQ, pIdx) ){ - int iIdxCol = findIndexCol(pParse, pDistinct, iBase, pIdx, i); - if( iIdxCol<0 || pTab->aCol[iCol].notNull==0 ){ - break; - } + if( 0==sqlite3WhereFindTerm(pWC, iBase, i, ~(Bitmask)0, WO_EQ, pIdx) ){ + if( findIndexCol(pParse, pDistinct, iBase, pIdx, i)<0 ) break; + if( indexColumnNotNull(pIdx, i)==0 ) break; } } if( i==pIdx->nKeyCol ){ @@ -2126,7 +2145,6 @@ static int whereLoopAddBtreeIndex( u16 saved_nSkip; /* Original value of pNew->nSkip */ u32 saved_wsFlags; /* Original value of pNew->wsFlags */ LogEst saved_nOut; /* Original value of pNew->nOut */ - int iCol; /* Index of the column in the table */ int rc = SQLITE_OK; /* Return code */ LogEst rSize; /* Number of rows in the table */ LogEst rLogSize; /* Logarithm of table size */ @@ -2147,16 +2165,15 @@ static int whereLoopAddBtreeIndex( if( pProbe->bUnordered ) opMask &= ~(WO_GT|WO_GE|WO_LT|WO_LE); assert( pNew->u.btree.nEqnColumn ); - iCol = pProbe->aiColumn[pNew->u.btree.nEq]; - pTerm = whereScanInit(&scan, pBuilder->pWC, pSrc->iCursor, iCol, - opMask, pProbe); saved_nEq = pNew->u.btree.nEq; saved_nSkip = pNew->nSkip; saved_nLTerm = pNew->nLTerm; saved_wsFlags = pNew->wsFlags; saved_prereq = pNew->prereq; saved_nOut = pNew->nOut; + pTerm = whereScanInit(&scan, pBuilder->pWC, pSrc->iCursor, saved_nEq, + opMask, pProbe); pNew->rSetup = 0; rSize = pProbe->aiRowLogEst[0]; rLogSize = estLog(rSize); @@ -2169,7 +2186,7 @@ static int whereLoopAddBtreeIndex( int nRecValid = pBuilder->nRecValid; #endif if( (eOp==WO_ISNULL || (pTerm->wtFlags&TERM_VNULL)!=0) - && (iCol<0 || pSrc->pTab->aCol[iCol].notNull) + && indexColumnNotNull(pProbe, saved_nEq) ){ continue; /* ignore IS [NOT] NULL constraints on NOT NULL columns */ } @@ -2206,8 +2223,10 @@ static int whereLoopAddBtreeIndex( ** changes "x IN (?)" into "x=?". */ }else if( eOp & (WO_EQ|WO_IS) ){ + int iCol = pProbe->aiColumn[saved_nEq]; pNew->wsFlags |= WHERE_COLUMN_EQ; - if( iCol<0 || (nInMul==0 && pNew->u.btree.nEq==pProbe->nKeyCol-1) ){ + assert( saved_nEq==pNew->u.btree.nEq ); + if( iCol==(-1) || (nInMul==0 && saved_nEq==pProbe->nKeyCol-1) ){ if( iCol>=0 && pProbe->uniqNotNull==0 ){ pNew->wsFlags |= WHERE_UNQ_WANTED; }else{ @@ -2258,7 +2277,7 @@ static int whereLoopAddBtreeIndex( assert( eOp & (WO_ISNULL|WO_EQ|WO_IN|WO_IS) ); assert( pNew->nOut==saved_nOut ); - if( pTerm->truthProb<=0 && iCol>=0 ){ + if( pTerm->truthProb<=0 && pProbe->aiColumn[saved_nEq]>=0 ){ assert( (eOp & WO_IN) || nIn==0 ); testcase( eOp & WO_IN ); pNew->nOut += pTerm->truthProb; @@ -3785,7 +3804,7 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){ ) continue; opMask = pIdx->uniqNotNull ? (WO_EQ|WO_IS) : WO_EQ; for(j=0; jnKeyCol; j++){ - pTerm = sqlite3WhereFindTerm(pWC, iCur, pIdx->aiColumn[j], 0, opMask, pIdx); + pTerm = sqlite3WhereFindTerm(pWC, iCur, j, 0, opMask, pIdx); if( pTerm==0 ) break; testcase( pTerm->eOperator & WO_IS ); pLoop->aLTerm[j] = pTerm; From 6860e6fa6fec87334b5c9fe1bc3d0be45cf5ef49 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 27 Aug 2015 18:24:02 +0000 Subject: [PATCH 004/100] Activate the ability to use expressions in indexes in a query. There are some test failures, but mostly this seems to work. FossilOrigin-Name: 42f93f582eccd8a778189aa6c113874f995ab751 --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/insert.c | 5 ++++- src/where.c | 7 ++++++- src/whereInt.h | 1 + src/whereexpr.c | 6 ++++++ 6 files changed, 27 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 1f899d463c..2b7ae5a002 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\schanges\sfrom\strunk. -D 2015-08-27T16:07:02.026 +C Activate\sthe\sability\sto\suse\sexpressions\sin\sindexes\sin\sa\squery.\s\sThere\sare\ssome\ntest\sfailures,\sbut\smostly\sthis\sseems\sto\swork. +D 2015-08-27T18:24:02.056 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -297,7 +297,7 @@ F src/global.c 508e4087f7b41d688e4762dcf4d4fe28cfbc87f9 F src/hash.c 4263fbc955f26c2e8cdc0cf214bc42435aa4e4f5 F src/hash.h c8f3c31722cf3277d03713909761e152a5b81094 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 -F src/insert.c b459da1440fd10cb839c716685701f619e3d86a5 +F src/insert.c ef4da9af15ed852509fe49ff9570da6bf4819db6 F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 92bafa308607dd985ca389a788cd9e0a2b608712 @@ -414,10 +414,10 @@ F src/vxworks.h c18586c8edc1bddbc15c004fa16aeb1e1342b4fb F src/wal.c 6fb6b68969e4692593c2552c4e7bff5882de2cb8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 2e14d17f592d176b6dc879c33fbdec4fbccaa2ba -F src/where.c 7b62bfb9a6d98cbba33bfc7914f823a9ad854b33 -F src/whereInt.h 901c17c1e3c82745ad9b85b4471543fa59c980e9 +F src/where.c 29851b4a1a93934fef1bde3d0a1e7407efd57543 +F src/whereInt.h 292d3ac90da4eab1e03ac8452f1add746bcafaa1 F src/wherecode.c 3d9113cc307ffeed58db41fe9f2d807c94787ab5 -F src/whereexpr.c 1a308d1ee5144890d21ea9cf70d49bc96a83432b +F src/whereexpr.c 990ed42b5940d4000e7e61887a4bbed412c80488 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1380,7 +1380,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 5611130a595e7f0b6d5f21d76f2755e9c09c7810 cbc3c9a8bf169ae0b21f26855038502c6cc25cfe -R 5eb2cbc19058900f308a8789aae8ba36 +P c80e9e8e8cc1e7676d7c782ee0827726875db79e +R ff3e8c131dda72b26dbe6f001a80b515 U drh -Z a8340d48ea4d99e733a68ecfeafbe44a +Z 3e34cddbe64a99afb651902a91a49bea diff --git a/manifest.uuid b/manifest.uuid index f9a3e724f5..62cf69b6e4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c80e9e8e8cc1e7676d7c782ee0827726875db79e \ No newline at end of file +42f93f582eccd8a778189aa6c113874f995ab751 \ No newline at end of file diff --git a/src/insert.c b/src/insert.c index 149096c045..73879303de 100644 --- a/src/insert.c +++ b/src/insert.c @@ -93,9 +93,12 @@ const char *sqlite3IndexAffinityStr(sqlite3 *db, Index *pIdx){ }else if( x==(-1) ){ pIdx->zColAff[n] = SQLITE_AFF_INTEGER; }else{ + char aff; assert( x==(-2) ); assert( pIdx->aColExpr!=0 ); - pIdx->zColAff[n] = sqlite3ExprAffinity(pIdx->aColExpr->a[n].pExpr); + aff = sqlite3ExprAffinity(pIdx->aColExpr->a[n].pExpr); + if( aff==0 ) aff = SQLITE_AFF_BLOB; + pIdx->zColAff[n] = aff; } } pIdx->zColAff[n] = 0; diff --git a/src/where.c b/src/where.c index 62a90cade1..ad1eecd920 100644 --- a/src/where.c +++ b/src/where.c @@ -180,10 +180,13 @@ static WhereTerm *whereScanNext(WhereScan *pScan){ while( pScan->iEquiv<=pScan->nEquiv ){ iCur = pScan->aiCur[pScan->iEquiv-1]; iColumn = pScan->aiColumn[pScan->iEquiv-1]; + assert( iColumn!=(-2) || pScan->pIdxExpr!=0 ); while( (pWC = pScan->pWC)!=0 ){ for(pTerm=pWC->a+k; knTerm; k++, pTerm++){ if( pTerm->leftCursor==iCur && pTerm->u.leftColumn==iColumn + && (iColumn!=(-2) + || sqlite3ExprCompare(pTerm->pExpr->pLeft,pScan->pIdxExpr,iCur)==0) && (pScan->iEquiv<=1 || !ExprHasProperty(pTerm->pExpr, EP_FromJoin)) ){ if( (pTerm->eOperator & WO_EQUIV)!=0 @@ -273,9 +276,11 @@ static WhereTerm *whereScanInit( /* memset(pScan, 0, sizeof(*pScan)); */ pScan->pOrigWC = pWC; pScan->pWC = pWC; + pScan->pIdxExpr = 0; if( pIdx ){ j = iColumn; iColumn = pIdx->aiColumn[j]; + if( iColumn==(-2) ) pScan->pIdxExpr = pIdx->aColExpr->a[j].pExpr; } if( pIdx && iColumn>=0 ){ pScan->idxaff = pIdx->pTable->aCol[iColumn].affinity; @@ -2224,7 +2229,7 @@ static int whereLoopAddBtreeIndex( int iCol = pProbe->aiColumn[saved_nEq]; pNew->wsFlags |= WHERE_COLUMN_EQ; assert( saved_nEq==pNew->u.btree.nEq ); - if( iCol==(-1) || (nInMul==0 && saved_nEq==pProbe->nKeyCol-1) ){ + if( iCol==(-1) || (iCol>0 && nInMul==0 && saved_nEq==pProbe->nKeyCol-1) ){ if( iCol>=0 && pProbe->uniqNotNull==0 ){ pNew->wsFlags |= WHERE_UNQ_WANTED; }else{ diff --git a/src/whereInt.h b/src/whereInt.h index 7138b85b25..65c1180048 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -286,6 +286,7 @@ struct WhereScan { WhereClause *pOrigWC; /* Original, innermost WhereClause */ WhereClause *pWC; /* WhereClause currently being scanned */ char *zCollName; /* Required collating sequence, if not NULL */ + Expr *pIdxExpr; /* Search for this index expression */ char idxaff; /* Must match this affinity, if zCollName!=NULL */ unsigned char nEquiv; /* Number of entries in aEquiv[] */ unsigned char iEquiv; /* Next unused slot in aEquiv[] */ diff --git a/src/whereexpr.c b/src/whereexpr.c index 88eb5b70aa..631a7e0dce 100644 --- a/src/whereexpr.c +++ b/src/whereexpr.c @@ -872,6 +872,12 @@ static void exprAnalyze( pTerm->leftCursor = pLeft->iTable; pTerm->u.leftColumn = pLeft->iColumn; pTerm->eOperator = operatorMask(op) & opMask; + }else if( prereqLeft!=0 && (prereqLeft&(prereqLeft-1))==0 ){ + int i; + for(i=0; (prereqLeft>>i)<1; i++){} + pTerm->leftCursor = pMaskSet->ix[i]; + pTerm->u.leftColumn = -2; + pTerm->eOperator = operatorMask(op) & opMask; } if( op==TK_IS ) pTerm->wtFlags |= TERM_IS; if( pRight && pRight->op==TK_COLUMN ){ From 29d03f6517918b1e67f630e9984d1f8d726df401 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 27 Aug 2015 19:56:49 +0000 Subject: [PATCH 005/100] Fix problems in the indexed-expression handling in the optimizer. FossilOrigin-Name: 03375017691d5b480e73d712c4318632e835060c --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 4 +++- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 2b7ae5a002..2a88bdcb4d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Activate\sthe\sability\sto\suse\sexpressions\sin\sindexes\sin\sa\squery.\s\sThere\sare\ssome\ntest\sfailures,\sbut\smostly\sthis\sseems\sto\swork. -D 2015-08-27T18:24:02.056 +C Fix\sproblems\sin\sthe\sindexed-expression\shandling\sin\sthe\soptimizer. +D 2015-08-27T19:56:49.408 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -414,7 +414,7 @@ F src/vxworks.h c18586c8edc1bddbc15c004fa16aeb1e1342b4fb F src/wal.c 6fb6b68969e4692593c2552c4e7bff5882de2cb8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 2e14d17f592d176b6dc879c33fbdec4fbccaa2ba -F src/where.c 29851b4a1a93934fef1bde3d0a1e7407efd57543 +F src/where.c fcdded4bf274bf527020436e8895b33d575f18f6 F src/whereInt.h 292d3ac90da4eab1e03ac8452f1add746bcafaa1 F src/wherecode.c 3d9113cc307ffeed58db41fe9f2d807c94787ab5 F src/whereexpr.c 990ed42b5940d4000e7e61887a4bbed412c80488 @@ -1380,7 +1380,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P c80e9e8e8cc1e7676d7c782ee0827726875db79e -R ff3e8c131dda72b26dbe6f001a80b515 +P 42f93f582eccd8a778189aa6c113874f995ab751 +R 475d6e7258921a9c57642e7cd0e82c9c U drh -Z 3e34cddbe64a99afb651902a91a49bea +Z 29a0c57d5a3080634424e9023a9fd1e9 diff --git a/manifest.uuid b/manifest.uuid index 62cf69b6e4..d483132b90 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -42f93f582eccd8a778189aa6c113874f995ab751 \ No newline at end of file +03375017691d5b480e73d712c4318632e835060c \ No newline at end of file diff --git a/src/where.c b/src/where.c index ad1eecd920..c78563f1c5 100644 --- a/src/where.c +++ b/src/where.c @@ -180,7 +180,7 @@ static WhereTerm *whereScanNext(WhereScan *pScan){ while( pScan->iEquiv<=pScan->nEquiv ){ iCur = pScan->aiCur[pScan->iEquiv-1]; iColumn = pScan->aiColumn[pScan->iEquiv-1]; - assert( iColumn!=(-2) || pScan->pIdxExpr!=0 ); + if( iColumn==(-2) && pScan->pIdxExpr==0 ) return 0; while( (pWC = pScan->pWC)!=0 ){ for(pTerm=pWC->a+k; knTerm; k++, pTerm++){ if( pTerm->leftCursor==iCur @@ -804,6 +804,7 @@ static sqlite3_index_info *allocateIndexInfo( testcase( pTerm->eOperator & WO_ALL ); if( (pTerm->eOperator & ~(WO_ISNULL|WO_EQUIV|WO_IS))==0 ) continue; if( pTerm->wtFlags & TERM_VNULL ) continue; + if( pTerm->u.leftColumn<(-1) ) continue; nTerm++; } @@ -859,6 +860,7 @@ static sqlite3_index_info *allocateIndexInfo( testcase( pTerm->eOperator & WO_ALL ); if( (pTerm->eOperator & ~(WO_ISNULL|WO_EQUIV|WO_IS))==0 ) continue; if( pTerm->wtFlags & TERM_VNULL ) continue; + if( pTerm->u.leftColumn<(-1) ) continue; pIdxCons[j].iColumn = pTerm->u.leftColumn; pIdxCons[j].iTermOffset = i; op = (u8)pTerm->eOperator & WO_ALL; From c7c4680ffd727443c81c4b926e518b2552275d3f Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 27 Aug 2015 20:33:38 +0000 Subject: [PATCH 006/100] Fix EXPLAIN QUERY PLAN output for indexed-expressions. Fix another obscure fault in the WHERE term scanner. FossilOrigin-Name: 73d361ce9e4d72c943def8b0b3caa227f9199aed --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 3 +-- src/wherecode.c | 18 +++++++++++++----- 4 files changed, 22 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 2a88bdcb4d..901fd70006 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sproblems\sin\sthe\sindexed-expression\shandling\sin\sthe\soptimizer. -D 2015-08-27T19:56:49.408 +C Fix\sEXPLAIN\sQUERY\sPLAN\soutput\sfor\sindexed-expressions.\s\sFix\sanother\nobscure\sfault\sin\sthe\sWHERE\sterm\sscanner. +D 2015-08-27T20:33:38.975 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -414,9 +414,9 @@ F src/vxworks.h c18586c8edc1bddbc15c004fa16aeb1e1342b4fb F src/wal.c 6fb6b68969e4692593c2552c4e7bff5882de2cb8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 2e14d17f592d176b6dc879c33fbdec4fbccaa2ba -F src/where.c fcdded4bf274bf527020436e8895b33d575f18f6 +F src/where.c b22b416694905555a0cbbd7d8881179392a56aba F src/whereInt.h 292d3ac90da4eab1e03ac8452f1add746bcafaa1 -F src/wherecode.c 3d9113cc307ffeed58db41fe9f2d807c94787ab5 +F src/wherecode.c b0bf45ca49e62fde68ba2e2ad2939d9cdeb4e409 F src/whereexpr.c 990ed42b5940d4000e7e61887a4bbed412c80488 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd @@ -1380,7 +1380,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 42f93f582eccd8a778189aa6c113874f995ab751 -R 475d6e7258921a9c57642e7cd0e82c9c +P 03375017691d5b480e73d712c4318632e835060c +R 8aa1e38f792c255d32b6cb36a0f80565 U drh -Z 29a0c57d5a3080634424e9023a9fd1e9 +Z dc558ff8656aec58d6de7d38d850c48a diff --git a/manifest.uuid b/manifest.uuid index d483132b90..77236f721d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -03375017691d5b480e73d712c4318632e835060c \ No newline at end of file +73d361ce9e4d72c943def8b0b3caa227f9199aed \ No newline at end of file diff --git a/src/where.c b/src/where.c index c78563f1c5..8d26cd4b4d 100644 --- a/src/where.c +++ b/src/where.c @@ -191,10 +191,9 @@ static WhereTerm *whereScanNext(WhereScan *pScan){ ){ if( (pTerm->eOperator & WO_EQUIV)!=0 && pScan->nEquivaiCur) + && (pX = sqlite3ExprSkipCollate(pTerm->pExpr->pRight))->op==TK_COLUMN ){ int j; - pX = sqlite3ExprSkipCollate(pTerm->pExpr->pRight); - assert( pX->op==TK_COLUMN ); for(j=0; jnEquiv; j++){ if( pScan->aiCur[j]==pX->iTable && pScan->aiColumn[j]==pX->iColumn ){ diff --git a/src/wherecode.c b/src/wherecode.c index e5c0b40b1a..0a9d7b35be 100644 --- a/src/wherecode.c +++ b/src/wherecode.c @@ -41,6 +41,16 @@ static void explainAppendTerm( sqlite3StrAccumAppend(pStr, "?", 1); } +/* +** Return the name of the i-th column of the pIdx index. +*/ +static const char *explainIndexColumnName(Index *pIdx, int i){ + i = pIdx->aiColumn[i]; + if( i==(-2) ) return ""; + if( i==(-1) ) return "rowid"; + return pIdx->pTable->aCol[i].zName; +} + /* ** Argument pLevel describes a strategy for scanning table pTab. This ** function appends text to pStr that describes the subset of table @@ -60,13 +70,11 @@ static void explainIndexRange(StrAccum *pStr, WhereLoop *pLoop, Table *pTab){ u16 nEq = pLoop->u.btree.nEq; u16 nSkip = pLoop->nSkip; int i, j; - Column *aCol = pTab->aCol; - i16 *aiColumn = pIndex->aiColumn; if( nEq==0 && (pLoop->wsFlags&(WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))==0 ) return; sqlite3StrAccumAppend(pStr, " (", 2); for(i=0; i=nSkip ){ explainAppendTerm(pStr, i, z, "="); }else{ @@ -77,11 +85,11 @@ static void explainIndexRange(StrAccum *pStr, WhereLoop *pLoop, Table *pTab){ j = i; if( pLoop->wsFlags&WHERE_BTM_LIMIT ){ - char *z = aiColumn[j] < 0 ? "rowid" : aCol[aiColumn[j]].zName; + const char *z = explainIndexColumnName(pIndex, i); explainAppendTerm(pStr, i++, z, ">"); } if( pLoop->wsFlags&WHERE_TOP_LIMIT ){ - char *z = aiColumn[j] < 0 ? "rowid" : aCol[aiColumn[j]].zName; + const char *z = explainIndexColumnName(pIndex, j); explainAppendTerm(pStr, i, z, "<"); } sqlite3StrAccumAppend(pStr, ")", 1); From 57e0add3f9227fb3325a73d41029168a9f08bfd2 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 28 Aug 2015 19:56:47 +0000 Subject: [PATCH 007/100] Change the fts5 tokenizer API to allow more than one token to occupy a single position within a document. FossilOrigin-Name: 90b85b42f2b2dd3e939b129b7df2b822a05e243d --- ext/fts5/fts5.h | 13 ++++++++++--- ext/fts5/fts5Int.h | 3 ++- ext/fts5/fts5_aux.c | 7 +++++-- ext/fts5/fts5_config.c | 7 +++++-- ext/fts5/fts5_expr.c | 8 ++++++-- ext/fts5/fts5_main.c | 17 ++++++++++------ ext/fts5/fts5_storage.c | 20 +++++++++++++------ ext/fts5/fts5_tcl.c | 11 +++++++---- ext/fts5/fts5_tokenize.c | 26 +++++++++++++++---------- ext/fts5/test/fts5matchinfo.test | 10 ++++------ manifest | 33 +++++++++++++++++--------------- manifest.uuid | 2 +- 12 files changed, 99 insertions(+), 58 deletions(-) diff --git a/ext/fts5/fts5.h b/ext/fts5/fts5.h index c123d6444c..2e145d1068 100644 --- a/ext/fts5/fts5.h +++ b/ext/fts5/fts5.h @@ -217,7 +217,7 @@ struct Fts5ExtensionApi { int (*xTokenize)(Fts5Context*, const char *pText, int nText, /* Text to tokenize */ void *pCtx, /* Context passed to xToken() */ - int (*xToken)(void*, const char*, int, int, int) /* Callback */ + int (*xToken)(void*, const char*, int, int, int, int) /* Callback */ ); int (*xPhraseCount)(Fts5Context*); @@ -309,17 +309,24 @@ struct fts5_tokenizer { void (*xDelete)(Fts5Tokenizer*); int (*xTokenize)(Fts5Tokenizer*, void *pCtx, + int flags, const char *pText, int nText, int (*xToken)( void *pCtx, /* Copy of 2nd argument to xTokenize() */ const char *pToken, /* Pointer to buffer containing token */ int nToken, /* Size of token in bytes */ int iStart, /* Byte offset of token within input text */ - int iEnd /* Byte offset of end of token within input text */ + int iEnd, /* Byte offset of end of token within input text */ + int iPos /* Number of tokens before this one in input text */ ) ); }; +#define FTS5_TOKENIZE_QUERY 0x0001 +#define FTS5_TOKENIZE_PREFIX 0x0002 +#define FTS5_TOKENIZE_DOCUMENT 0x0004 +#define FTS5_TOKENIZE_AUX 0x0008 + /* ** END OF CUSTOM TOKENIZERS *************************************************************************/ @@ -329,7 +336,7 @@ struct fts5_tokenizer { */ typedef struct fts5_api fts5_api; struct fts5_api { - int iVersion; /* Currently always set to 1 */ + int iVersion; /* Currently always set to 2 */ /* Create a new tokenizer */ int (*xCreateTokenizer)( diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index 5298429437..0357903c7b 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -166,9 +166,10 @@ int sqlite3Fts5ConfigDeclareVtab(Fts5Config *pConfig); int sqlite3Fts5Tokenize( Fts5Config *pConfig, /* FTS5 Configuration object */ + int flags, /* FTS5_TOKENIZE_* flags */ const char *pText, int nText, /* Text to tokenize */ void *pCtx, /* Context passed to xToken() */ - int (*xToken)(void*, const char*, int, int, int) /* Callback */ + int (*xToken)(void*, const char*, int, int, int, int) /* Callback */ ); void sqlite3Fts5Dequote(char *z); diff --git a/ext/fts5/fts5_aux.c b/ext/fts5/fts5_aux.c index 818dfcf297..0e608721f7 100644 --- a/ext/fts5/fts5_aux.c +++ b/ext/fts5/fts5_aux.c @@ -151,11 +151,14 @@ static int fts5HighlightCb( const char *pToken, /* Buffer containing token */ int nToken, /* Size of token in bytes */ int iStartOff, /* Start offset of token */ - int iEndOff /* End offset of token */ + int iEndOff, /* End offset of token */ + int iPos ){ HighlightContext *p = (HighlightContext*)pContext; int rc = SQLITE_OK; - int iPos = p->iPos++; + + if( iPosiPos ) return SQLITE_OK; + p->iPos = iPos+1; if( p->iRangeEnd>0 ){ if( iPosiRangeStart || iPos>p->iRangeEnd ) return SQLITE_OK; diff --git a/ext/fts5/fts5_config.c b/ext/fts5/fts5_config.c index 7e991fc21d..7987b8af76 100644 --- a/ext/fts5/fts5_config.c +++ b/ext/fts5/fts5_config.c @@ -645,12 +645,15 @@ int sqlite3Fts5ConfigDeclareVtab(Fts5Config *pConfig){ */ int sqlite3Fts5Tokenize( Fts5Config *pConfig, /* FTS5 Configuration object */ + int flags, /* FTS5_TOKENIZE_* flags */ const char *pText, int nText, /* Text to tokenize */ void *pCtx, /* Context passed to xToken() */ - int (*xToken)(void*, const char*, int, int, int) /* Callback */ + int (*xToken)(void*, const char*, int, int, int, int) /* Callback */ ){ if( pText==0 ) return SQLITE_OK; - return pConfig->pTokApi->xTokenize(pConfig->pTok, pCtx, pText, nText, xToken); + return pConfig->pTokApi->xTokenize( + pConfig->pTok, pCtx, flags, pText, nText, xToken + ); } /* diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index a713bb7c5a..46b4091e37 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -1341,7 +1341,8 @@ static int fts5ParseTokenize( const char *pToken, /* Buffer containing token */ int nToken, /* Size of token in bytes */ int iStart, /* Start offset of token */ - int iEnd /* End offset of token */ + int iEnd, /* End offset of token */ + int iPos ){ int rc = SQLITE_OK; const int SZALLOC = 8; @@ -1417,8 +1418,11 @@ Fts5ExprPhrase *sqlite3Fts5ParseTerm( rc = fts5ParseStringFromToken(pToken, &z); if( rc==SQLITE_OK ){ + int flags = FTS5_TOKENIZE_QUERY | (bPrefix ? FTS5_TOKENIZE_QUERY : 0); + int n; sqlite3Fts5Dequote(z); - rc = sqlite3Fts5Tokenize(pConfig, z, strlen(z), &sCtx, fts5ParseTokenize); + n = strlen(z); + rc = sqlite3Fts5Tokenize(pConfig, flags, z, n, &sCtx, fts5ParseTokenize); } sqlite3_free(z); if( rc ){ diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index 1f42716514..5f0a90eadf 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -1498,11 +1498,13 @@ static int fts5ApiTokenize( Fts5Context *pCtx, const char *pText, int nText, void *pUserData, - int (*xToken)(void*, const char*, int, int, int) + int (*xToken)(void*, const char*, int, int, int, int) ){ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); - return sqlite3Fts5Tokenize(pTab->pConfig, pText, nText, pUserData, xToken); + return sqlite3Fts5Tokenize( + pTab->pConfig, FTS5_TOKENIZE_AUX, pText, nText, pUserData, xToken + ); } static int fts5ApiPhraseCount(Fts5Context *pCtx){ @@ -1658,10 +1660,11 @@ static int fts5ColumnSizeCb( const char *pToken, /* Buffer containing token */ int nToken, /* Size of token in bytes */ int iStart, /* Start offset of token */ - int iEnd /* End offset of token */ + int iEnd, /* End offset of token */ + int iPos ){ int *pCnt = (int*)pContext; - *pCnt = *pCnt + 1; + *pCnt = iPos+1; return SQLITE_OK; } @@ -1691,7 +1694,9 @@ static int fts5ApiColumnSize(Fts5Context *pCtx, int iCol, int *pnToken){ pCsr->aColumnSize[i] = 0; rc = fts5ApiColumnText(pCtx, i, &z, &n); if( rc==SQLITE_OK ){ - rc = sqlite3Fts5Tokenize(pConfig, z, n, p, fts5ColumnSizeCb); + rc = sqlite3Fts5Tokenize( + pConfig, FTS5_TOKENIZE_AUX, z, n, p, fts5ColumnSizeCb + ); } } } @@ -2344,7 +2349,7 @@ int sqlite3_fts5_init( void *p = (void*)pGlobal; memset(pGlobal, 0, sizeof(Fts5Global)); pGlobal->db = db; - pGlobal->api.iVersion = 1; + pGlobal->api.iVersion = 2; pGlobal->api.xCreateFunction = fts5CreateAux; pGlobal->api.xCreateTokenizer = fts5CreateTokenizer; pGlobal->api.xFindTokenizer = fts5FindTokenizer; diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c index f09b7d9158..6b1c69c008 100644 --- a/ext/fts5/fts5_storage.c +++ b/ext/fts5/fts5_storage.c @@ -362,11 +362,13 @@ static int fts5StorageInsertCallback( const char *pToken, /* Buffer containing token */ int nToken, /* Size of token in bytes */ int iStart, /* Start offset of token */ - int iEnd /* End offset of token */ + int iEnd, /* End offset of token */ + int iPos ){ Fts5InsertCtx *pCtx = (Fts5InsertCtx*)pContext; Fts5Index *pIdx = pCtx->pStorage->pIndex; - int iPos = pCtx->szCol++; + assert( iPos+1>=pCtx->szCol ); + pCtx->szCol = iPos+1; return sqlite3Fts5IndexWrite(pIdx, pCtx->iCol, iPos, pToken, nToken); } @@ -394,6 +396,7 @@ static int fts5StorageDeleteFromIndex(Fts5Storage *p, i64 iDel){ if( pConfig->abUnindexed[iCol-1] ) continue; ctx.szCol = 0; rc = sqlite3Fts5Tokenize(pConfig, + FTS5_TOKENIZE_DOCUMENT, (const char*)sqlite3_column_text(pSeek, iCol), sqlite3_column_bytes(pSeek, iCol), (void*)&ctx, @@ -565,6 +568,7 @@ int sqlite3Fts5StorageSpecialDelete( if( pConfig->abUnindexed[iCol] ) continue; ctx.szCol = 0; rc = sqlite3Fts5Tokenize(pConfig, + FTS5_TOKENIZE_DOCUMENT, (const char*)sqlite3_value_text(apVal[iCol]), sqlite3_value_bytes(apVal[iCol]), (void*)&ctx, @@ -654,6 +658,7 @@ int sqlite3Fts5StorageRebuild(Fts5Storage *p){ ctx.szCol = 0; if( pConfig->abUnindexed[ctx.iCol]==0 ){ rc = sqlite3Fts5Tokenize(pConfig, + FTS5_TOKENIZE_DOCUMENT, (const char*)sqlite3_column_text(pScan, ctx.iCol+1), sqlite3_column_bytes(pScan, ctx.iCol+1), (void*)&ctx, @@ -771,6 +776,7 @@ int sqlite3Fts5StorageInsert( ctx.szCol = 0; if( pConfig->abUnindexed[ctx.iCol]==0 ){ rc = sqlite3Fts5Tokenize(pConfig, + FTS5_TOKENIZE_DOCUMENT, (const char*)sqlite3_value_text(apVal[ctx.iCol+2]), sqlite3_value_bytes(apVal[ctx.iCol+2]), (void*)&ctx, @@ -841,10 +847,12 @@ static int fts5StorageIntegrityCallback( const char *pToken, /* Buffer containing token */ int nToken, /* Size of token in bytes */ int iStart, /* Start offset of token */ - int iEnd /* End offset of token */ + int iEnd, /* End offset of token */ + int iPos ){ Fts5IntegrityCtx *pCtx = (Fts5IntegrityCtx*)pContext; - int iPos = pCtx->szCol++; + assert( iPos+1>=pCtx->szCol ); + pCtx->szCol = iPos+1; pCtx->cksum ^= sqlite3Fts5IndexCksum( pCtx->pConfig, pCtx->iRowid, pCtx->iCol, iPos, pToken, nToken ); @@ -886,8 +894,8 @@ int sqlite3Fts5StorageIntegrity(Fts5Storage *p){ if( pConfig->abUnindexed[i] ) continue; ctx.iCol = i; ctx.szCol = 0; - rc = sqlite3Fts5Tokenize( - pConfig, + rc = sqlite3Fts5Tokenize(pConfig, + FTS5_TOKENIZE_DOCUMENT, (const char*)sqlite3_column_text(pScan, i+1), sqlite3_column_bytes(pScan, i+1), (void*)&ctx, diff --git a/ext/fts5/fts5_tcl.c b/ext/fts5/fts5_tcl.c index 82f3e0390d..fecbd41d32 100644 --- a/ext/fts5/fts5_tcl.c +++ b/ext/fts5/fts5_tcl.c @@ -142,7 +142,7 @@ struct F5tAuxData { static int xTokenizeCb( void *pCtx, const char *zToken, int nToken, - int iStart, int iEnd + int iStart, int iEnd, int iPos ){ F5tFunction *p = (F5tFunction*)pCtx; Tcl_Obj *pEval = Tcl_DuplicateObj(p->pScript); @@ -585,7 +585,7 @@ struct F5tTokenizeCtx { static int xTokenizeCb2( void *pCtx, const char *zToken, int nToken, - int iStart, int iEnd + int iStart, int iEnd, int iPos ){ F5tTokenizeCtx *p = (F5tTokenizeCtx*)pCtx; if( p->bSubst ){ @@ -666,7 +666,9 @@ static int f5tTokenize( ctx.bSubst = (objc==5); ctx.pRet = pRet; ctx.zInput = zText; - rc = tokenizer.xTokenize(pTok, (void*)&ctx, zText, nText, xTokenizeCb2); + rc = tokenizer.xTokenize( + pTok, (void*)&ctx, FTS5_TOKENIZE_DOCUMENT, zText, nText, xTokenizeCb2 + ); tokenizer.xDelete(pTok); if( rc!=SQLITE_OK ){ Tcl_AppendResult(interp, "error in tokenizer.xTokenize()", 0); @@ -748,8 +750,9 @@ static void f5tTokenizerDelete(Fts5Tokenizer *p){ static int f5tTokenizerTokenize( Fts5Tokenizer *p, void *pCtx, + int flags, const char *pText, int nText, - int (*xToken)(void*, const char*, int, int, int) + int (*xToken)(void*, const char*, int, int, int, int) ){ F5tTokenizerInstance *pInst = (F5tTokenizerInstance*)p; void *pOldCtx; diff --git a/ext/fts5/fts5_tokenize.c b/ext/fts5/fts5_tokenize.c index 426e35551b..d2477a050b 100644 --- a/ext/fts5/fts5_tokenize.c +++ b/ext/fts5/fts5_tokenize.c @@ -116,13 +116,15 @@ static void asciiFold(char *aOut, const char *aIn, int nByte){ static int fts5AsciiTokenize( Fts5Tokenizer *pTokenizer, void *pCtx, + int flags, const char *pText, int nText, - int (*xToken)(void*, const char*, int nToken, int iStart, int iEnd) + int (*xToken)(void*, const char*, int nToken, int iStart, int iEnd, int iPos) ){ AsciiTokenizer *p = (AsciiTokenizer*)pTokenizer; int rc = SQLITE_OK; int ie; int is = 0; + int iPos = 0; char aFold[64]; int nFold = sizeof(aFold); @@ -158,7 +160,7 @@ static int fts5AsciiTokenize( asciiFold(pFold, &pText[is], nByte); /* Invoke the token callback */ - rc = xToken(pCtx, pFold, nByte, is, ie); + rc = xToken(pCtx, pFold, nByte, is, ie, iPos++); is = ie+1; } @@ -385,12 +387,14 @@ static int fts5UnicodeIsAlnum(Unicode61Tokenizer *p, int iCode){ static int fts5UnicodeTokenize( Fts5Tokenizer *pTokenizer, void *pCtx, + int flags, const char *pText, int nText, - int (*xToken)(void*, const char*, int nToken, int iStart, int iEnd) + int (*xToken)(void*, const char*, int nToken, int iStart, int iEnd, int iPos) ){ Unicode61Tokenizer *p = (Unicode61Tokenizer*)pTokenizer; int rc = SQLITE_OK; unsigned char *a = p->aTokenChar; + int iPos = 0; unsigned char *zTerm = (unsigned char*)&pText[nText]; unsigned char *zCsr = (unsigned char *)pText; @@ -475,7 +479,7 @@ static int fts5UnicodeTokenize( } /* Invoke the token callback */ - rc = xToken(pCtx, aFold, zOut-aFold, is, ie); + rc = xToken(pCtx, aFold, zOut-aFold, is, ie, iPos++); } tokenize_done: @@ -553,7 +557,7 @@ static int fts5PorterCreate( typedef struct PorterContext PorterContext; struct PorterContext { void *pCtx; - int (*xToken)(void*, const char*, int, int, int); + int (*xToken)(void*, const char*, int, int, int, int); char *aBuf; }; @@ -1121,7 +1125,8 @@ static int fts5PorterCb( const char *pToken, int nToken, int iStart, - int iEnd + int iEnd, + int iPos ){ PorterContext *p = (PorterContext*)pCtx; @@ -1175,10 +1180,10 @@ static int fts5PorterCb( nBuf--; } - return p->xToken(p->pCtx, aBuf, nBuf, iStart, iEnd); + return p->xToken(p->pCtx, aBuf, nBuf, iStart, iEnd, iPos); pass_through: - return p->xToken(p->pCtx, pToken, nToken, iStart, iEnd); + return p->xToken(p->pCtx, pToken, nToken, iStart, iEnd, iPos); } /* @@ -1187,8 +1192,9 @@ static int fts5PorterCb( static int fts5PorterTokenize( Fts5Tokenizer *pTokenizer, void *pCtx, + int flags, const char *pText, int nText, - int (*xToken)(void*, const char*, int nToken, int iStart, int iEnd) + int (*xToken)(void*, const char*, int nToken, int iStart, int iEnd, int iPos) ){ PorterTokenizer *p = (PorterTokenizer*)pTokenizer; PorterContext sCtx; @@ -1196,7 +1202,7 @@ static int fts5PorterTokenize( sCtx.pCtx = pCtx; sCtx.aBuf = p->aBuf; return p->tokenizer.xTokenize( - p->pTokenizer, (void*)&sCtx, pText, nText, fts5PorterCb + p->pTokenizer, (void*)&sCtx, flags, pText, nText, fts5PorterCb ); } diff --git a/ext/fts5/test/fts5matchinfo.test b/ext/fts5/test/fts5matchinfo.test index 359702eff6..21f9b003e7 100644 --- a/ext/fts5/test/fts5matchinfo.test +++ b/ext/fts5/test/fts5matchinfo.test @@ -355,10 +355,10 @@ do_execsql_test 10.1 { #--------------------------------------------------------------------------- # Test the 'y' matchinfo flag # -set sqlite_fts3_enable_parentheses 1 reset_db +sqlite3_fts5_register_matchinfo db do_execsql_test 11.0 { - CREATE VIRTUAL TABLE tt USING fts3(x, y); + CREATE VIRTUAL TABLE tt USING fts5(x, y); INSERT INTO tt VALUES('c d a c d d', 'e a g b d a'); -- 1 INSERT INTO tt VALUES('c c g a e b', 'c g d g e c'); -- 2 INSERT INTO tt VALUES('b e f d e g', 'b a c b c g'); -- 3 @@ -432,19 +432,18 @@ foreach {tn expr res} { SELECT rowid, mit(matchinfo(tt, 'b')) FROM tt WHERE tt MATCH $expr } $r2 } -set sqlite_fts3_enable_parentheses 0 #--------------------------------------------------------------------------- # Test the 'b' matchinfo flag # -set sqlite_fts3_enable_parentheses 1 reset_db +sqlite3_fts5_register_matchinfo db db func mit mit do_test 12.0 { set cols [list] for {set i 0} {$i < 50} {incr i} { lappend cols "c$i" } - execsql "CREATE VIRTUAL TABLE tt USING fts3([join $cols ,])" + execsql "CREATE VIRTUAL TABLE tt USING fts5([join $cols ,])" } {} do_execsql_test 12.1 { @@ -452,6 +451,5 @@ do_execsql_test 12.1 { SELECT mit(matchinfo(tt, 'b')) FROM tt WHERE tt MATCH 'abc'; } [list [list [expr 1<<4] [expr 1<<(45-32)]]] -set sqlite_fts3_enable_parentheses 0 finish_test diff --git a/manifest b/manifest index f4ea3f942d..824f32d4c0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\scompiler\swarnings\sin\srbu\scode. -D 2015-08-28T16:41:45.530 +C Change\sthe\sfts5\stokenizer\sAPI\sto\sallow\smore\sthan\sone\stoken\sto\soccupy\sa\ssingle\sposition\swithin\sa\sdocument. +D 2015-08-28T19:56:47.300 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -105,19 +105,19 @@ F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7 F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95 F ext/fts5/extract_api_docs.tcl 06583c935f89075ea0b32f85efa5dd7619fcbd03 -F ext/fts5/fts5.h 1950ec0544de667a24c1d8af9b2fde5db7db3bc9 -F ext/fts5/fts5Int.h 45f2ceb3c030f70e2cc4c199e9f700c2f2367f77 -F ext/fts5/fts5_aux.c 044cb176a815f4388308738437f6e130aa384fb0 +F ext/fts5/fts5.h b9dfb487ada3caab4400210609b8309b71a4fb4d +F ext/fts5/fts5Int.h b0cfe44ec9451f766b77c4e5f771e7919c6dc8d5 +F ext/fts5/fts5_aux.c 7d0e275ee94ad7afdd4208d6b071b4319e8f9ca0 F ext/fts5/fts5_buffer.c 80f9ba4431848cb857e3d2158f5280093dcd8015 -F ext/fts5/fts5_config.c fdfa63ae8e527ecfaa50f94063c610429cc887cf -F ext/fts5/fts5_expr.c d075d36c84975a1cfcf070442d28e28027b61c25 +F ext/fts5/fts5_config.c ab81c8ccff6c0fb79f21c369e18e8e0dec365ec5 +F ext/fts5/fts5_expr.c f53917b6e68dee62e4c525466edacacf82eb7cbc F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246 F ext/fts5/fts5_index.c 076c4995bf06a6d1559a6e31f9a86b90f2105374 -F ext/fts5/fts5_main.c fc47ad734dfb55765b7542a345cee981170e7caa -F ext/fts5/fts5_storage.c 22ec9b5d35a39e2b5b65daf4ba7cd47fbb2d0df5 -F ext/fts5/fts5_tcl.c 96a3b9e982c4a64a242eefd752fa6669cd405a67 +F ext/fts5/fts5_main.c 7afdb84ac40b0e5bbb920a07a5cd5e062963816c +F ext/fts5/fts5_storage.c 9c263323479a4aa554738e421813cd05615d379c +F ext/fts5/fts5_tcl.c 41e2d6b455547a157085fd35fd59d4fd890dc7d3 F ext/fts5/fts5_test_mi.c 80a9e86fb4c5b6b58f8fefac05e9b96d1a6574e1 -F ext/fts5/fts5_tokenize.c 2836f6728bd74c7efac7487f5d9c27ca3e1b509c +F ext/fts5/fts5_tokenize.c 07a894410bc074685ddc0a9d89b5e7bf57ea4482 F ext/fts5/fts5_unicode2.c 78273fbd588d1d9bd0a7e4e0ccc9207348bae33c F ext/fts5/fts5_varint.c 3f86ce09cab152e3d45490d7586b7ed2e40c13f1 F ext/fts5/fts5_vocab.c 4622e0b7d84a488a1585aaa56eb214ee67a988bc @@ -160,7 +160,7 @@ F ext/fts5/test/fts5fault6.test 234dc6355f8d3f8b5be2763f30699d770247c215 F ext/fts5/test/fts5full.test 6f6143af0c6700501d9fd597189dfab1555bb741 F ext/fts5/test/fts5hash.test 42eb066f667e9a389a63437cb7038c51974d4fc6 F ext/fts5/test/fts5integrity.test 29f41d2c7126c6122fbb5d54e556506456876145 -F ext/fts5/test/fts5matchinfo.test ee6e7b130096c708c12049fa9c1ceb628954c4f9 +F ext/fts5/test/fts5matchinfo.test 2163b0013e824bba65499da9e34ea4da41349cc2 F ext/fts5/test/fts5merge.test 8f3cdba2ec9c5e7e568246e81b700ad37f764367 F ext/fts5/test/fts5near.test b214cddb1c1f1bddf45c75af768f20145f7e71cc F ext/fts5/test/fts5optimize.test 42741e7c085ee0a1276140a752d4407d97c2c9f5 @@ -1380,7 +1380,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P a84cf4f5d326270a61faf4ff867260f2dd1e68a6 -R 25627e3633787096f1c6a6ac573b9337 +P 0fdc36fe35ae2fc8e9688fe6c53437f4d47502d9 +R 694c0e23ba08ed9bcc32d2c502ed8f13 +T *branch * fts5-incompatible +T *sym-fts5-incompatible * +T -sym-trunk * U dan -Z 8fac2bb28cf0b676d514df6920f608e8 +Z 745a50831400d199b74f44c2476ec260 diff --git a/manifest.uuid b/manifest.uuid index 453c3b2f4d..91a79bb98f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0fdc36fe35ae2fc8e9688fe6c53437f4d47502d9 \ No newline at end of file +90b85b42f2b2dd3e939b129b7df2b822a05e243d \ No newline at end of file From ee0c0a8de3cdcb336b062677eedd92e8f3d44cdb Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 29 Aug 2015 15:44:27 +0000 Subject: [PATCH 008/100] Another change to the fts5 tokenizer API. FossilOrigin-Name: fc71868496f45f9c7a79ed2bf2d164a7c4718ce1 --- ext/fts5/fts5.h | 11 ++++++++--- ext/fts5/fts5Int.h | 2 +- ext/fts5/fts5_aux.c | 9 +++++---- ext/fts5/fts5_config.c | 2 +- ext/fts5/fts5_expr.c | 6 ++++-- ext/fts5/fts5_main.c | 10 ++++++---- ext/fts5/fts5_storage.c | 22 ++++++++++++---------- ext/fts5/fts5_tcl.c | 14 ++++++++------ ext/fts5/fts5_tokenize.c | 22 ++++++++++------------ manifest | 31 ++++++++++++++----------------- manifest.uuid | 2 +- 11 files changed, 70 insertions(+), 61 deletions(-) diff --git a/ext/fts5/fts5.h b/ext/fts5/fts5.h index 2e145d1068..c9eb91d4cc 100644 --- a/ext/fts5/fts5.h +++ b/ext/fts5/fts5.h @@ -217,7 +217,7 @@ struct Fts5ExtensionApi { int (*xTokenize)(Fts5Context*, const char *pText, int nText, /* Text to tokenize */ void *pCtx, /* Context passed to xToken() */ - int (*xToken)(void*, const char*, int, int, int, int) /* Callback */ + int (*xToken)(void*, int, const char*, int, int, int) /* Callback */ ); int (*xPhraseCount)(Fts5Context*); @@ -313,20 +313,25 @@ struct fts5_tokenizer { const char *pText, int nText, int (*xToken)( void *pCtx, /* Copy of 2nd argument to xTokenize() */ + int tflags, /* Mask of FTS5_TOKEN_* flags */ const char *pToken, /* Pointer to buffer containing token */ int nToken, /* Size of token in bytes */ int iStart, /* Byte offset of token within input text */ - int iEnd, /* Byte offset of end of token within input text */ - int iPos /* Number of tokens before this one in input text */ + int iEnd /* Byte offset of end of token within input text */ ) ); }; +/* Flags that may be passed as the third argument to xTokenize() */ #define FTS5_TOKENIZE_QUERY 0x0001 #define FTS5_TOKENIZE_PREFIX 0x0002 #define FTS5_TOKENIZE_DOCUMENT 0x0004 #define FTS5_TOKENIZE_AUX 0x0008 +/* Flags that may be passed by the tokenizer implementation back to FTS5 +** as the third argument to the supplied xToken callback. */ +#define FTS5_TOKEN_COLOCATED 0x0001 /* Same position as prev. token */ + /* ** END OF CUSTOM TOKENIZERS *************************************************************************/ diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index 0357903c7b..c61f8c4052 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -169,7 +169,7 @@ int sqlite3Fts5Tokenize( int flags, /* FTS5_TOKENIZE_* flags */ const char *pText, int nText, /* Text to tokenize */ void *pCtx, /* Context passed to xToken() */ - int (*xToken)(void*, const char*, int, int, int, int) /* Callback */ + int (*xToken)(void*, int, const char*, int, int, int) /* Callback */ ); void sqlite3Fts5Dequote(char *z); diff --git a/ext/fts5/fts5_aux.c b/ext/fts5/fts5_aux.c index 0e608721f7..2e33d5132f 100644 --- a/ext/fts5/fts5_aux.c +++ b/ext/fts5/fts5_aux.c @@ -148,17 +148,18 @@ static void fts5HighlightAppend( */ static int fts5HighlightCb( void *pContext, /* Pointer to HighlightContext object */ + int tflags, /* Mask of FTS5_TOKEN_* flags */ const char *pToken, /* Buffer containing token */ int nToken, /* Size of token in bytes */ int iStartOff, /* Start offset of token */ - int iEndOff, /* End offset of token */ - int iPos + int iEndOff /* End offset of token */ ){ HighlightContext *p = (HighlightContext*)pContext; int rc = SQLITE_OK; + int iPos; - if( iPosiPos ) return SQLITE_OK; - p->iPos = iPos+1; + if( tflags & FTS5_TOKEN_COLOCATED ) return SQLITE_OK; + iPos = p->iPos++; if( p->iRangeEnd>0 ){ if( iPosiRangeStart || iPos>p->iRangeEnd ) return SQLITE_OK; diff --git a/ext/fts5/fts5_config.c b/ext/fts5/fts5_config.c index 7987b8af76..74faf6dd30 100644 --- a/ext/fts5/fts5_config.c +++ b/ext/fts5/fts5_config.c @@ -648,7 +648,7 @@ int sqlite3Fts5Tokenize( int flags, /* FTS5_TOKENIZE_* flags */ const char *pText, int nText, /* Text to tokenize */ void *pCtx, /* Context passed to xToken() */ - int (*xToken)(void*, const char*, int, int, int, int) /* Callback */ + int (*xToken)(void*, int, const char*, int, int, int) /* Callback */ ){ if( pText==0 ) return SQLITE_OK; return pConfig->pTokApi->xTokenize( diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index 46b4091e37..9b52bb08c6 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -1338,11 +1338,11 @@ struct TokenCtx { */ static int fts5ParseTokenize( void *pContext, /* Pointer to Fts5InsertCtx object */ + int tflags, /* Mask of FTS5_TOKEN_* flags */ const char *pToken, /* Buffer containing token */ int nToken, /* Size of token in bytes */ int iStart, /* Start offset of token */ - int iEnd, /* End offset of token */ - int iPos + int iEnd /* End offset of token */ ){ int rc = SQLITE_OK; const int SZALLOC = 8; @@ -1350,6 +1350,8 @@ static int fts5ParseTokenize( Fts5ExprPhrase *pPhrase = pCtx->pPhrase; Fts5ExprTerm *pTerm; + if( tflags & FTS5_TOKEN_COLOCATED ) return rc; + if( pPhrase==0 || (pPhrase->nTerm % SZALLOC)==0 ){ Fts5ExprPhrase *pNew; int nNew = SZALLOC + (pPhrase ? pPhrase->nTerm : 0); diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index 5f0a90eadf..4704fb8906 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -1498,7 +1498,7 @@ static int fts5ApiTokenize( Fts5Context *pCtx, const char *pText, int nText, void *pUserData, - int (*xToken)(void*, const char*, int, int, int, int) + int (*xToken)(void*, int, const char*, int, int, int) ){ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); @@ -1657,14 +1657,16 @@ static int fts5ApiColumnText( static int fts5ColumnSizeCb( void *pContext, /* Pointer to int */ + int tflags, const char *pToken, /* Buffer containing token */ int nToken, /* Size of token in bytes */ int iStart, /* Start offset of token */ - int iEnd, /* End offset of token */ - int iPos + int iEnd /* End offset of token */ ){ int *pCnt = (int*)pContext; - *pCnt = iPos+1; + if( (tflags & FTS5_TOKEN_COLOCATED)==0 ){ + (*pCnt)++; + } return SQLITE_OK; } diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c index 6b1c69c008..d0572e60e6 100644 --- a/ext/fts5/fts5_storage.c +++ b/ext/fts5/fts5_storage.c @@ -359,17 +359,18 @@ struct Fts5InsertCtx { */ static int fts5StorageInsertCallback( void *pContext, /* Pointer to Fts5InsertCtx object */ + int tflags, const char *pToken, /* Buffer containing token */ int nToken, /* Size of token in bytes */ int iStart, /* Start offset of token */ - int iEnd, /* End offset of token */ - int iPos + int iEnd /* End offset of token */ ){ Fts5InsertCtx *pCtx = (Fts5InsertCtx*)pContext; Fts5Index *pIdx = pCtx->pStorage->pIndex; - assert( iPos+1>=pCtx->szCol ); - pCtx->szCol = iPos+1; - return sqlite3Fts5IndexWrite(pIdx, pCtx->iCol, iPos, pToken, nToken); + if( (tflags & FTS5_TOKEN_COLOCATED)==0 ){ + pCtx->szCol++; + } + return sqlite3Fts5IndexWrite(pIdx, pCtx->iCol, pCtx->szCol-1, pToken, nToken); } /* @@ -844,17 +845,18 @@ struct Fts5IntegrityCtx { */ static int fts5StorageIntegrityCallback( void *pContext, /* Pointer to Fts5InsertCtx object */ + int tflags, const char *pToken, /* Buffer containing token */ int nToken, /* Size of token in bytes */ int iStart, /* Start offset of token */ - int iEnd, /* End offset of token */ - int iPos + int iEnd /* End offset of token */ ){ Fts5IntegrityCtx *pCtx = (Fts5IntegrityCtx*)pContext; - assert( iPos+1>=pCtx->szCol ); - pCtx->szCol = iPos+1; + if( (tflags & FTS5_TOKEN_COLOCATED)==0 ){ + pCtx->szCol++; + } pCtx->cksum ^= sqlite3Fts5IndexCksum( - pCtx->pConfig, pCtx->iRowid, pCtx->iCol, iPos, pToken, nToken + pCtx->pConfig, pCtx->iRowid, pCtx->iCol, pCtx->szCol-1, pToken, nToken ); return SQLITE_OK; } diff --git a/ext/fts5/fts5_tcl.c b/ext/fts5/fts5_tcl.c index fecbd41d32..528ca958f2 100644 --- a/ext/fts5/fts5_tcl.c +++ b/ext/fts5/fts5_tcl.c @@ -141,8 +141,9 @@ struct F5tAuxData { static int xTokenizeCb( void *pCtx, + int tflags, const char *zToken, int nToken, - int iStart, int iEnd, int iPos + int iStart, int iEnd ){ F5tFunction *p = (F5tFunction*)pCtx; Tcl_Obj *pEval = Tcl_DuplicateObj(p->pScript); @@ -584,8 +585,9 @@ struct F5tTokenizeCtx { static int xTokenizeCb2( void *pCtx, + int tflags, const char *zToken, int nToken, - int iStart, int iEnd, int iPos + int iStart, int iEnd ){ F5tTokenizeCtx *p = (F5tTokenizeCtx*)pCtx; if( p->bSubst ){ @@ -694,7 +696,7 @@ typedef struct F5tTokenizerModule F5tTokenizerInstance; struct F5tTokenizerContext { void *pCtx; - int (*xToken)(void*, const char*, int, int, int); + int (*xToken)(void*, int, const char*, int, int, int); }; struct F5tTokenizerModule { @@ -752,11 +754,11 @@ static int f5tTokenizerTokenize( void *pCtx, int flags, const char *pText, int nText, - int (*xToken)(void*, const char*, int, int, int, int) + int (*xToken)(void*, int, const char*, int, int, int) ){ F5tTokenizerInstance *pInst = (F5tTokenizerInstance*)p; void *pOldCtx; - int (*xOldToken)(void*, const char*, int, int, int); + int (*xOldToken)(void*, int, const char*, int, int, int); Tcl_Obj *pEval; int rc; @@ -813,7 +815,7 @@ static int f5tTokenizerReturn( return TCL_ERROR; } - rc = p->xToken(p->pCtx, zToken, nToken, iStart, iEnd); + rc = p->xToken(p->pCtx, 0, zToken, nToken, iStart, iEnd); Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE); return TCL_OK; } diff --git a/ext/fts5/fts5_tokenize.c b/ext/fts5/fts5_tokenize.c index d2477a050b..6b1129c24d 100644 --- a/ext/fts5/fts5_tokenize.c +++ b/ext/fts5/fts5_tokenize.c @@ -118,13 +118,12 @@ static int fts5AsciiTokenize( void *pCtx, int flags, const char *pText, int nText, - int (*xToken)(void*, const char*, int nToken, int iStart, int iEnd, int iPos) + int (*xToken)(void*, int, const char*, int nToken, int iStart, int iEnd) ){ AsciiTokenizer *p = (AsciiTokenizer*)pTokenizer; int rc = SQLITE_OK; int ie; int is = 0; - int iPos = 0; char aFold[64]; int nFold = sizeof(aFold); @@ -160,7 +159,7 @@ static int fts5AsciiTokenize( asciiFold(pFold, &pText[is], nByte); /* Invoke the token callback */ - rc = xToken(pCtx, pFold, nByte, is, ie, iPos++); + rc = xToken(pCtx, 0, pFold, nByte, is, ie); is = ie+1; } @@ -389,12 +388,11 @@ static int fts5UnicodeTokenize( void *pCtx, int flags, const char *pText, int nText, - int (*xToken)(void*, const char*, int nToken, int iStart, int iEnd, int iPos) + int (*xToken)(void*, int, const char*, int nToken, int iStart, int iEnd) ){ Unicode61Tokenizer *p = (Unicode61Tokenizer*)pTokenizer; int rc = SQLITE_OK; unsigned char *a = p->aTokenChar; - int iPos = 0; unsigned char *zTerm = (unsigned char*)&pText[nText]; unsigned char *zCsr = (unsigned char *)pText; @@ -479,7 +477,7 @@ static int fts5UnicodeTokenize( } /* Invoke the token callback */ - rc = xToken(pCtx, aFold, zOut-aFold, is, ie, iPos++); + rc = xToken(pCtx, 0, aFold, zOut-aFold, is, ie); } tokenize_done: @@ -557,7 +555,7 @@ static int fts5PorterCreate( typedef struct PorterContext PorterContext; struct PorterContext { void *pCtx; - int (*xToken)(void*, const char*, int, int, int, int); + int (*xToken)(void*, int, const char*, int, int, int); char *aBuf; }; @@ -1122,11 +1120,11 @@ static void fts5PorterStep1A(char *aBuf, int *pnBuf){ static int fts5PorterCb( void *pCtx, + int tflags, const char *pToken, int nToken, int iStart, - int iEnd, - int iPos + int iEnd ){ PorterContext *p = (PorterContext*)pCtx; @@ -1180,10 +1178,10 @@ static int fts5PorterCb( nBuf--; } - return p->xToken(p->pCtx, aBuf, nBuf, iStart, iEnd, iPos); + return p->xToken(p->pCtx, tflags, aBuf, nBuf, iStart, iEnd); pass_through: - return p->xToken(p->pCtx, pToken, nToken, iStart, iEnd, iPos); + return p->xToken(p->pCtx, tflags, pToken, nToken, iStart, iEnd); } /* @@ -1194,7 +1192,7 @@ static int fts5PorterTokenize( void *pCtx, int flags, const char *pText, int nText, - int (*xToken)(void*, const char*, int nToken, int iStart, int iEnd, int iPos) + int (*xToken)(void*, int, const char*, int nToken, int iStart, int iEnd) ){ PorterTokenizer *p = (PorterTokenizer*)pTokenizer; PorterContext sCtx; diff --git a/manifest b/manifest index 824f32d4c0..964f295f10 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\sfts5\stokenizer\sAPI\sto\sallow\smore\sthan\sone\stoken\sto\soccupy\sa\ssingle\sposition\swithin\sa\sdocument. -D 2015-08-28T19:56:47.300 +C Another\schange\sto\sthe\sfts5\stokenizer\sAPI. +D 2015-08-29T15:44:27.938 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -105,19 +105,19 @@ F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7 F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95 F ext/fts5/extract_api_docs.tcl 06583c935f89075ea0b32f85efa5dd7619fcbd03 -F ext/fts5/fts5.h b9dfb487ada3caab4400210609b8309b71a4fb4d -F ext/fts5/fts5Int.h b0cfe44ec9451f766b77c4e5f771e7919c6dc8d5 -F ext/fts5/fts5_aux.c 7d0e275ee94ad7afdd4208d6b071b4319e8f9ca0 +F ext/fts5/fts5.h 0784692f406588e6c90e13a78e1f36e7e3236e42 +F ext/fts5/fts5Int.h 9fd31e682acae32806f77e7c3b543c4294274c92 +F ext/fts5/fts5_aux.c 7a307760a9c57c750d043188ec0bad59f5b5ec7e F ext/fts5/fts5_buffer.c 80f9ba4431848cb857e3d2158f5280093dcd8015 -F ext/fts5/fts5_config.c ab81c8ccff6c0fb79f21c369e18e8e0dec365ec5 -F ext/fts5/fts5_expr.c f53917b6e68dee62e4c525466edacacf82eb7cbc +F ext/fts5/fts5_config.c 80b61fd2c6844b64a3e72a64572d50a812da9384 +F ext/fts5/fts5_expr.c 7ea46f676491989069d31ae1f75c9439b0858711 F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246 F ext/fts5/fts5_index.c 076c4995bf06a6d1559a6e31f9a86b90f2105374 -F ext/fts5/fts5_main.c 7afdb84ac40b0e5bbb920a07a5cd5e062963816c -F ext/fts5/fts5_storage.c 9c263323479a4aa554738e421813cd05615d379c -F ext/fts5/fts5_tcl.c 41e2d6b455547a157085fd35fd59d4fd890dc7d3 +F ext/fts5/fts5_main.c b00834ac543431dc35edbe18018b4befe0c7fd42 +F ext/fts5/fts5_storage.c 9820e7b53ea12baf3c818485efd66346b73030c3 +F ext/fts5/fts5_tcl.c 058f8da51964458e9859edfc1ee13b1863edaeae F ext/fts5/fts5_test_mi.c 80a9e86fb4c5b6b58f8fefac05e9b96d1a6574e1 -F ext/fts5/fts5_tokenize.c 07a894410bc074685ddc0a9d89b5e7bf57ea4482 +F ext/fts5/fts5_tokenize.c 710541513ecf3fe6d9365326fc85aee6efe97229 F ext/fts5/fts5_unicode2.c 78273fbd588d1d9bd0a7e4e0ccc9207348bae33c F ext/fts5/fts5_varint.c 3f86ce09cab152e3d45490d7586b7ed2e40c13f1 F ext/fts5/fts5_vocab.c 4622e0b7d84a488a1585aaa56eb214ee67a988bc @@ -1380,10 +1380,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 0fdc36fe35ae2fc8e9688fe6c53437f4d47502d9 -R 694c0e23ba08ed9bcc32d2c502ed8f13 -T *branch * fts5-incompatible -T *sym-fts5-incompatible * -T -sym-trunk * +P 90b85b42f2b2dd3e939b129b7df2b822a05e243d +R f343432805e01f14633e088d58d566cf U dan -Z 745a50831400d199b74f44c2476ec260 +Z e4288542e4294b868813263b0597051d diff --git a/manifest.uuid b/manifest.uuid index 91a79bb98f..c7283c3a07 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -90b85b42f2b2dd3e939b129b7df2b822a05e243d \ No newline at end of file +fc71868496f45f9c7a79ed2bf2d164a7c4718ce1 \ No newline at end of file From 0cb01f5c2c3d6e7cca3c9fcbc99751a8bdbf421e Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 29 Aug 2015 18:46:12 +0000 Subject: [PATCH 009/100] Add a test for an fts5 tokenizer that supports synonyms by adding multiple entries to the fts index. FossilOrigin-Name: 98d07d16cab92f1e7001afbe370df3ec6343fc1f --- ext/fts5/fts5_storage.c | 4 +- ext/fts5/fts5_tcl.c | 80 ++++++++++--- ext/fts5/test/fts5synonym.test | 198 +++++++++++++++++++++++++++++++++ manifest | 15 +-- manifest.uuid | 2 +- 5 files changed, 271 insertions(+), 28 deletions(-) create mode 100644 ext/fts5/test/fts5synonym.test diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c index d0572e60e6..6ff81d428d 100644 --- a/ext/fts5/fts5_storage.c +++ b/ext/fts5/fts5_storage.c @@ -367,7 +367,7 @@ static int fts5StorageInsertCallback( ){ Fts5InsertCtx *pCtx = (Fts5InsertCtx*)pContext; Fts5Index *pIdx = pCtx->pStorage->pIndex; - if( (tflags & FTS5_TOKEN_COLOCATED)==0 ){ + if( (tflags & FTS5_TOKEN_COLOCATED)==0 || pCtx->szCol==0 ){ pCtx->szCol++; } return sqlite3Fts5IndexWrite(pIdx, pCtx->iCol, pCtx->szCol-1, pToken, nToken); @@ -852,7 +852,7 @@ static int fts5StorageIntegrityCallback( int iEnd /* End offset of token */ ){ Fts5IntegrityCtx *pCtx = (Fts5IntegrityCtx*)pContext; - if( (tflags & FTS5_TOKEN_COLOCATED)==0 ){ + if( (tflags & FTS5_TOKEN_COLOCATED)==0 || pCtx->szCol==0 ){ pCtx->szCol++; } pCtx->cksum ^= sqlite3Fts5IndexCksum( diff --git a/ext/fts5/fts5_tcl.c b/ext/fts5/fts5_tcl.c index 528ca958f2..21ce5f7bde 100644 --- a/ext/fts5/fts5_tcl.c +++ b/ext/fts5/fts5_tcl.c @@ -692,7 +692,7 @@ static int f5tTokenize( typedef struct F5tTokenizerContext F5tTokenizerContext; typedef struct F5tTokenizerCb F5tTokenizerCb; typedef struct F5tTokenizerModule F5tTokenizerModule; -typedef struct F5tTokenizerModule F5tTokenizerInstance; +typedef struct F5tTokenizerInstance F5tTokenizerInstance; struct F5tTokenizerContext { void *pCtx; @@ -705,6 +705,12 @@ struct F5tTokenizerModule { F5tTokenizerContext *pContext; }; +struct F5tTokenizerInstance { + Tcl_Interp *interp; + Tcl_Obj *pScript; + F5tTokenizerContext *pContext; +}; + static int f5tTokenizerCreate( void *pCtx, const char **azArg, @@ -761,18 +767,44 @@ static int f5tTokenizerTokenize( int (*xOldToken)(void*, int, const char*, int, int, int); Tcl_Obj *pEval; int rc; + const char *zFlags; pOldCtx = pInst->pContext->pCtx; xOldToken = pInst->pContext->xToken; + pInst->pContext->pCtx = pCtx; + pInst->pContext->xToken = xToken; + + assert( + flags==FTS5_TOKENIZE_DOCUMENT + || flags==FTS5_TOKENIZE_AUX + || flags==FTS5_TOKENIZE_QUERY + || flags==(FTS5_TOKENIZE_QUERY | FTS5_TOKENIZE_PREFIX) + ); pEval = Tcl_DuplicateObj(pInst->pScript); Tcl_IncrRefCount(pEval); - rc = Tcl_ListObjAppendElement( - pInst->interp, pEval, Tcl_NewStringObj(pText, nText) - ); - if( rc==TCL_OK ){ - rc = Tcl_EvalObjEx(pInst->interp, pEval, TCL_GLOBAL_ONLY); + switch( flags ){ + case FTS5_TOKENIZE_DOCUMENT: + zFlags = "document"; + break; + case FTS5_TOKENIZE_AUX: + zFlags = "aux"; + break; + case FTS5_TOKENIZE_QUERY: + zFlags = "query"; + break; + case (FTS5_TOKENIZE_PREFIX | FTS5_TOKENIZE_QUERY): + zFlags = "prefixquery"; + break; + default: + assert( 0 ); + zFlags = "invalid"; + break; } + + Tcl_ListObjAppendElement(pInst->interp, pEval, Tcl_NewStringObj(zFlags, -1)); + Tcl_ListObjAppendElement(pInst->interp, pEval, Tcl_NewStringObj(pText,nText)); + rc = Tcl_EvalObjEx(pInst->interp, pEval, TCL_GLOBAL_ONLY); Tcl_DecrRefCount(pEval); pInst->pContext->pCtx = pOldCtx; @@ -781,7 +813,7 @@ static int f5tTokenizerTokenize( } /* -** sqlite3_fts5_token TEXT START END POS +** sqlite3_fts5_token ?-colocated? TEXT START END */ static int f5tTokenizerReturn( void * clientData, @@ -793,14 +825,29 @@ static int f5tTokenizerReturn( int iStart; int iEnd; int nToken; + int tflags = 0; char *zToken; int rc; - assert( p ); - if( objc!=4 ){ - Tcl_WrongNumArgs(interp, 1, objv, "TEXT START END"); + if( objc==5 ){ + int nArg; + char *zArg = Tcl_GetStringFromObj(objv[1], &nArg); + if( nArg<=10 && nArg>=2 && memcmp("-colocated", zArg, nArg)==0 ){ + tflags |= FTS5_TOKEN_COLOCATED; + }else{ + goto usage; + } + }else if( objc!=4 ){ + goto usage; + } + + zToken = Tcl_GetStringFromObj(objv[objc-3], &nToken); + if( Tcl_GetIntFromObj(interp, objv[objc-2], &iStart) + || Tcl_GetIntFromObj(interp, objv[objc-1], &iEnd) + ){ return TCL_ERROR; } + if( p->xToken==0 ){ Tcl_AppendResult(interp, "sqlite3_fts5_token may only be used by tokenizer callback", 0 @@ -808,16 +855,13 @@ static int f5tTokenizerReturn( return TCL_ERROR; } - zToken = Tcl_GetStringFromObj(objv[1], &nToken); - if( Tcl_GetIntFromObj(interp, objv[2], &iStart) - || Tcl_GetIntFromObj(interp, objv[3], &iEnd) - ){ - return TCL_ERROR; - } - - rc = p->xToken(p->pCtx, 0, zToken, nToken, iStart, iEnd); + rc = p->xToken(p->pCtx, tflags, zToken, nToken, iStart, iEnd); Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE); return TCL_OK; + + usage: + Tcl_WrongNumArgs(interp, 1, objv, "?-colocated? TEXT START END"); + return TCL_ERROR; } static void f5tDelTokenizer(void *pCtx){ diff --git a/ext/fts5/test/fts5synonym.test b/ext/fts5/test/fts5synonym.test new file mode 100644 index 0000000000..812f7c5882 --- /dev/null +++ b/ext/fts5/test/fts5synonym.test @@ -0,0 +1,198 @@ +# 2014 Dec 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests focusing on custom tokenizers that support synonyms. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5synonym + +# If SQLITE_ENABLE_FTS5 is defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + + +proc gobble_whitespace {textvar} { + upvar $textvar t + regexp {([ ]*)(.*)} $t -> space t + return [string length $space] +} + +proc gobble_text {textvar wordvar} { + upvar $textvar t + upvar $wordvar w + regexp {([^ ]*)(.*)} $t -> w t + return [string length $w] +} + +proc do_tokenize_split {text} { + set token "" + set ret [list] + set iOff [gobble_whitespace text] + while {[set nToken [gobble_text text word]]} { + lappend ret $word $iOff [expr $iOff+$nToken] + incr iOff $nToken + incr iOff [gobble_whitespace text] + } + + set ret +} + +proc tcl_tokenize {tflags text} { + foreach {w iStart iEnd} [do_tokenize_split $text] { + sqlite3_fts5_token $w $iStart $iEnd + } +} + +proc tcl_create {args} { + return "tcl_tokenize" +} + +sqlite3_fts5_create_tokenizer db tcl tcl_create + +#------------------------------------------------------------------------- +# Warm body test for the code in fts5_tcl.c. +# +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE ft USING fts5(x, tokenize = tcl); + INSERT INTO ft VALUES('abc def ghi'); + INSERT INTO ft VALUES('jkl mno pqr'); + SELECT rowid, x FROM ft WHERE ft MATCH 'def'; + SELECT x, rowid FROM ft WHERE ft MATCH 'pqr'; +} {1 {abc def ghi} {jkl mno pqr} 2} + +#------------------------------------------------------------------------- +# Test a tokenizer that supports synonyms by adding extra entries to the +# FTS index. +# +foreach S { + {zero 0} + {one 1} + {two 2} + {three 3 iii} + {four 4} + {five 5} + {six 6} + {seven 7} + {eight 8} + {nine 9} +} { + foreach s $S { + set o [list] + foreach x $S {if {$x!=$s} {lappend o $x}} + set ::syn($s) $o + } +} + +proc tcl_tokenize {tflags text} { + foreach {w iStart iEnd} [do_tokenize_split $text] { + sqlite3_fts5_token $w $iStart $iEnd + if {$tflags=="document" && [info exists ::syn($w)]} { + foreach s $::syn($w) { + sqlite3_fts5_token -colo $s $iStart $iEnd + } + } + } +} +reset_db +sqlite3_fts5_create_tokenizer db tcl tcl_create + +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE ft USING fts5(x, tokenize = tcl); + INSERT INTO ft VALUES('one two three'); + INSERT INTO ft VALUES('four five six'); + INSERT INTO ft VALUES('eight nine ten'); +} {} + +foreach {tn expr res} { + 1 "3" 1 + 2 "eight OR 8 OR 5" {2 3} + 3 "10" {} + 4 "1*" {1} +} { + do_execsql_test 2.1.$tn { + SELECT rowid FROM ft WHERE ft MATCH $expr + } $res +} + +#------------------------------------------------------------------------- +# Test some broken tokenizers: +# +# 3.1.*: A tokenizer that declares the very first token to be colocated. +# +# 3.2.*: A tokenizer that reports two identical tokens at the same position. +# This is allowed. +# +reset_db +sqlite3_fts5_create_tokenizer db tcl tcl_create +proc tcl_tokenize {tflags text} { + set bColo 1 + foreach {w iStart iEnd} [do_tokenize_split $text] { + if {$bColo} { + sqlite3_fts5_token -colo $w $iStart $iEnd + set bColo 0 + } { + sqlite3_fts5_token $w $iStart $iEnd + } + } +} +do_execsql_test 3.1.0 { + CREATE VIRTUAL TABLE ft USING fts5(x, tokenize = tcl); + INSERT INTO ft VALUES('one two three'); + CREATE VIRTUAL TABLE vv USING fts5vocab(ft, row); + SELECT * FROM vv; +} { + one 1 1 three 1 1 two 1 1 +} + +do_execsql_test 3.1.1 { + INSERT INTO ft(ft) VALUES('integrity-check'); +} {} + +proc tcl_tokenize {tflags text} { + foreach {w iStart iEnd} [do_tokenize_split $text] { + sqlite3_fts5_token $w $iStart $iEnd + } +} + +do_execsql_test 3.1.2 { + SELECT rowid FROM ft WHERE ft MATCH 'one two three' +} {1} + +reset_db +sqlite3_fts5_create_tokenizer db tcl tcl_create +proc tcl_tokenize {tflags text} { + foreach {w iStart iEnd} [do_tokenize_split $text] { + sqlite3_fts5_token $w $iStart $iEnd + sqlite3_fts5_token -colo $w $iStart $iEnd + } +} +do_execsql_test 3.2.0 { + CREATE VIRTUAL TABLE ft USING fts5(x, tokenize = tcl); + INSERT INTO ft VALUES('one one two three'); + CREATE VIRTUAL TABLE vv USING fts5vocab(ft, row); + SELECT * FROM vv; +} { + one 1 4 three 1 2 two 1 2 +} +do_execsql_test 3.2.1 { + SELECT rowid FROM ft WHERE ft MATCH 'one two three'; + SELECT rowid FROM ft WHERE ft MATCH 'one + one + two + three'; +} {1 1} +do_execsql_test 3.2.2 { + SELECT rowid FROM ft WHERE ft MATCH 'one two two three'; + SELECT rowid FROM ft WHERE ft MATCH 'one + two + two + three'; +} {1} + +finish_test + diff --git a/manifest b/manifest index 964f295f10..fd2b4870e3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Another\schange\sto\sthe\sfts5\stokenizer\sAPI. -D 2015-08-29T15:44:27.938 +C Add\sa\stest\sfor\san\sfts5\stokenizer\sthat\ssupports\ssynonyms\sby\sadding\smultiple\sentries\sto\sthe\sfts\sindex. +D 2015-08-29T18:46:12.456 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -114,8 +114,8 @@ F ext/fts5/fts5_expr.c 7ea46f676491989069d31ae1f75c9439b0858711 F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246 F ext/fts5/fts5_index.c 076c4995bf06a6d1559a6e31f9a86b90f2105374 F ext/fts5/fts5_main.c b00834ac543431dc35edbe18018b4befe0c7fd42 -F ext/fts5/fts5_storage.c 9820e7b53ea12baf3c818485efd66346b73030c3 -F ext/fts5/fts5_tcl.c 058f8da51964458e9859edfc1ee13b1863edaeae +F ext/fts5/fts5_storage.c c888defbb961d64c12299b3d1725a24a770b047e +F ext/fts5/fts5_tcl.c 6da58d6e8f42a93c4486b5ba9b187a7f995dee37 F ext/fts5/fts5_test_mi.c 80a9e86fb4c5b6b58f8fefac05e9b96d1a6574e1 F ext/fts5/fts5_tokenize.c 710541513ecf3fe6d9365326fc85aee6efe97229 F ext/fts5/fts5_unicode2.c 78273fbd588d1d9bd0a7e4e0ccc9207348bae33c @@ -172,6 +172,7 @@ F ext/fts5/test/fts5rank.test 11dcebba31d822f7e99685b4ea2c2ae3ec0b16f1 F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17 F ext/fts5/test/fts5rowid.test 6f9833b23b176dc4aa15b7fc02afeb2b220fd460 +F ext/fts5/test/fts5synonym.test 6f1cfa5022bdae999f018075254e9fc51da2e618 F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89 F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841 F ext/fts5/test/fts5unicode2.test c1dd890ba32b7609adba78e420faa847abe43b59 @@ -1380,7 +1381,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 90b85b42f2b2dd3e939b129b7df2b822a05e243d -R f343432805e01f14633e088d58d566cf +P fc71868496f45f9c7a79ed2bf2d164a7c4718ce1 +R 8d19f3ef91012c48b3ad32684321db98 U dan -Z e4288542e4294b868813263b0597051d +Z 14116ac3039ffa0cba068c002654bc88 diff --git a/manifest.uuid b/manifest.uuid index c7283c3a07..3c4a7727be 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fc71868496f45f9c7a79ed2bf2d164a7c4718ce1 \ No newline at end of file +98d07d16cab92f1e7001afbe370df3ec6343fc1f \ No newline at end of file From 47991425cb67375e341d5c8179f14a47e2c1188d Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 31 Aug 2015 15:58:06 +0000 Subject: [PATCH 010/100] Improved analysis and usage of indexed expressions in the query planner. FossilOrigin-Name: f8893696387cba9d293a05a68dc38228077b3dc5 --- manifest | 15 +++++----- manifest.uuid | 2 +- src/where.c | 26 +++++++---------- src/whereexpr.c | 66 ++++++++++++++++++++++++++++++++++++-------- test/indexexpr1.test | 45 ++++++++++++++++++++++++++++++ 5 files changed, 118 insertions(+), 36 deletions(-) create mode 100644 test/indexexpr1.test diff --git a/manifest b/manifest index 5612003055..64b3dc444a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\slatest\senhancements\sfrom\strunk. -D 2015-08-31T14:27:29.928 +C Improved\sanalysis\sand\susage\sof\sindexed\sexpressions\sin\sthe\squery\splanner. +D 2015-08-31T15:58:06.350 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -414,10 +414,10 @@ F src/vxworks.h c18586c8edc1bddbc15c004fa16aeb1e1342b4fb F src/wal.c 8cd07f1f99e1a81346db1c9da879bef6c6f97cf6 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 2e14d17f592d176b6dc879c33fbdec4fbccaa2ba -F src/where.c acec45dc602a4f58e80e6fa088b9379ccfffd3a4 +F src/where.c 385f927f06a89d06de6a4ce7627fbd1684f157bb F src/whereInt.h 292d3ac90da4eab1e03ac8452f1add746bcafaa1 F src/wherecode.c b0bf45ca49e62fde68ba2e2ad2939d9cdeb4e409 -F src/whereexpr.c 990ed42b5940d4000e7e61887a4bbed412c80488 +F src/whereexpr.c 2473e4350e30f9b55d1c6a8f66ca23c689f23f1d F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -780,6 +780,7 @@ F test/index5.test 8621491915800ec274609e42e02a97d67e9b13e7 F test/index6.test 7102ec371414c42dfb1d5ca37eb4519aa9edc23a F test/index7.test 9c6765a74fc3fcde7aebc5b3bd40d98df14a527c F test/indexedby.test 5f527a78bae74c61b8046ae3037f9dfb0bf0c353 +F test/indexexpr1.test 36b7aab1fdb03710b99ee876ff764b92c7b96e3e F test/indexfault.test 31d4ab9a7d2f6e9616933eb079722362a883eb1d F test/init.test 15c823093fdabbf7b531fe22cf037134d09587a7 F test/insert.test 38742b5e9601c8f8d76e9b7555f7270288c2d371 @@ -1380,7 +1381,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P cf452028d1be2c5578a07f6e21b4d8b613373eb8 1da60c3dda4254620052a83c853c2d2b6dd5009f -R 64522485917d5b93eb19dc63143d2895 +P 7bde6d4d8cf05e1beb9bdf20b85760dc3e7a76c9 +R b7744e5a00368fcccd1913a83a0521e8 U drh -Z 590b51e0f03c976eccd82950be2fb3fc +Z 32945199df036dc2360847bdab3537ac diff --git a/manifest.uuid b/manifest.uuid index a7185b1794..327673b19f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7bde6d4d8cf05e1beb9bdf20b85760dc3e7a76c9 \ No newline at end of file +f8893696387cba9d293a05a68dc38228077b3dc5 \ No newline at end of file diff --git a/src/where.c b/src/where.c index ef92523a44..f39afb7e3a 100644 --- a/src/where.c +++ b/src/where.c @@ -4050,14 +4050,12 @@ WhereInfo *sqlite3WhereBegin( /* Assign a bit from the bitmask to every term in the FROM clause. ** - ** When assigning bitmask values to FROM clause cursors, it must be - ** the case that if X is the bitmask for the N-th FROM clause term then - ** the bitmask for all FROM clause terms to the left of the N-th term - ** is (X-1). An expression from the ON clause of a LEFT JOIN can use - ** its Expr.iRightJoinTable value to find the bitmask of the right table - ** of the join. Subtracting one from the right table bitmask gives a - ** bitmask for all tables to the left of the join. Knowing the bitmask - ** for all tables to the left of a left join is important. Ticket #3015. + ** The N-th term of the FROM clause is assigned a bitmask of 1<nSrc tables in ** pTabList, not just the first nTabList tables. nTabList is normally @@ -4068,14 +4066,10 @@ WhereInfo *sqlite3WhereBegin( createMask(pMaskSet, pTabList->a[ii].iCursor); sqlite3WhereTabFuncArgs(pParse, &pTabList->a[ii], &pWInfo->sWC); } -#ifndef NDEBUG - { - Bitmask toTheLeft = 0; - for(ii=0; iinSrc; ii++){ - Bitmask m = sqlite3WhereGetMask(pMaskSet, pTabList->a[ii].iCursor); - assert( (m-1)==toTheLeft ); - toTheLeft |= m; - } +#ifdef SQLITE_DEBUG + for(ii=0; iinSrc; ii++){ + Bitmask m = sqlite3WhereGetMask(pMaskSet, pTabList->a[ii].iCursor); + assert( m==MASKBIT(ii) ); } #endif diff --git a/src/whereexpr.c b/src/whereexpr.c index 631a7e0dce..dff425d0ea 100644 --- a/src/whereexpr.c +++ b/src/whereexpr.c @@ -795,6 +795,51 @@ static Bitmask exprSelectUsage(WhereMaskSet *pMaskSet, Select *pS){ return mask; } +/* +** Expression pExpr is one operand of a comparison operator that might +** be useful for indexing. This routine checks to see if pExpr appears +** in any index. Return TRUE (1) if pExpr is an indexed term and return +** FALSE (0) if not. If TRUE is returned, also set *piCur to the cursor +** number of the table that is indexed and *piColumn to the column number +** of the column that is indexed, or -2 if an expression is being indexed. +** +** If pExpr is a TK_COLUMN column reference, then this routine always returns +** true even if that particular column is not indexed, because the column +** might be added to an automatic index later. +*/ +static int exprMightBeIndexed( + SrcList *pFrom, /* The FROM clause */ + Bitmask mPrereq, /* Bitmask of FROM clause terms referenced by pExpr */ + Expr *pExpr, /* An operand of a comparison operator */ + int *piCur, /* Write the referenced table cursor number here */ + int *piColumn /* Write the referenced table column number here */ +){ + Index *pIdx; + int i; + int iCur; + if( pExpr->op==TK_COLUMN ){ + *piCur = pExpr->iTable; + *piColumn = pExpr->iColumn; + return 1; + } + if( mPrereq==0 ) return 0; /* No table references */ + if( (mPrereq&(mPrereq-1))!=0 ) return 0; /* Refs more than one table */ + for(i=0; mPrereq>1; i++, mPrereq>>=1){} + iCur = pFrom->a[i].iCursor; + for(pIdx=pFrom->a[i].pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + if( pIdx->aColExpr==0 ) continue; + for(i=0; inKeyCol; i++){ + if( pIdx->aiColumn[i]!=(-2) ) continue; + if( sqlite3ExprCompare(pExpr, pIdx->aColExpr->a[i].pExpr, iCur)==0 ){ + *piCur = iCur; + *piColumn = -2; + return 1; + } + } + } + return 0; +} + /* ** The input to this routine is an WhereTerm structure with only the ** "pExpr" field filled in. The job of this routine is to analyze the @@ -865,22 +910,19 @@ static void exprAnalyze( pTerm->iParent = -1; pTerm->eOperator = 0; if( allowedOp(op) ){ + int iCur, iColumn; Expr *pLeft = sqlite3ExprSkipCollate(pExpr->pLeft); Expr *pRight = sqlite3ExprSkipCollate(pExpr->pRight); u16 opMask = (pTerm->prereqRight & prereqLeft)==0 ? WO_ALL : WO_EQUIV; - if( pLeft->op==TK_COLUMN ){ - pTerm->leftCursor = pLeft->iTable; - pTerm->u.leftColumn = pLeft->iColumn; - pTerm->eOperator = operatorMask(op) & opMask; - }else if( prereqLeft!=0 && (prereqLeft&(prereqLeft-1))==0 ){ - int i; - for(i=0; (prereqLeft>>i)<1; i++){} - pTerm->leftCursor = pMaskSet->ix[i]; - pTerm->u.leftColumn = -2; + if( exprMightBeIndexed(pSrc, prereqLeft, pLeft, &iCur, &iColumn) ){ + pTerm->leftCursor = iCur; + pTerm->u.leftColumn = iColumn; pTerm->eOperator = operatorMask(op) & opMask; } if( op==TK_IS ) pTerm->wtFlags |= TERM_IS; - if( pRight && pRight->op==TK_COLUMN ){ + if( pRight + && exprMightBeIndexed(pSrc, pTerm->prereqRight, pRight, &iCur, &iColumn) + ){ WhereTerm *pNew; Expr *pDup; u16 eExtraOp = 0; /* Extra bits for pNew->eOperator */ @@ -909,8 +951,8 @@ static void exprAnalyze( } exprCommute(pParse, pDup); pLeft = sqlite3ExprSkipCollate(pDup->pLeft); - pNew->leftCursor = pLeft->iTable; - pNew->u.leftColumn = pLeft->iColumn; + pNew->leftCursor = iCur; + pNew->u.leftColumn = iColumn; testcase( (prereqLeft | extraRight) != prereqLeft ); pNew->prereqRight = prereqLeft | extraRight; pNew->prereqAll = prereqAll; diff --git a/test/indexexpr1.test b/test/indexexpr1.test new file mode 100644 index 0000000000..c912226ce5 --- /dev/null +++ b/test/indexexpr1.test @@ -0,0 +1,45 @@ +# 2015-08-31 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing indexes on expressions. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +do_execsql_test indexexpr1-100 { + CREATE TABLE t1(a,b,c); + INSERT INTO t1(a,b,c) + VALUES('In the beginning was the Word',1,1), + ('and the Word was with God',1,2), + ('and the Word was God',1,3), + ('The same was in the beginning with God',2,1), + ('All things were made by him',3,1), + ('and without him was not any thing made that was made',3,2); + CREATE INDEX t1a1 ON t1(substr(a,1,12)); +} {} +do_execsql_test indexexpr1-110 { + SELECT b, c, '|' FROM t1 WHERE substr(a,1,12)=='and the Word' ORDER BY b, c; +} {1 2 | 1 3 |} +do_execsql_test indexexpr1-110eqp { + EXPLAIN QUERY PLAN + SELECT b, c, '|' FROM t1 WHERE substr(a,1,12)=='and the Word' ORDER BY b, c; +} {/USING INDEX t1a1/} +do_execsql_test indexexpr1-120 { + SELECT b, c, '|' FROM t1 WHERE 'and the Word'==substr(a,1,12) ORDER BY b, c; +} {1 2 | 1 3 |} +do_execsql_test indexexpr1-120eqp { + EXPLAIN QUERY PLAN + SELECT b, c, '|' FROM t1 WHERE 'and the Word'==substr(a,1,12) ORDER BY b, c; +} {/USING INDEX t1a1/} + + +finish_test From 1d85e405e6c7674d0df4d4a720ceaeb47273b9ba Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 31 Aug 2015 17:34:41 +0000 Subject: [PATCH 011/100] Make the distinction between truly deterministic functions and date/time functions which only return the same answer for a single query. Only truly deterministic functions are allowed in indexes. Add new expression index test cases. FossilOrigin-Name: c77554b5c42327106a7b90334e9cc3c07b007c76 --- manifest | 18 +++---- manifest.uuid | 2 +- src/date.c | 16 +++--- src/resolve.c | 10 +++- src/sqliteInt.h | 34 +++++++----- test/indexexpr1.test | 123 +++++++++++++++++++++++++++++++++++++++---- 6 files changed, 160 insertions(+), 43 deletions(-) diff --git a/manifest b/manifest index 64b3dc444a..bcc1d9de6b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\sanalysis\sand\susage\sof\sindexed\sexpressions\sin\sthe\squery\splanner. -D 2015-08-31T15:58:06.350 +C Make\sthe\sdistinction\sbetween\struly\sdeterministic\sfunctions\sand\sdate/time\nfunctions\swhich\sonly\sreturn\sthe\ssame\sanswer\sfor\sa\ssingle\squery.\s\sOnly\struly\ndeterministic\sfunctions\sare\sallowed\sin\sindexes.\s\sAdd\snew\sexpression\sindex\ntest\scases. +D 2015-08-31T17:34:41.308 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -286,7 +286,7 @@ F src/build.c e0902658fc86dbd60a5c6772ca45429c69ee81fe F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 F src/complete.c addcd8160b081131005d5bc2d34adf20c1c5c92f F src/ctime.c 5a0b735dc95604766f5dac73973658eef782ee8b -F src/date.c 8ec787fed4929d8ccdf6b1bc360fccc3e1d2ca58 +F src/date.c fb1c99172017dcc8e237339132c91a21a0788584 F src/dbstat.c f402e77e25089c6003d0c60b3233b9b3947d599a F src/delete.c 224e5c9a0eae7fbdd3b461ca971465c992851354 F src/expr.c cf0c5ef6467d3eb65e3ff5c88eb2f602ba3b5163 @@ -335,14 +335,14 @@ F src/pragma.h 631a91c8b0e6ca8f051a1d8a4a0da4150e04620a F src/prepare.c 82e5db1013846a819f198336fed72c44c974e7b1 F src/printf.c 2bc439ff20a4aad0e0ad50a37a67b5eae7d20edc F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 -F src/resolve.c e6dc5a5490cf93afc1cc2cb58280c98da56acb3c +F src/resolve.c 3ec6456a534bcbf0f3fc11ad83c6208301db60a2 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c b52c80f2b1bdb62491f9ce40eea0c5f80c78d105 F src/shell.c 6332ef06db1390ef812cfdff1fc97b4fd76cdd42 F src/sqlite.h.in 378bebc8fe6a88bade25e5f23b7e6123fdc64b00 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h f700e6a9dd1fdcccc9951ab022b366fb66b9e413 -F src/sqliteInt.h 340fc115a707838ad365b52057f126b143f6aa60 +F src/sqliteInt.h 332ae4f753d91c92421a2c4227c195298d5b154c F src/sqliteLimit.h 216557999cb45f2e3578ed53ebefe228d779cb46 F src/status.c f266ad8a2892d659b74f0f50cb6a88b6e7c12179 F src/table.c 51b46b2a62d1b3a959633d593b89bab5e2c9155e @@ -780,7 +780,7 @@ F test/index5.test 8621491915800ec274609e42e02a97d67e9b13e7 F test/index6.test 7102ec371414c42dfb1d5ca37eb4519aa9edc23a F test/index7.test 9c6765a74fc3fcde7aebc5b3bd40d98df14a527c F test/indexedby.test 5f527a78bae74c61b8046ae3037f9dfb0bf0c353 -F test/indexexpr1.test 36b7aab1fdb03710b99ee876ff764b92c7b96e3e +F test/indexexpr1.test 1e57b4bb93b65a451803e7547a51f685af1c35db F test/indexfault.test 31d4ab9a7d2f6e9616933eb079722362a883eb1d F test/init.test 15c823093fdabbf7b531fe22cf037134d09587a7 F test/insert.test 38742b5e9601c8f8d76e9b7555f7270288c2d371 @@ -1381,7 +1381,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 7bde6d4d8cf05e1beb9bdf20b85760dc3e7a76c9 -R b7744e5a00368fcccd1913a83a0521e8 +P f8893696387cba9d293a05a68dc38228077b3dc5 +R 73b6719fd207957d3d30c3e1b6c218bc U drh -Z 32945199df036dc2360847bdab3537ac +Z fe2263cb4a747333f78b4cc6eb661f66 diff --git a/manifest.uuid b/manifest.uuid index 327673b19f..3dc8f017b9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f8893696387cba9d293a05a68dc38228077b3dc5 \ No newline at end of file +c77554b5c42327106a7b90334e9cc3c07b007c76 \ No newline at end of file diff --git a/src/date.c b/src/date.c index 6b11d9904a..8a66eae900 100644 --- a/src/date.c +++ b/src/date.c @@ -1115,14 +1115,14 @@ static void currentTimeFunc( void sqlite3RegisterDateTimeFunctions(void){ static SQLITE_WSD FuncDef aDateTimeFuncs[] = { #ifndef SQLITE_OMIT_DATETIME_FUNCS - FUNCTION(julianday, -1, 0, 0, juliandayFunc ), - FUNCTION(date, -1, 0, 0, dateFunc ), - FUNCTION(time, -1, 0, 0, timeFunc ), - FUNCTION(datetime, -1, 0, 0, datetimeFunc ), - FUNCTION(strftime, -1, 0, 0, strftimeFunc ), - FUNCTION(current_time, 0, 0, 0, ctimeFunc ), - FUNCTION(current_timestamp, 0, 0, 0, ctimestampFunc), - FUNCTION(current_date, 0, 0, 0, cdateFunc ), + DFUNCTION(julianday, -1, 0, 0, juliandayFunc ), + DFUNCTION(date, -1, 0, 0, dateFunc ), + DFUNCTION(time, -1, 0, 0, timeFunc ), + DFUNCTION(datetime, -1, 0, 0, datetimeFunc ), + DFUNCTION(strftime, -1, 0, 0, strftimeFunc ), + DFUNCTION(current_time, 0, 0, 0, ctimeFunc ), + DFUNCTION(current_timestamp, 0, 0, 0, ctimestampFunc), + DFUNCTION(current_date, 0, 0, 0, cdateFunc ), #else STR_FUNCTION(current_time, 0, "%H:%M:%S", 0, currentTimeFunc), STR_FUNCTION(current_date, 0, "%Y-%m-%d", 0, currentTimeFunc), diff --git a/src/resolve.c b/src/resolve.c index eb43396454..0b8dde7cc6 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -732,9 +732,15 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ return WRC_Prune; } #endif - if( pDef->funcFlags & SQLITE_FUNC_CONSTANT ){ + if( pDef->funcFlags & (SQLITE_FUNC_CONSTANT|SQLITE_FUNC_DATETIME) ){ + /* For the purposes of the EP_ConstFunc flag, date and time + ** functions are considered constant because the the time does + ** not change for the duration of a query. */ ExprSetProperty(pExpr,EP_ConstFunc); - }else{ + } + if( (pDef->funcFlags & SQLITE_FUNC_CONSTANT)==0 ){ + /* DATETIME functions are considered non-deterministic because of + ** the 'now' capability */ notValid(pParse, pNC, "non-deterministic functions", NC_IdxExpr, 0); } } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 69f98da26d..613f351aa5 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1384,18 +1384,19 @@ struct FuncDestructor { ** values must correspond to OPFLAG_LENGTHARG and OPFLAG_TYPEOFARG. There ** are assert() statements in the code to verify this. */ -#define SQLITE_FUNC_ENCMASK 0x003 /* SQLITE_UTF8, SQLITE_UTF16BE or UTF16LE */ -#define SQLITE_FUNC_LIKE 0x004 /* Candidate for the LIKE optimization */ -#define SQLITE_FUNC_CASE 0x008 /* Case-sensitive LIKE-type function */ -#define SQLITE_FUNC_EPHEM 0x010 /* Ephemeral. Delete with VDBE */ -#define SQLITE_FUNC_NEEDCOLL 0x020 /* sqlite3GetFuncCollSeq() might be called */ -#define SQLITE_FUNC_LENGTH 0x040 /* Built-in length() function */ -#define SQLITE_FUNC_TYPEOF 0x080 /* Built-in typeof() function */ -#define SQLITE_FUNC_COUNT 0x100 /* Built-in count(*) aggregate */ -#define SQLITE_FUNC_COALESCE 0x200 /* Built-in coalesce() or ifnull() */ -#define SQLITE_FUNC_UNLIKELY 0x400 /* Built-in unlikely() function */ -#define SQLITE_FUNC_CONSTANT 0x800 /* Constant inputs give a constant output */ -#define SQLITE_FUNC_MINMAX 0x1000 /* True for min() and max() aggregates */ +#define SQLITE_FUNC_ENCMASK 0x0003 /* SQLITE_UTF8, SQLITE_UTF16BE or UTF16LE */ +#define SQLITE_FUNC_LIKE 0x0004 /* Candidate for the LIKE optimization */ +#define SQLITE_FUNC_CASE 0x0008 /* Case-sensitive LIKE-type function */ +#define SQLITE_FUNC_EPHEM 0x0010 /* Ephemeral. Delete with VDBE */ +#define SQLITE_FUNC_NEEDCOLL 0x0020 /* sqlite3GetFuncCollSeq() might be called*/ +#define SQLITE_FUNC_LENGTH 0x0040 /* Built-in length() function */ +#define SQLITE_FUNC_TYPEOF 0x0080 /* Built-in typeof() function */ +#define SQLITE_FUNC_COUNT 0x0100 /* Built-in count(*) aggregate */ +#define SQLITE_FUNC_COALESCE 0x0200 /* Built-in coalesce() or ifnull() */ +#define SQLITE_FUNC_UNLIKELY 0x0400 /* Built-in unlikely() function */ +#define SQLITE_FUNC_CONSTANT 0x0800 /* Constant inputs give a constant output */ +#define SQLITE_FUNC_MINMAX 0x1000 /* True for min() and max() aggregates */ +#define SQLITE_FUNC_DATETIME 0x2000 /* Function returns a date and/or time */ /* ** The following three macros, FUNCTION(), LIKEFUNC() and AGGREGATE() are @@ -1411,6 +1412,10 @@ struct FuncDestructor { ** VFUNCTION(zName, nArg, iArg, bNC, xFunc) ** Like FUNCTION except it omits the SQLITE_FUNC_CONSTANT flag. ** +** DFUNCTION(zName, nArg, iArg, bNC, xFunc) +** Like FUNCTION except it omits the SQLITE_FUNC_CONSTANT flag and +** adds the SQLITE_FUNC_DATETIME flag. Used for date & time functions. +** ** AGGREGATE(zName, nArg, iArg, bNC, xStep, xFinal) ** Used to create an aggregate function definition implemented by ** the C functions xStep and xFinal. The first four parameters @@ -1431,6 +1436,9 @@ struct FuncDestructor { #define VFUNCTION(zName, nArg, iArg, bNC, xFunc) \ {nArg, SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0, 0} +#define DFUNCTION(zName, nArg, iArg, bNC, xFunc) \ + {nArg, SQLITE_FUNC_DATETIME|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ + SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0, 0} #define FUNCTION2(zName, nArg, iArg, bNC, xFunc, extraFlags) \ {nArg,SQLITE_FUNC_CONSTANT|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL)|extraFlags,\ SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0, 0} @@ -2121,7 +2129,7 @@ struct Expr { #define EP_MemToken 0x010000 /* Need to sqlite3DbFree() Expr.zToken */ #define EP_NoReduce 0x020000 /* Cannot EXPRDUP_REDUCE this Expr */ #define EP_Unlikely 0x040000 /* unlikely() or likelihood() function */ -#define EP_ConstFunc 0x080000 /* Node is a SQLITE_FUNC_CONSTANT function */ +#define EP_ConstFunc 0x080000 /* A SQLITE_FUNC_CONSTANT or _DATETIME function */ #define EP_CanBeNull 0x100000 /* Can be null despite NOT NULL constraint */ #define EP_Subquery 0x200000 /* Tree contains a TK_SELECT operator */ diff --git a/test/indexexpr1.test b/test/indexexpr1.test index c912226ce5..0b3e7c6603 100644 --- a/test/indexexpr1.test +++ b/test/indexexpr1.test @@ -18,28 +18,131 @@ source $testdir/tester.tcl do_execsql_test indexexpr1-100 { CREATE TABLE t1(a,b,c); INSERT INTO t1(a,b,c) - VALUES('In the beginning was the Word',1,1), - ('and the Word was with God',1,2), - ('and the Word was God',1,3), - ('The same was in the beginning with God',2,1), - ('All things were made by him',3,1), - ('and without him was not any thing made that was made',3,2); + VALUES('In_the_beginning_was_the_Word',1,1), + ('and_the_Word_was_with_God',1,2), + ('and_the_Word_was_God',1,3), + ('The_same_was_in_the_beginning_with_God',2,1), + ('All_things_were_made_by_him',3,1), + ('and_without_him_was_not_any_thing_made_that_was_made',3,2); CREATE INDEX t1a1 ON t1(substr(a,1,12)); } {} do_execsql_test indexexpr1-110 { - SELECT b, c, '|' FROM t1 WHERE substr(a,1,12)=='and the Word' ORDER BY b, c; + SELECT b, c, '|' FROM t1 WHERE substr(a,1,12)=='and_the_Word' ORDER BY b, c; } {1 2 | 1 3 |} do_execsql_test indexexpr1-110eqp { EXPLAIN QUERY PLAN - SELECT b, c, '|' FROM t1 WHERE substr(a,1,12)=='and the Word' ORDER BY b, c; + SELECT b, c, '|' FROM t1 WHERE substr(a,1,12)=='and_the_Word' ORDER BY b, c; } {/USING INDEX t1a1/} do_execsql_test indexexpr1-120 { - SELECT b, c, '|' FROM t1 WHERE 'and the Word'==substr(a,1,12) ORDER BY b, c; + SELECT b, c, '|' FROM t1 WHERE 'and_the_Word'==substr(a,1,12) ORDER BY b, c; } {1 2 | 1 3 |} do_execsql_test indexexpr1-120eqp { EXPLAIN QUERY PLAN - SELECT b, c, '|' FROM t1 WHERE 'and the Word'==substr(a,1,12) ORDER BY b, c; + SELECT b, c, '|' FROM t1 WHERE 'and_the_Word'==substr(a,1,12) ORDER BY b, c; } {/USING INDEX t1a1/} +do_execsql_test indexexpr1-130 { + CREATE INDEX t1ba ON t1(b,substr(a,2,3),c); + SELECT c FROM t1 WHERE b=1 AND substr(a,2,3)='nd_' ORDER BY c; +} {2 3} +do_execsql_test indexexpr1-130eqp { + EXPLAIN QUERY PLAN + SELECT c FROM t1 WHERE b=1 AND substr(a,2,3)='nd_' ORDER BY c; +} {/USING INDEX t1ba/} + +do_execsql_test indexexpr1-140 { + SELECT rowid, substr(a,b,3), '|' FROM t1 ORDER BY 2; +} {1 In_ | 2 and | 3 and | 6 d_w | 4 he_ | 5 l_t |} +do_execsql_test indexexpr1-141 { + CREATE INDEX t1abx ON t1(substr(a,b,3)); + SELECT rowid FROM t1 WHERE substr(a,b,3)<='and' ORDER BY +rowid; +} {1 2 3} +do_execsql_test indexexpr1-141eqp { + EXPLAIN QUERY PLAN + SELECT rowid FROM t1 WHERE substr(a,b,3)<='and' ORDER BY +rowid; +} {/USING INDEX t1abx/} +do_execsql_test indexexpr1-142 { + SELECT rowid FROM t1 WHERE +substr(a,b,3)<='and' ORDER BY +rowid; +} {1 2 3} +do_execsql_test indexexpr1-150 { + SELECT rowid FROM t1 WHERE substr(a,b,3) IN ('and','l_t','xyz') + ORDER BY +rowid; +} {2 3 5} +do_execsql_test indexexpr1-150eqp { + EXPLAIN QUERY PLAN + SELECT rowid FROM t1 WHERE substr(a,b,3) IN ('and','l_t','xyz') + ORDER BY +rowid; +} {/USING INDEX t1abx/} + +do_execsql_test indexexpr1-200 { + DROP TABLE t1; + CREATE TABLE t1(id ANY PRIMARY KEY, a,b,c) WITHOUT ROWID; + INSERT INTO t1(id,a,b,c) + VALUES(1,'In_the_beginning_was_the_Word',1,1), + (2,'and_the_Word_was_with_God',1,2), + (3,'and_the_Word_was_God',1,3), + (4,'The_same_was_in_the_beginning_with_God',2,1), + (5,'All_things_were_made_by_him',3,1), + (6,'and_without_him_was_not_any_thing_made_that_was_made',3,2); + CREATE INDEX t1a1 ON t1(substr(a,1,12)); +} {} +do_execsql_test indexexpr1-210 { + SELECT b, c, '|' FROM t1 WHERE substr(a,1,12)=='and_the_Word' ORDER BY b, c; +} {1 2 | 1 3 |} +do_execsql_test indexexpr1-210eqp { + EXPLAIN QUERY PLAN + SELECT b, c, '|' FROM t1 WHERE substr(a,1,12)=='and_the_Word' ORDER BY b, c; +} {/USING INDEX t1a1/} +do_execsql_test indexexpr1-220 { + SELECT b, c, '|' FROM t1 WHERE 'and_the_Word'==substr(a,1,12) ORDER BY b, c; +} {1 2 | 1 3 |} +do_execsql_test indexexpr1-220eqp { + EXPLAIN QUERY PLAN + SELECT b, c, '|' FROM t1 WHERE 'and_the_Word'==substr(a,1,12) ORDER BY b, c; +} {/USING INDEX t1a1/} + +do_execsql_test indexexpr1-230 { + CREATE INDEX t1ba ON t1(b,substr(a,2,3),c); + SELECT c FROM t1 WHERE b=1 AND substr(a,2,3)='nd_' ORDER BY c; +} {2 3} +do_execsql_test indexexpr1-230eqp { + EXPLAIN QUERY PLAN + SELECT c FROM t1 WHERE b=1 AND substr(a,2,3)='nd_' ORDER BY c; +} {/USING INDEX t1ba/} + +do_execsql_test indexexpr1-240 { + SELECT id, substr(a,b,3), '|' FROM t1 ORDER BY 2; +} {1 In_ | 2 and | 3 and | 6 d_w | 4 he_ | 5 l_t |} +do_execsql_test indexexpr1-241 { + CREATE INDEX t1abx ON t1(substr(a,b,3)); + SELECT id FROM t1 WHERE substr(a,b,3)<='and' ORDER BY +id; +} {1 2 3} +do_execsql_test indexexpr1-241eqp { + EXPLAIN QUERY PLAN + SELECT id FROM t1 WHERE substr(a,b,3)<='and' ORDER BY +id; +} {/USING INDEX t1abx/} +do_execsql_test indexexpr1-242 { + SELECT id FROM t1 WHERE +substr(a,b,3)<='and' ORDER BY +id; +} {1 2 3} +do_execsql_test indexexpr1-250 { + SELECT id FROM t1 WHERE substr(a,b,3) IN ('and','l_t','xyz') + ORDER BY +id; +} {2 3 5} +do_execsql_test indexexpr1-250eqp { + EXPLAIN QUERY PLAN + SELECT id FROM t1 WHERE substr(a,b,3) IN ('and','l_t','xyz') + ORDER BY +id; +} {/USING INDEX t1abx/} + +do_catchsql_test indexexpr1-300 { + CREATE TABLE t2(a,b,c); + CREATE INDEX t2x1 ON t2(a,b+random()); +} {1 {non-deterministic functions prohibited in index expressions}} +do_catchsql_test indexexpr1-301 { + CREATE INDEX t2x1 ON t2(a+julianday('now')); +} {1 {non-deterministic functions prohibited in index expressions}} +do_catchsql_test indexexpr1-310 { + CREATE INDEX t2x2 ON t2(a,b+(SELECT 15)); +} {1 {subqueries prohibited in index expressions}} finish_test From 390b88a448d219a25498a2cdb379f45a914b006d Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 31 Aug 2015 18:13:01 +0000 Subject: [PATCH 012/100] Case should not be significant when comparing function names. FossilOrigin-Name: e2f1caf117b0a9632d52246717ab202852982339 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/expr.c | 4 +++- test/indexexpr1.test | 41 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 52 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index bcc1d9de6b..3215993778 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\sthe\sdistinction\sbetween\struly\sdeterministic\sfunctions\sand\sdate/time\nfunctions\swhich\sonly\sreturn\sthe\ssame\sanswer\sfor\sa\ssingle\squery.\s\sOnly\struly\ndeterministic\sfunctions\sare\sallowed\sin\sindexes.\s\sAdd\snew\sexpression\sindex\ntest\scases. -D 2015-08-31T17:34:41.308 +C Case\sshould\snot\sbe\ssignificant\swhen\scomparing\sfunction\snames. +D 2015-08-31T18:13:01.161 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/ctime.c 5a0b735dc95604766f5dac73973658eef782ee8b F src/date.c fb1c99172017dcc8e237339132c91a21a0788584 F src/dbstat.c f402e77e25089c6003d0c60b3233b9b3947d599a F src/delete.c 224e5c9a0eae7fbdd3b461ca971465c992851354 -F src/expr.c cf0c5ef6467d3eb65e3ff5c88eb2f602ba3b5163 +F src/expr.c 500c0ee6ac2e5d60c10df62b8c9b96dc75c2cbb1 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 3ce33dd49f12c72376cec9adc7a4d8e7111cedcc F src/func.c 824bea430d3a2b7dbc62806ad54da8fdb8ed9e3f @@ -780,7 +780,7 @@ F test/index5.test 8621491915800ec274609e42e02a97d67e9b13e7 F test/index6.test 7102ec371414c42dfb1d5ca37eb4519aa9edc23a F test/index7.test 9c6765a74fc3fcde7aebc5b3bd40d98df14a527c F test/indexedby.test 5f527a78bae74c61b8046ae3037f9dfb0bf0c353 -F test/indexexpr1.test 1e57b4bb93b65a451803e7547a51f685af1c35db +F test/indexexpr1.test d04a3905275fb9a0fb82447e7d43773f98fad282 F test/indexfault.test 31d4ab9a7d2f6e9616933eb079722362a883eb1d F test/init.test 15c823093fdabbf7b531fe22cf037134d09587a7 F test/insert.test 38742b5e9601c8f8d76e9b7555f7270288c2d371 @@ -1381,7 +1381,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f8893696387cba9d293a05a68dc38228077b3dc5 -R 73b6719fd207957d3d30c3e1b6c218bc +P c77554b5c42327106a7b90334e9cc3c07b007c76 +R 0c8c1c66a00ef8c41547d29bee27b359 U drh -Z fe2263cb4a747333f78b4cc6eb661f66 +Z 0525f466a18434624bccbe0dbed916be diff --git a/manifest.uuid b/manifest.uuid index 3dc8f017b9..51f3249ab6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c77554b5c42327106a7b90334e9cc3c07b007c76 \ No newline at end of file +e2f1caf117b0a9632d52246717ab202852982339 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 9ae611ba7c..ee3fb1a940 100644 --- a/src/expr.c +++ b/src/expr.c @@ -3788,7 +3788,9 @@ int sqlite3ExprCompare(Expr *pA, Expr *pB, int iTab){ return 2; } if( pA->op!=TK_COLUMN && ALWAYS(pA->op!=TK_AGG_COLUMN) && pA->u.zToken ){ - if( strcmp(pA->u.zToken,pB->u.zToken)!=0 ){ + if( pA->op==TK_FUNCTION ){ + if( sqlite3StrICmp(pA->u.zToken,pB->u.zToken)!=0 ) return 2; + }else if( strcmp(pA->u.zToken,pB->u.zToken)!=0 ){ return pA->op==TK_COLLATE ? 1 : 2; } } diff --git a/test/indexexpr1.test b/test/indexexpr1.test index 0b3e7c6603..aa056e55d4 100644 --- a/test/indexexpr1.test +++ b/test/indexexpr1.test @@ -18,6 +18,7 @@ source $testdir/tester.tcl do_execsql_test indexexpr1-100 { CREATE TABLE t1(a,b,c); INSERT INTO t1(a,b,c) + /* 123456789 123456789 123456789 123456789 123456789 123456789 */ VALUES('In_the_beginning_was_the_Word',1,1), ('and_the_Word_was_with_God',1,2), ('and_the_Word_was_God',1,3), @@ -74,6 +75,20 @@ do_execsql_test indexexpr1-150eqp { ORDER BY +rowid; } {/USING INDEX t1abx/} +do_execsql_test indexexpr1-160 { + ALTER TABLE t1 ADD COLUMN d; + UPDATE t1 SET d=length(a); + CREATE INDEX t1a2 ON t1(SUBSTR(a, 27, 3)) WHERE d>=29; + SELECT rowid, b, c FROM t1 + WHERE substr(a,27,3)=='ord' AND d>=29; +} {1 1 1} +do_execsql_test indexexpr1-160eqp { + EXPLAIN QUERY PLAN + SELECT rowid, b, c FROM t1 + WHERE substr(a,27,3)=='ord' AND d>=29; +} {/USING INDEX t1a2/} + + do_execsql_test indexexpr1-200 { DROP TABLE t1; CREATE TABLE t1(id ANY PRIMARY KEY, a,b,c) WITHOUT ROWID; @@ -134,6 +149,20 @@ do_execsql_test indexexpr1-250eqp { ORDER BY +id; } {/USING INDEX t1abx/} +do_execsql_test indexexpr1-260 { + ALTER TABLE t1 ADD COLUMN d; + UPDATE t1 SET d=length(a); + CREATE INDEX t1a2 ON t1(SUBSTR(a, 27, 3)) WHERE d>=29; + SELECT id, b, c FROM t1 + WHERE substr(a,27,3)=='ord' AND d>=29; +} {1 1 1} +do_execsql_test indexexpr1-260eqp { + EXPLAIN QUERY PLAN + SELECT id, b, c FROM t1 + WHERE substr(a,27,3)=='ord' AND d>=29; +} {/USING INDEX t1a2/} + + do_catchsql_test indexexpr1-300 { CREATE TABLE t2(a,b,c); CREATE INDEX t2x1 ON t2(a,b+random()); @@ -144,5 +173,17 @@ do_catchsql_test indexexpr1-301 { do_catchsql_test indexexpr1-310 { CREATE INDEX t2x2 ON t2(a,b+(SELECT 15)); } {1 {subqueries prohibited in index expressions}} +do_catchsql_test indexexpr1-320 { + CREATE TABLE e1(x,y,UNIQUE(y,substr(x,1,5))); +} {1 {expressions prohibited in PRIMARY KEY and UNIQUE constraints}} +do_catchsql_test indexexpr1-330 { + CREATE TABLE e1(x,y,PRIMARY KEY(y,substr(x,1,5))); +} {1 {expressions prohibited in PRIMARY KEY and UNIQUE constraints}} +do_catchsql_test indexexpr1-331 { + CREATE TABLE e1(x,y,PRIMARY KEY(y,substr(x,1,5))) WITHOUT ROWID; +} {1 {expressions prohibited in PRIMARY KEY and UNIQUE constraints}} +do_catchsql_test indexexpr1-340 { + CREATE TABLE e1(x,y,FOREIGN KEY(substr(y,1,5)) REFERENCES t1); +} {1 {near "(": syntax error}} finish_test From 8492653c37795ec45673d22ee8bb5e8b3e848653 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 31 Aug 2015 19:38:42 +0000 Subject: [PATCH 013/100] Always assume that indexed expressions can generate a NULL. Get indexed expressions working for the case of two or more expressions in the same index. FossilOrigin-Name: cc60321a67bf9f169c090b47afb505f589a6925e --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/build.c | 11 +++++++---- src/where.c | 2 +- 4 files changed, 16 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 3215993778..392a86929f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Case\sshould\snot\sbe\ssignificant\swhen\scomparing\sfunction\snames. -D 2015-08-31T18:13:01.161 +C Always\sassume\sthat\sindexed\sexpressions\scan\sgenerate\sa\sNULL.\s\sGet\nindexed\sexpressions\sworking\sfor\sthe\scase\sof\stwo\sor\smore\sexpressions\sin\nthe\ssame\sindex. +D 2015-08-31T19:38:42.715 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -282,7 +282,7 @@ F src/btmutex.c 45a968cc85afed9b5e6cf55bf1f42f8d18107f79 F src/btree.c f48b3ef91676c06a90a8832987ecef6b94c931ee F src/btree.h 969adc948e89e449220ff0ff724c94bb2a52e9f1 F src/btreeInt.h 8177c9ab90d772d6d2c6c517e05bed774b7c92c0 -F src/build.c e0902658fc86dbd60a5c6772ca45429c69ee81fe +F src/build.c 9e7216acaf612d371c1b4a6bf9711886472f7ac6 F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 F src/complete.c addcd8160b081131005d5bc2d34adf20c1c5c92f F src/ctime.c 5a0b735dc95604766f5dac73973658eef782ee8b @@ -414,7 +414,7 @@ F src/vxworks.h c18586c8edc1bddbc15c004fa16aeb1e1342b4fb F src/wal.c 8cd07f1f99e1a81346db1c9da879bef6c6f97cf6 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 2e14d17f592d176b6dc879c33fbdec4fbccaa2ba -F src/where.c 385f927f06a89d06de6a4ce7627fbd1684f157bb +F src/where.c 04e0f87e4d82a85c283176958465bed6ea5434e0 F src/whereInt.h 292d3ac90da4eab1e03ac8452f1add746bcafaa1 F src/wherecode.c b0bf45ca49e62fde68ba2e2ad2939d9cdeb4e409 F src/whereexpr.c 2473e4350e30f9b55d1c6a8f66ca23c689f23f1d @@ -1381,7 +1381,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P c77554b5c42327106a7b90334e9cc3c07b007c76 -R 0c8c1c66a00ef8c41547d29bee27b359 +P e2f1caf117b0a9632d52246717ab202852982339 +R 9a8972907f2a35dc9161585e1e0d88b8 U drh -Z 0525f466a18434624bccbe0dbed916be +Z 7baf4f251bc37ea5f508ac4ed0b603b4 diff --git a/manifest.uuid b/manifest.uuid index 51f3249ab6..693f9f59dd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e2f1caf117b0a9632d52246717ab202852982339 \ No newline at end of file +cc60321a67bf9f169c090b47afb505f589a6925e \ No newline at end of file diff --git a/src/build.c b/src/build.c index 59c2a73021..f217ab251a 100644 --- a/src/build.c +++ b/src/build.c @@ -3126,13 +3126,16 @@ Index *sqlite3CreateIndex( goto exit_create_index; } if( pIndex->aColExpr==0 ){ - pIndex->aColExpr = sqlite3ExprListDup(db, pList, 0); + ExprList *pCopy = sqlite3ExprListDup(db, pList, 0); + pIndex->aColExpr = pCopy; + if( !db->mallocFailed ){ + assert( pCopy!=0 ); + pListItem = &pCopy->a[i]; + } } j = -2; pIndex->aiColumn[i] = -2; - if( sqlite3ExprCanBeNull(pList->a[i].pExpr) ){ - pIndex->uniqNotNull = 1; - } + pIndex->uniqNotNull = 0; }else{ j = pCExpr->iColumn; assert( j<=0x7fff ); diff --git a/src/where.c b/src/where.c index f39afb7e3a..6dc326be0d 100644 --- a/src/where.c +++ b/src/where.c @@ -395,7 +395,7 @@ static int indexColumnNotNull(Index *pIdx, int iCol){ return 1; }else{ assert( j==(-2) ); - return !sqlite3ExprCanBeNull(pIdx->aColExpr->a[iCol].pExpr); + return 0; /* Assume an indexed expression can always yield a NULL */ } } From cb8b58caf79f42bd5d067f8673d339ff6c0c64cc Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 31 Aug 2015 20:06:06 +0000 Subject: [PATCH 014/100] Begin changes to allow synonym support by adding multiple terms to a query (an alternative to adding multiple terms to the FTS index). FossilOrigin-Name: ad7feaed4cd6b1d6e6376bb82d1f5664ddd083f3 --- ext/fts5/fts5Int.h | 2 +- ext/fts5/fts5_expr.c | 284 ++++++++++++++++++++++++++------- ext/fts5/test/fts5aa.test | 2 +- ext/fts5/test/fts5ea.test | 6 + ext/fts5/test/fts5eb.test | 24 +-- ext/fts5/test/fts5synonym.test | 61 ++++--- manifest | 22 +-- manifest.uuid | 2 +- 8 files changed, 297 insertions(+), 106 deletions(-) diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index c61f8c4052..e8d997b060 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -236,7 +236,7 @@ struct Fts5PoslistReader { int i; /* Current offset in a[] */ /* Output variables */ - int bEof; /* Set to true at EOF */ + u8 bEof; /* Set to true at EOF */ i64 iPos; /* (iCol<<32) + iPos */ }; int sqlite3Fts5PoslistReaderInit( diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index 9b52bb08c6..bba9932daf 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -73,6 +73,7 @@ struct Fts5ExprTerm { int bPrefix; /* True for a prefix term */ char *zTerm; /* nul-terminated term */ Fts5IndexIter *pIter; /* Iterator for this term */ + Fts5ExprTerm *pSynonym; /* Pointer to first in list of synonyms */ }; /* @@ -181,6 +182,10 @@ static int fts5ExprGetToken( default: { const char *z2; + if( sqlite3Fts5IsBareword(z[0])==0 ){ + sqlite3Fts5ParseError(pParse, "fts5: syntax error near \"%.1s\"", z); + return FTS5_EOF; + } tok = FTS5_STRING; for(z2=&z[1]; sqlite3Fts5IsBareword(*z2); z2++); pToken->n = (z2 - z); @@ -350,6 +355,60 @@ static int fts5ExprColsetTest(Fts5ExprColset *pColset, int iCol){ return 0; } +/* +** Argument pTerm must be a synonym iterator. Return the current rowid +** that it points to. +*/ +static i64 fts5ExprSynonymRowid(Fts5ExprTerm *pTerm, int bDesc){ + i64 iRet; + int bRetValid = 0; + Fts5ExprTerm *p; + + assert( pTerm->pSynonym ); + assert( bDesc==0 || bDesc==1 ); + for(p=pTerm; p; p=p->pSynonym){ + if( 0==sqlite3Fts5IterEof(p->pIter) ){ + i64 iRowid = sqlite3Fts5IterRowid(p->pIter); + if( bRetValid==0 || (bDesc!=(iRowidpSynonym ); + for(p=pTerm; p; p=p->pSynonym){ + Fts5IndexIter *pIter = p->pIter; + if( sqlite3Fts5IterEof(pIter)==0 && sqlite3Fts5IterRowid(pIter)==iRowid ){ + i64 dummy; + int rc = sqlite3Fts5IterPoslist(pIter, (const u8**)pa, pn, &dummy); + return rc; + } + } + + assert( 0 ); + return SQLITE_ERROR; +} + + /* ** All individual term iterators in pPhrase are guaranteed to be valid and ** pointing to the same rowid when this function is called. This function @@ -362,7 +421,7 @@ static int fts5ExprColsetTest(Fts5ExprColset *pColset, int iCol){ ** not a match. */ static int fts5ExprPhraseIsMatch( - Fts5Expr *pExpr, /* Expression pPhrase belongs to */ + Fts5ExprNode *pNode, /* Node pPhrase belongs to */ Fts5ExprColset *pColset, /* Restrict matches to these columns */ Fts5ExprPhrase *pPhrase, /* Phrase object to initialize */ int *pbMatch /* OUT: Set to true if really a match */ @@ -391,10 +450,15 @@ static int fts5ExprPhraseIsMatch( /* Initialize a term iterator for each term in the phrase */ for(i=0; inTerm; i++){ + Fts5ExprTerm *pTerm = &pPhrase->aTerm[i]; i64 dummy; int n; const u8 *a; - rc = sqlite3Fts5IterPoslist(pPhrase->aTerm[i].pIter, &a, &n, &dummy); + if( pTerm->pSynonym ){ + rc = fts5ExprSynonymPoslist(pTerm, pNode->iRowid, (u8**)&a, &n); + }else{ + rc = sqlite3Fts5IterPoslist(pTerm->pIter, &a, &n, &dummy); + } if( rc || sqlite3Fts5PoslistReaderInit(iCol, a, n, &aIter[i]) ){ goto ismatch_out; } @@ -598,17 +662,43 @@ static int fts5ExprNearAdvanceFirst( int bFromValid, i64 iFrom ){ - Fts5IndexIter *pIter = pNode->pNear->apPhrase[0]->aTerm[0].pIter; + Fts5ExprTerm *pTerm = &pNode->pNear->apPhrase[0]->aTerm[0]; int rc; - assert( Fts5NodeIsString(pNode) ); - if( bFromValid ){ - rc = sqlite3Fts5IterNextFrom(pIter, iFrom); + if( pTerm->pSynonym ){ + int bEof = 1; + Fts5ExprTerm *p; + + /* Find the firstest rowid any synonym points to. */ + i64 iRowid = fts5ExprSynonymRowid(pTerm, pExpr->bDesc); + + /* Advance each iterator that currently points to iRowid */ + for(p=pTerm; p; p=p->pSynonym){ + if( sqlite3Fts5IterEof(p->pIter)==0 ){ + bEof = 0; + if( sqlite3Fts5IterRowid(p->pIter)==iRowid ){ + rc = sqlite3Fts5IterNext(p->pIter); + if( rc!=SQLITE_OK ) break; + } + } + } + + /* Set the EOF flag if either all synonym iterators are at EOF or an + ** error has occurred. */ + pNode->bEof = (rc || bEof); }else{ - rc = sqlite3Fts5IterNext(pIter); + Fts5IndexIter *pIter = pTerm->pIter; + + assert( Fts5NodeIsString(pNode) ); + if( bFromValid ){ + rc = sqlite3Fts5IterNextFrom(pIter, iFrom); + }else{ + rc = sqlite3Fts5IterNext(pIter); + } + + pNode->bEof = (rc || sqlite3Fts5IterEof(pIter)); } - pNode->bEof = (rc || sqlite3Fts5IterEof(pIter)); return rc; } @@ -719,7 +809,7 @@ static int fts5ExprNearTest( Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; if( pPhrase->nTerm>1 || pNear->pColset ){ int bMatch = 0; - rc = fts5ExprPhraseIsMatch(pExpr, pNear->pColset, pPhrase, &bMatch); + rc = fts5ExprPhraseIsMatch(pNode, pNear->pColset, pPhrase, &bMatch); if( bMatch==0 ) break; }else{ rc = sqlite3Fts5IterPoslistBuffer( @@ -755,6 +845,7 @@ static int fts5ExprTokenTest( assert( pNode->eType==FTS5_TERM ); assert( pNear->nPhrase==1 && pPhrase->nTerm==1 ); + assert( pPhrase->aTerm[0].pSynonym==0 ); rc = sqlite3Fts5IterPoslist(pIter, &pPos, &nPos, &pNode->iRowid); @@ -801,69 +892,107 @@ static int fts5ExprNearNextMatch( i64 iLast; /* Lastest rowid any iterator points to */ int i, j; /* Phrase and token index, respectively */ int bMatch; /* True if all terms are at the same rowid */ + const int bDesc = pExpr->bDesc; - assert( pNear->nPhrase>1 || pNear->apPhrase[0]->nTerm>1 ); + /* Check that this node should not be FTS5_TERM */ + assert( pNear->nPhrase>1 + || pNear->apPhrase[0]->nTerm>1 + || pNear->apPhrase[0]->aTerm[0].pSynonym + ); /* Initialize iLast, the "lastest" rowid any iterator points to. If the ** iterator skips through rowids in the default ascending order, this means ** the maximum rowid. Or, if the iterator is "ORDER BY rowid DESC", then it ** means the minimum rowid. */ - iLast = sqlite3Fts5IterRowid(pLeft->aTerm[0].pIter); + if( pLeft->aTerm[0].pSynonym ){ + iLast = fts5ExprSynonymRowid(&pLeft->aTerm[0], bDesc); + }else{ + iLast = sqlite3Fts5IterRowid(pLeft->aTerm[0].pIter); + } do { bMatch = 1; for(i=0; inPhrase; i++){ Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; for(j=0; jnTerm; j++){ - Fts5IndexIter *pIter = pPhrase->aTerm[j].pIter; - i64 iRowid = sqlite3Fts5IterRowid(pIter); - if( iRowid!=iLast ) bMatch = 0; - if( fts5ExprAdvanceto(pIter, pExpr->bDesc, &iLast,&rc,&pNode->bEof) ){ - return rc; + Fts5ExprTerm *pTerm = &pPhrase->aTerm[j]; + if( pTerm->pSynonym ){ + Fts5ExprTerm *p; + int bEof = 1; + i64 iRowid = fts5ExprSynonymRowid(pTerm, bDesc); + if( iRowid==iLast ) continue; + for(p=pTerm; p; p=p->pSynonym){ + Fts5IndexIter *pIter = p->pIter; + int dummy; + if( 0==sqlite3Fts5IterEof(pIter) + && 0==fts5ExprAdvanceto(pIter, bDesc, &iLast, &rc, &dummy)==0 + ){ + bEof = 0; + } + } + if( bEof || rc ){ + pNode->bEof = 1; + return rc; + } + }else{ + Fts5IndexIter *pIter = pPhrase->aTerm[j].pIter; + i64 iRowid = sqlite3Fts5IterRowid(pIter); + if( iRowid==iLast ) continue; + bMatch = 0; + if( fts5ExprAdvanceto(pIter, bDesc, &iLast, &rc, &pNode->bEof) ){ + return rc; + } } } } }while( bMatch==0 ); - pNode->bNomatch = (0==fts5ExprNearTest(&rc, pExpr, pNode)); pNode->iRowid = iLast; + pNode->bNomatch = (0==fts5ExprNearTest(&rc, pExpr, pNode)); return rc; } /* ** Initialize all term iterators in the pNear object. If any term is found -** to match no documents at all, set *pbEof to true and return immediately, -** without initializing any further iterators. +** to match no documents at all, return immediately without initializing any +** further iterators. */ static int fts5ExprNearInitAll( Fts5Expr *pExpr, Fts5ExprNode *pNode ){ Fts5ExprNearset *pNear = pNode->pNear; - Fts5ExprTerm *pTerm; - Fts5ExprPhrase *pPhrase; int i, j; int rc = SQLITE_OK; for(i=0; rc==SQLITE_OK && inPhrase; i++){ - pPhrase = pNear->apPhrase[i]; + Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; for(j=0; jnTerm; j++){ - pTerm = &pPhrase->aTerm[j]; - if( pTerm->pIter ){ - sqlite3Fts5IterClose(pTerm->pIter); - pTerm->pIter = 0; + Fts5ExprTerm *pTerm = &pPhrase->aTerm[j]; + Fts5ExprTerm *p; + int bEof = 1; + + for(p=pTerm; p && rc==SQLITE_OK; p=p->pSynonym){ + if( p->pIter ){ + sqlite3Fts5IterClose(p->pIter); + p->pIter = 0; + } + rc = sqlite3Fts5IndexQuery( + pExpr->pIndex, p->zTerm, strlen(p->zTerm), + (pTerm->bPrefix ? FTS5INDEX_QUERY_PREFIX : 0) | + (pExpr->bDesc ? FTS5INDEX_QUERY_DESC : 0), + &p->pIter + ); + assert( rc==SQLITE_OK || p->pIter==0 ); + if( p->pIter && 0==sqlite3Fts5IterEof(pTerm->pIter) ){ + bEof = 0; + } } - rc = sqlite3Fts5IndexQuery( - pExpr->pIndex, pTerm->zTerm, strlen(pTerm->zTerm), - (pTerm->bPrefix ? FTS5INDEX_QUERY_PREFIX : 0) | - (pExpr->bDesc ? FTS5INDEX_QUERY_DESC : 0), - &pTerm->pIter - ); - assert( rc==SQLITE_OK || pTerm->pIter==0 ); - if( pTerm->pIter==0 || sqlite3Fts5IterEof(pTerm->pIter) ){ + + if( bEof ){ pNode->bEof = 1; - break; + return rc; } } } @@ -1266,10 +1395,16 @@ static void fts5ExprPhraseFree(Fts5ExprPhrase *pPhrase){ if( pPhrase ){ int i; for(i=0; inTerm; i++){ + Fts5ExprTerm *pSyn; + Fts5ExprTerm *pNext; Fts5ExprTerm *pTerm = &pPhrase->aTerm[i]; sqlite3_free(pTerm->zTerm); - if( pTerm->pIter ){ - sqlite3Fts5IterClose(pTerm->pIter); + sqlite3Fts5IterClose(pTerm->pIter); + + for(pSyn=pTerm->pSynonym; pSyn; pSyn=pNext){ + pNext = pSyn->pSynonym; + sqlite3Fts5IterClose(pSyn->pIter); + sqlite3_free(pSyn); } } if( pPhrase->poslist.nSpace>0 ) fts5BufferFree(&pPhrase->poslist); @@ -1348,27 +1483,37 @@ static int fts5ParseTokenize( const int SZALLOC = 8; TokenCtx *pCtx = (TokenCtx*)pContext; Fts5ExprPhrase *pPhrase = pCtx->pPhrase; - Fts5ExprTerm *pTerm; - if( tflags & FTS5_TOKEN_COLOCATED ) return rc; + assert( pPhrase==0 || pPhrase->nTerm>0 ); + if( pPhrase && (tflags & FTS5_TOKEN_COLOCATED) ){ + Fts5ExprTerm *pSyn; + int nByte = sizeof(Fts5ExprTerm) + nToken+1; + pSyn = (Fts5ExprTerm*)sqlite3_malloc(nByte); + if( pSyn==0 ) return SQLITE_NOMEM; + memset(pSyn, 0, nByte); + pSyn->zTerm = (char*)&pSyn[1]; + memcpy(pSyn->zTerm, pToken, nToken); + pSyn->pSynonym = pPhrase->aTerm[pPhrase->nTerm-1].pSynonym; + pPhrase->aTerm[pPhrase->nTerm-1].pSynonym = pSyn; + }else{ + Fts5ExprTerm *pTerm; + if( pPhrase==0 || (pPhrase->nTerm % SZALLOC)==0 ){ + Fts5ExprPhrase *pNew; + int nNew = SZALLOC + (pPhrase ? pPhrase->nTerm : 0); - if( pPhrase==0 || (pPhrase->nTerm % SZALLOC)==0 ){ - Fts5ExprPhrase *pNew; - int nNew = SZALLOC + (pPhrase ? pPhrase->nTerm : 0); + pNew = (Fts5ExprPhrase*)sqlite3_realloc(pPhrase, + sizeof(Fts5ExprPhrase) + sizeof(Fts5ExprTerm) * nNew + ); + if( pNew==0 ) return SQLITE_NOMEM; + if( pPhrase==0 ) memset(pNew, 0, sizeof(Fts5ExprPhrase)); + pCtx->pPhrase = pPhrase = pNew; + pNew->nTerm = nNew - SZALLOC; + } - pNew = (Fts5ExprPhrase*)sqlite3_realloc(pPhrase, - sizeof(Fts5ExprPhrase) + sizeof(Fts5ExprTerm) * nNew - ); - if( pNew==0 ) return SQLITE_NOMEM; - if( pPhrase==0 ) memset(pNew, 0, sizeof(Fts5ExprPhrase)); - pCtx->pPhrase = pPhrase = pNew; - pNew->nTerm = nNew - SZALLOC; + pTerm = &pPhrase->aTerm[pPhrase->nTerm++]; + memset(pTerm, 0, sizeof(Fts5ExprTerm)); + pTerm->zTerm = sqlite3Fts5Strndup(&rc, pToken, nToken); } - - pTerm = &pPhrase->aTerm[pPhrase->nTerm++]; - memset(pTerm, 0, sizeof(Fts5ExprTerm)); - pTerm->zTerm = sqlite3Fts5Strndup(&rc, pToken, nToken); - return rc; } @@ -1636,7 +1781,10 @@ Fts5ExprNode *sqlite3Fts5ParseNode( for(iPhrase=0; iPhrasenPhrase; iPhrase++){ pNear->apPhrase[iPhrase]->pNode = pRet; } - if( pNear->nPhrase==1 && pNear->apPhrase[0]->nTerm==1 ){ + if( pNear->nPhrase==1 + && pNear->apPhrase[0]->nTerm==1 + && pNear->apPhrase[0]->aTerm[0].pSynonym==0 + ){ pRet->eType = FTS5_TERM; } }else{ @@ -1656,16 +1804,28 @@ Fts5ExprNode *sqlite3Fts5ParseNode( } static char *fts5ExprTermPrint(Fts5ExprTerm *pTerm){ - char *zQuoted = sqlite3_malloc(strlen(pTerm->zTerm) * 2 + 3 + 2); + int nByte = 0; + Fts5ExprTerm *p; + char *zQuoted; + + /* Determine the maximum amount of space required. */ + for(p=pTerm; p; p=p->pSynonym){ + nByte += strlen(pTerm->zTerm) * 2 + 3 + 2; + } + zQuoted = sqlite3_malloc(nByte); + if( zQuoted ){ int i = 0; - char *zIn = pTerm->zTerm; - zQuoted[i++] = '"'; - while( *zIn ){ - if( *zIn=='"' ) zQuoted[i++] = '"'; - zQuoted[i++] = *zIn++; + for(p=pTerm; p; p=p->pSynonym){ + char *zIn = p->zTerm; + zQuoted[i++] = '"'; + while( *zIn ){ + if( *zIn=='"' ) zQuoted[i++] = '"'; + zQuoted[i++] = *zIn++; + } + zQuoted[i++] = '"'; + if( p->pSynonym ) zQuoted[i++] = '|'; } - zQuoted[i++] = '"'; if( pTerm->bPrefix ){ zQuoted[i++] = ' '; zQuoted[i++] = '*'; diff --git a/ext/fts5/test/fts5aa.test b/ext/fts5/test/fts5aa.test index daa535cd9b..22aa1b0f35 100644 --- a/ext/fts5/test/fts5aa.test +++ b/ext/fts5/test/fts5aa.test @@ -343,7 +343,7 @@ do_execsql_test 13.5 { } {1} do_execsql_test 13.6 { - SELECT rowid FROM t1 WHERE t1 MATCH '.'; + SELECT rowid FROM t1 WHERE t1 MATCH '""'; } {} #------------------------------------------------------------------------- diff --git a/ext/fts5/test/fts5ea.test b/ext/fts5/test/fts5ea.test index ad05412ba9..3ccbd7d7a2 100644 --- a/ext/fts5/test/fts5ea.test +++ b/ext/fts5/test/fts5ea.test @@ -87,6 +87,12 @@ do_execsql_test 4.0 { SELECT fts5_expr('a AND """"', 'x', 'tokenize="unicode61 tokenchars ''""''"'); } {{"a" AND """"}} +#------------------------------------------------------------------------- +# Experiment with a tokenizer that considers " to be a token character. +# +do_catchsql_test 5.0 { + SELECT fts5_expr('abc | def'); +} {1 {fts5: syntax error near "|"}} diff --git a/ext/fts5/test/fts5eb.test b/ext/fts5/test/fts5eb.test index 352e1b4a17..55c4b15cf3 100644 --- a/ext/fts5/test/fts5eb.test +++ b/ext/fts5/test/fts5eb.test @@ -30,18 +30,18 @@ proc do_syntax_test {tn expr res} { } foreach {tn expr res} { - 1 {abc} {"abc"} - 2 {abc .} {"abc"} - 3 {.} {} - 4 {abc OR .} {"abc"} - 5 {abc NOT .} {"abc"} - 6 {abc AND .} {"abc"} - 7 {. OR abc} {"abc"} - 8 {. NOT abc} {"abc"} - 9 {. AND abc} {"abc"} - 10 {abc + . + def} {"abc" + "def"} - 11 {abc . def} {"abc" AND "def"} - 12 {r+e OR w} {"r" + "e" OR "w"} + 1 {abc} {"abc"} + 2 {abc ""} {"abc"} + 3 {""} {} + 4 {abc OR ""} {"abc"} + 5 {abc NOT ""} {"abc"} + 6 {abc AND ""} {"abc"} + 7 {"" OR abc} {"abc"} + 8 {"" NOT abc} {"abc"} + 9 {"" AND abc} {"abc"} + 10 {abc + "" + def} {"abc" + "def"} + 11 {abc "" def} {"abc" AND "def"} + 12 {r+e OR w} {"r" + "e" OR "w"} } { do_execsql_test 1.$tn {SELECT fts5_expr($expr)} [list $res] } diff --git a/ext/fts5/test/fts5synonym.test b/ext/fts5/test/fts5synonym.test index 812f7c5882..a607b7e29c 100644 --- a/ext/fts5/test/fts5synonym.test +++ b/ext/fts5/test/fts5synonym.test @@ -21,6 +21,24 @@ ifcapable !fts5 { return } +foreach S { + {zero 0} + {one 1} + {two 2} + {three 3 iii} + {four 4} + {five 5} + {six 6} + {seven 7} + {eight 8} + {nine 9} +} { + foreach s $S { + set o [list] + foreach x $S {if {$x!=$s} {lappend o $x}} + set ::syn($s) $o + } +} proc gobble_whitespace {textvar} { upvar $textvar t @@ -75,24 +93,6 @@ do_execsql_test 1.0 { # Test a tokenizer that supports synonyms by adding extra entries to the # FTS index. # -foreach S { - {zero 0} - {one 1} - {two 2} - {three 3 iii} - {four 4} - {five 5} - {six 6} - {seven 7} - {eight 8} - {nine 9} -} { - foreach s $S { - set o [list] - foreach x $S {if {$x!=$s} {lappend o $x}} - set ::syn($s) $o - } -} proc tcl_tokenize {tflags text} { foreach {w iStart iEnd} [do_tokenize_split $text] { @@ -194,5 +194,30 @@ do_execsql_test 3.2.2 { SELECT rowid FROM ft WHERE ft MATCH 'one + two + two + three'; } {1} +#------------------------------------------------------------------------- +# Check that expressions with synonyms can be parsed. +# +reset_db +sqlite3_fts5_create_tokenizer db tcl tcl_create +proc tcl_tokenize {tflags text} { + foreach {w iStart iEnd} [do_tokenize_split $text] { + sqlite3_fts5_token $w $iStart $iEnd + if {$tflags=="query" && [info exists ::syn($w)]} { + foreach s $::syn($w) { + sqlite3_fts5_token -colo $s $iStart $iEnd + } + } + } +} + +foreach {tn expr res} { + 1 {abc} {"abc"} + 2 {one} {"one"|"1"} + 3 {3} {"3"|"iii"|"three"} + 4 {3*} {"3"|"iii"|"three" *} +} { + do_execsql_test 4.$tn {SELECT fts5_expr($expr, 'tokenize=tcl')} [list $res] +} + finish_test diff --git a/manifest b/manifest index fd2b4870e3..4d5891206e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\stest\sfor\san\sfts5\stokenizer\sthat\ssupports\ssynonyms\sby\sadding\smultiple\sentries\sto\sthe\sfts\sindex. -D 2015-08-29T18:46:12.456 +C Begin\schanges\sto\sallow\ssynonym\ssupport\sby\sadding\smultiple\sterms\sto\sa\squery\s(an\salternative\sto\sadding\smultiple\sterms\sto\sthe\sFTS\sindex). +D 2015-08-31T20:06:06.235 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -106,11 +106,11 @@ F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95 F ext/fts5/extract_api_docs.tcl 06583c935f89075ea0b32f85efa5dd7619fcbd03 F ext/fts5/fts5.h 0784692f406588e6c90e13a78e1f36e7e3236e42 -F ext/fts5/fts5Int.h 9fd31e682acae32806f77e7c3b543c4294274c92 +F ext/fts5/fts5Int.h 9c538f0fcc1c3bc2fa12f7199d1326bd2362ce9c F ext/fts5/fts5_aux.c 7a307760a9c57c750d043188ec0bad59f5b5ec7e F ext/fts5/fts5_buffer.c 80f9ba4431848cb857e3d2158f5280093dcd8015 F ext/fts5/fts5_config.c 80b61fd2c6844b64a3e72a64572d50a812da9384 -F ext/fts5/fts5_expr.c 7ea46f676491989069d31ae1f75c9439b0858711 +F ext/fts5/fts5_expr.c 1458c3e33c1ec3ad99284f9692edfc49c44afd7c F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246 F ext/fts5/fts5_index.c 076c4995bf06a6d1559a6e31f9a86b90f2105374 F ext/fts5/fts5_main.c b00834ac543431dc35edbe18018b4befe0c7fd42 @@ -124,7 +124,7 @@ F ext/fts5/fts5_vocab.c 4622e0b7d84a488a1585aaa56eb214ee67a988bc F ext/fts5/fts5parse.y 833db1101b78c0c47686ab1b84918e38c36e9452 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba F ext/fts5/test/fts5_common.tcl 3338968de1880ca12b0451ae8f9b8b12d14e0ba7 -F ext/fts5/test/fts5aa.test c6e680a0d1b6c2616a382f1006d5d91eca697bd0 +F ext/fts5/test/fts5aa.test caa44c528f7270aa4e325c4f2c28d355c1e8c307 F ext/fts5/test/fts5ab.test 6fe3a56731d15978afbb74ae51b355fc9310f2ad F ext/fts5/test/fts5ac.test 9737992d08c56bfd4803e933744d2d764e23795c F ext/fts5/test/fts5ad.test b2edee8b7de0c21d2c88f8a18c195034aad6952d @@ -149,8 +149,8 @@ F ext/fts5/test/fts5corrupt2.test 1a830ccd6dbe1b601c7e3f5bbc1cf77bd8c8803b F ext/fts5/test/fts5corrupt3.test 1ccf575f5126e79f9fec7979fd02a1f40a076be3 F ext/fts5/test/fts5dlidx.test 59b80bbe34169a082c575d9c26f0a7019a7b79c1 F ext/fts5/test/fts5doclist.test 8edb5b57e5f144030ed74ec00ef6fa4294fed79b -F ext/fts5/test/fts5ea.test 451bb37310ee6df8ef72e4354fda5621b3b51448 -F ext/fts5/test/fts5eb.test 46f49497edc25ef3b2bff9fb6d75b6d201e2b39e +F ext/fts5/test/fts5ea.test b01e3a18cdfabbff8104a96a5242a06a68a998a0 +F ext/fts5/test/fts5eb.test 3e5869af2008cbc4ad03a175a0b6f6e58134cd43 F ext/fts5/test/fts5fault1.test 7a562367cb4a735b57b410dbdb62dcc8d971faec F ext/fts5/test/fts5fault2.test 28c36c843bb39ae855ba79827417ecc37f114341 F ext/fts5/test/fts5fault3.test d6e9577d4312e331a913c72931bf131704efc8f3 @@ -172,7 +172,7 @@ F ext/fts5/test/fts5rank.test 11dcebba31d822f7e99685b4ea2c2ae3ec0b16f1 F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17 F ext/fts5/test/fts5rowid.test 6f9833b23b176dc4aa15b7fc02afeb2b220fd460 -F ext/fts5/test/fts5synonym.test 6f1cfa5022bdae999f018075254e9fc51da2e618 +F ext/fts5/test/fts5synonym.test 3343a05f7b1d3fc6fa4559bd3bd0ee979928c30d F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89 F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841 F ext/fts5/test/fts5unicode2.test c1dd890ba32b7609adba78e420faa847abe43b59 @@ -1381,7 +1381,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P fc71868496f45f9c7a79ed2bf2d164a7c4718ce1 -R 8d19f3ef91012c48b3ad32684321db98 +P 98d07d16cab92f1e7001afbe370df3ec6343fc1f +R e2a3883c4d0440bf5b53c5a71ba46260 U dan -Z 14116ac3039ffa0cba068c002654bc88 +Z 7c01c61f2a729597f3365ce7db7a6ce6 diff --git a/manifest.uuid b/manifest.uuid index 3c4a7727be..b0ebea4bef 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -98d07d16cab92f1e7001afbe370df3ec6343fc1f \ No newline at end of file +ad7feaed4cd6b1d6e6376bb82d1f5664ddd083f3 \ No newline at end of file From 03bf26d90dedaf774c0afbdb54fbf1904234c09e Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 31 Aug 2015 21:16:36 +0000 Subject: [PATCH 015/100] Not only date/time functions, but also functions like sqlite_version() and changes() need to be prohibited from use inside of indexes. FossilOrigin-Name: 487131303980f15dd5e1b6695b4f29efda229eb8 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/func.c | 12 ++++++------ src/resolve.c | 25 +++++++++++-------------- src/sqliteInt.h | 12 +++++++----- 5 files changed, 33 insertions(+), 34 deletions(-) diff --git a/manifest b/manifest index 392a86929f..b244252bcb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Always\sassume\sthat\sindexed\sexpressions\scan\sgenerate\sa\sNULL.\s\sGet\nindexed\sexpressions\sworking\sfor\sthe\scase\sof\stwo\sor\smore\sexpressions\sin\nthe\ssame\sindex. -D 2015-08-31T19:38:42.715 +C Not\sonly\sdate/time\sfunctions,\sbut\salso\sfunctions\slike\ssqlite_version()\sand\nchanges()\sneed\sto\sbe\sprohibited\sfrom\suse\sinside\sof\sindexes. +D 2015-08-31T21:16:36.552 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -292,7 +292,7 @@ F src/delete.c 224e5c9a0eae7fbdd3b461ca971465c992851354 F src/expr.c 500c0ee6ac2e5d60c10df62b8c9b96dc75c2cbb1 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 3ce33dd49f12c72376cec9adc7a4d8e7111cedcc -F src/func.c 824bea430d3a2b7dbc62806ad54da8fdb8ed9e3f +F src/func.c ecdd69ec6a1e406f04cc73324be2ebbf6354197f F src/global.c 508e4087f7b41d688e4762dcf4d4fe28cfbc87f9 F src/hash.c 4263fbc955f26c2e8cdc0cf214bc42435aa4e4f5 F src/hash.h c8f3c31722cf3277d03713909761e152a5b81094 @@ -335,14 +335,14 @@ F src/pragma.h 631a91c8b0e6ca8f051a1d8a4a0da4150e04620a F src/prepare.c 82e5db1013846a819f198336fed72c44c974e7b1 F src/printf.c 2bc439ff20a4aad0e0ad50a37a67b5eae7d20edc F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 -F src/resolve.c 3ec6456a534bcbf0f3fc11ad83c6208301db60a2 +F src/resolve.c 203146a9adfed08c94cdb4bd59421b066a00e03f F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c b52c80f2b1bdb62491f9ce40eea0c5f80c78d105 F src/shell.c 6332ef06db1390ef812cfdff1fc97b4fd76cdd42 F src/sqlite.h.in 378bebc8fe6a88bade25e5f23b7e6123fdc64b00 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h f700e6a9dd1fdcccc9951ab022b366fb66b9e413 -F src/sqliteInt.h 332ae4f753d91c92421a2c4227c195298d5b154c +F src/sqliteInt.h fde5d96c3d547084f7c9ba85b87bad3e4a9a613f F src/sqliteLimit.h 216557999cb45f2e3578ed53ebefe228d779cb46 F src/status.c f266ad8a2892d659b74f0f50cb6a88b6e7c12179 F src/table.c 51b46b2a62d1b3a959633d593b89bab5e2c9155e @@ -1381,7 +1381,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P e2f1caf117b0a9632d52246717ab202852982339 -R 9a8972907f2a35dc9161585e1e0d88b8 +P cc60321a67bf9f169c090b47afb505f589a6925e +R d6937d2b3277ce224a948b2dc07646a7 U drh -Z 7baf4f251bc37ea5f508ac4ed0b603b4 +Z c474b9a8c6f81c985226d970f2d27c66 diff --git a/manifest.uuid b/manifest.uuid index 693f9f59dd..79cdf18fae 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cc60321a67bf9f169c090b47afb505f589a6925e \ No newline at end of file +487131303980f15dd5e1b6695b4f29efda229eb8 \ No newline at end of file diff --git a/src/func.c b/src/func.c index 6ecd743969..8ea1169327 100644 --- a/src/func.c +++ b/src/func.c @@ -1737,15 +1737,15 @@ void sqlite3RegisterGlobalFunctions(void){ VFUNCTION(random, 0, 0, 0, randomFunc ), VFUNCTION(randomblob, 1, 0, 0, randomBlob ), FUNCTION(nullif, 2, 0, 1, nullifFunc ), - FUNCTION(sqlite_version, 0, 0, 0, versionFunc ), - FUNCTION(sqlite_source_id, 0, 0, 0, sourceidFunc ), + DFUNCTION(sqlite_version, 0, 0, 0, versionFunc ), + DFUNCTION(sqlite_source_id, 0, 0, 0, sourceidFunc ), FUNCTION(sqlite_log, 2, 0, 0, errlogFunc ), #if SQLITE_USER_AUTHENTICATION FUNCTION(sqlite_crypt, 2, 0, 0, sqlite3CryptFunc ), #endif #ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS - FUNCTION(sqlite_compileoption_used,1, 0, 0, compileoptionusedFunc ), - FUNCTION(sqlite_compileoption_get, 1, 0, 0, compileoptiongetFunc ), + DFUNCTION(sqlite_compileoption_used,1, 0, 0, compileoptionusedFunc ), + DFUNCTION(sqlite_compileoption_get, 1, 0, 0, compileoptiongetFunc ), #endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */ FUNCTION(quote, 1, 0, 0, quoteFunc ), VFUNCTION(last_insert_rowid, 0, 0, 0, last_insert_rowid), @@ -1757,8 +1757,8 @@ void sqlite3RegisterGlobalFunctions(void){ FUNCTION(soundex, 1, 0, 0, soundexFunc ), #endif #ifndef SQLITE_OMIT_LOAD_EXTENSION - FUNCTION(load_extension, 1, 0, 0, loadExt ), - FUNCTION(load_extension, 2, 0, 0, loadExt ), + VFUNCTION(load_extension, 1, 0, 0, loadExt ), + VFUNCTION(load_extension, 2, 0, 0, loadExt ), #endif AGGREGATE(sum, 1, 0, 0, sumStep, sumFinalize ), AGGREGATE(total, 1, 0, 0, sumStep, totalFinalize ), diff --git a/src/resolve.c b/src/resolve.c index 0b8dde7cc6..9522f63cea 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -548,19 +548,16 @@ Expr *sqlite3CreateColumnExpr(sqlite3 *db, SrcList *pSrc, int iSrc, int iCol){ /* ** Report an error that an expression is not valid for some set of -** pNC->ncFlags values determined by validMask. If +** pNC->ncFlags values determined by validMask. */ static void notValid( Parse *pParse, /* Leave error message here */ NameContext *pNC, /* The name context */ const char *zMsg, /* Type of error */ - int validMask, /* Set of contexts for which prohibited */ - int okForInit /* No error if pParse->db->init.busy is true */ + int validMask /* Set of contexts for which prohibited */ ){ assert( (validMask&~(NC_IsCheck|NC_PartIdx|NC_IdxExpr))==0 ); - if( (pNC->ncFlags & validMask)!=0 - && (pParse->db->init.busy==0 || !okForInit) - ){ + if( (pNC->ncFlags & validMask)!=0 ){ const char *zIn = "partial index WHERE clauses"; if( pNC->ncFlags & NC_IdxExpr ) zIn = "index expressions"; #ifndef SQLITE_OMIT_CHECK @@ -653,7 +650,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ Expr *pRight; /* if( pSrcList==0 ) break; */ - notValid(pParse, pNC, "the \".\" operator", NC_IdxExpr, 0); + notValid(pParse, pNC, "the \".\" operator", NC_IdxExpr); /*notValid(pParse, pNC, "the \".\" operator", NC_PartIdx|NC_IsCheck, 1);*/ pRight = pExpr->pRight; if( pRight->op==TK_ID ){ @@ -684,7 +681,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ u8 enc = ENC(pParse->db); /* The database encoding */ assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); - notValid(pParse, pNC, "functions", NC_PartIdx, 0); + notValid(pParse, pNC, "functions", NC_PartIdx); zId = pExpr->u.zToken; nId = sqlite3Strlen30(zId); pDef = sqlite3FindFunction(pParse->db, zId, nId, n, enc, 0); @@ -732,16 +729,16 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ return WRC_Prune; } #endif - if( pDef->funcFlags & (SQLITE_FUNC_CONSTANT|SQLITE_FUNC_DATETIME) ){ + if( pDef->funcFlags & (SQLITE_FUNC_CONSTANT|SQLITE_FUNC_VARYING) ){ /* For the purposes of the EP_ConstFunc flag, date and time - ** functions are considered constant because the the time does - ** not change for the duration of a query. */ + ** functions and other functions that change slowly are considered + ** constant because they are constant for the duration of one query */ ExprSetProperty(pExpr,EP_ConstFunc); } if( (pDef->funcFlags & SQLITE_FUNC_CONSTANT)==0 ){ /* DATETIME functions are considered non-deterministic because of ** the 'now' capability */ - notValid(pParse, pNC, "non-deterministic functions", NC_IdxExpr, 0); + notValid(pParse, pNC, "non-deterministic functions", NC_IdxExpr); } } if( is_agg && (pNC->ncFlags & NC_AllowAgg)==0 ){ @@ -788,7 +785,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ testcase( pExpr->op==TK_IN ); if( ExprHasProperty(pExpr, EP_xIsSelect) ){ int nRef = pNC->nRef; - notValid(pParse, pNC, "subqueries", NC_IsCheck|NC_PartIdx|NC_IdxExpr,0); + notValid(pParse, pNC, "subqueries", NC_IsCheck|NC_PartIdx|NC_IdxExpr); sqlite3WalkSelect(pWalker, pExpr->x.pSelect); assert( pNC->nRef>=nRef ); if( nRef!=pNC->nRef ){ @@ -798,7 +795,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ break; } case TK_VARIABLE: { - notValid(pParse, pNC, "parameters", NC_IsCheck|NC_PartIdx|NC_IdxExpr, 0); + notValid(pParse, pNC, "parameters", NC_IsCheck|NC_PartIdx|NC_IdxExpr); break; } } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 613f351aa5..8bbb913325 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1396,7 +1396,7 @@ struct FuncDestructor { #define SQLITE_FUNC_UNLIKELY 0x0400 /* Built-in unlikely() function */ #define SQLITE_FUNC_CONSTANT 0x0800 /* Constant inputs give a constant output */ #define SQLITE_FUNC_MINMAX 0x1000 /* True for min() and max() aggregates */ -#define SQLITE_FUNC_DATETIME 0x2000 /* Function returns a date and/or time */ +#define SQLITE_FUNC_VARYING 0x2000 /* Function that change in the long term */ /* ** The following three macros, FUNCTION(), LIKEFUNC() and AGGREGATE() are @@ -1414,7 +1414,9 @@ struct FuncDestructor { ** ** DFUNCTION(zName, nArg, iArg, bNC, xFunc) ** Like FUNCTION except it omits the SQLITE_FUNC_CONSTANT flag and -** adds the SQLITE_FUNC_DATETIME flag. Used for date & time functions. +** adds the SQLITE_FUNC_VARYING flag. Used for date & time functions +** and functions like sqlite_version() that can change, but not during +** a single query. ** ** AGGREGATE(zName, nArg, iArg, bNC, xStep, xFinal) ** Used to create an aggregate function definition implemented by @@ -1437,13 +1439,13 @@ struct FuncDestructor { {nArg, SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0, 0} #define DFUNCTION(zName, nArg, iArg, bNC, xFunc) \ - {nArg, SQLITE_FUNC_DATETIME|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ + {nArg, SQLITE_FUNC_VARYING|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0, 0} #define FUNCTION2(zName, nArg, iArg, bNC, xFunc, extraFlags) \ {nArg,SQLITE_FUNC_CONSTANT|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL)|extraFlags,\ SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0, 0} #define STR_FUNCTION(zName, nArg, pArg, bNC, xFunc) \ - {nArg, SQLITE_FUNC_CONSTANT|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ + {nArg, SQLITE_FUNC_VARYING|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ pArg, 0, xFunc, 0, 0, #zName, 0, 0} #define LIKEFUNC(zName, nArg, arg, flags) \ {nArg, SQLITE_FUNC_CONSTANT|SQLITE_UTF8|flags, \ @@ -2129,7 +2131,7 @@ struct Expr { #define EP_MemToken 0x010000 /* Need to sqlite3DbFree() Expr.zToken */ #define EP_NoReduce 0x020000 /* Cannot EXPRDUP_REDUCE this Expr */ #define EP_Unlikely 0x040000 /* unlikely() or likelihood() function */ -#define EP_ConstFunc 0x080000 /* A SQLITE_FUNC_CONSTANT or _DATETIME function */ +#define EP_ConstFunc 0x080000 /* A SQLITE_FUNC_CONSTANT or _VARYING function */ #define EP_CanBeNull 0x100000 /* Can be null despite NOT NULL constraint */ #define EP_Subquery 0x200000 /* Tree contains a TK_SELECT operator */ From 8b576422592d9845240cc3b9e923d1ba2dbff4f1 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 31 Aug 2015 23:09:42 +0000 Subject: [PATCH 016/100] Fix a bug in error reporting when a UNIQUE index on expressions fails its uniqueness test. FossilOrigin-Name: 5a2c0e90a1933545b4768d91d8f7c42c8f391019 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/build.c | 18 ++++++++++-------- test/indexexpr1.test | 12 ++++++++++++ 4 files changed, 30 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index b244252bcb..ab89121f74 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Not\sonly\sdate/time\sfunctions,\sbut\salso\sfunctions\slike\ssqlite_version()\sand\nchanges()\sneed\sto\sbe\sprohibited\sfrom\suse\sinside\sof\sindexes. -D 2015-08-31T21:16:36.552 +C Fix\sa\sbug\sin\serror\sreporting\swhen\sa\sUNIQUE\sindex\son\sexpressions\sfails\sits\nuniqueness\stest. +D 2015-08-31T23:09:42.268 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -282,7 +282,7 @@ F src/btmutex.c 45a968cc85afed9b5e6cf55bf1f42f8d18107f79 F src/btree.c f48b3ef91676c06a90a8832987ecef6b94c931ee F src/btree.h 969adc948e89e449220ff0ff724c94bb2a52e9f1 F src/btreeInt.h 8177c9ab90d772d6d2c6c517e05bed774b7c92c0 -F src/build.c 9e7216acaf612d371c1b4a6bf9711886472f7ac6 +F src/build.c f85ede9d6dbf833f8f8d43bd78619c2a112ab161 F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 F src/complete.c addcd8160b081131005d5bc2d34adf20c1c5c92f F src/ctime.c 5a0b735dc95604766f5dac73973658eef782ee8b @@ -780,7 +780,7 @@ F test/index5.test 8621491915800ec274609e42e02a97d67e9b13e7 F test/index6.test 7102ec371414c42dfb1d5ca37eb4519aa9edc23a F test/index7.test 9c6765a74fc3fcde7aebc5b3bd40d98df14a527c F test/indexedby.test 5f527a78bae74c61b8046ae3037f9dfb0bf0c353 -F test/indexexpr1.test d04a3905275fb9a0fb82447e7d43773f98fad282 +F test/indexexpr1.test 3c5033412f851f225e3a37d6795709df71bea638 F test/indexfault.test 31d4ab9a7d2f6e9616933eb079722362a883eb1d F test/init.test 15c823093fdabbf7b531fe22cf037134d09587a7 F test/insert.test 38742b5e9601c8f8d76e9b7555f7270288c2d371 @@ -1381,7 +1381,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P cc60321a67bf9f169c090b47afb505f589a6925e -R d6937d2b3277ce224a948b2dc07646a7 +P 487131303980f15dd5e1b6695b4f29efda229eb8 +R 9ef09a7ae54b897d1ff4fd4dfc1065a6 U drh -Z c474b9a8c6f81c985226d970f2d27c66 +Z 8fa9de950d104f084061f9372f8e6439 diff --git a/manifest.uuid b/manifest.uuid index 79cdf18fae..145029b629 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -487131303980f15dd5e1b6695b4f29efda229eb8 \ No newline at end of file +5a2c0e90a1933545b4768d91d8f7c42c8f391019 \ No newline at end of file diff --git a/src/build.c b/src/build.c index f217ab251a..0d5aa34fd1 100644 --- a/src/build.c +++ b/src/build.c @@ -4117,14 +4117,16 @@ void sqlite3UniqueConstraint( Table *pTab = pIdx->pTable; sqlite3StrAccumInit(&errMsg, pParse->db, 0, 0, 200); - for(j=0; jnKeyCol; j++){ - char *zCol; - assert( pIdx->aiColumn[j]>=0 ); - zCol = pTab->aCol[pIdx->aiColumn[j]].zName; - if( j ) sqlite3StrAccumAppend(&errMsg, ", ", 2); - sqlite3StrAccumAppendAll(&errMsg, pTab->zName); - sqlite3StrAccumAppend(&errMsg, ".", 1); - sqlite3StrAccumAppendAll(&errMsg, zCol); + if( pIdx->aColExpr ){ + sqlite3XPrintf(&errMsg, 0, "index '%q'", pIdx->zName); + }else{ + for(j=0; jnKeyCol; j++){ + char *zCol; + assert( pIdx->aiColumn[j]>=0 ); + zCol = pTab->aCol[pIdx->aiColumn[j]].zName; + if( j ) sqlite3StrAccumAppend(&errMsg, ", ", 2); + sqlite3XPrintf(&errMsg, 0, "%s.%s", pTab->zName, zCol); + } } zErr = sqlite3StrAccumFinish(&errMsg); sqlite3HaltConstraint(pParse, diff --git a/test/indexexpr1.test b/test/indexexpr1.test index aa056e55d4..faaf292de5 100644 --- a/test/indexexpr1.test +++ b/test/indexexpr1.test @@ -186,4 +186,16 @@ do_catchsql_test indexexpr1-340 { CREATE TABLE e1(x,y,FOREIGN KEY(substr(y,1,5)) REFERENCES t1); } {1 {near "(": syntax error}} +do_execsql_test indexexpr1-400 { + CREATE TABLE t3(a,b,c); + WITH RECURSIVE c(x) AS (VALUES(1) UNION SELECT x+1 FROM c WHERE x<30) + INSERT INTO t3(a,b,c) + SELECT x, printf('ab%04xyz',x), random() FROM c; + CREATE UNIQUE INDEX t3abc ON t3(CAST(a AS text), b, substr(c,1,3)); + SELECT a FROM t3 WHERE CAST(a AS text)<='10' ORDER BY +a; +} {1 10} +do_catchsql_test indexexpr1-410 { + INSERT INTO t3 SELECT * FROM t3 WHERE rowid=10; +} {1 {UNIQUE constraint failed: index 't3abc'}} + finish_test From 7d3d9daea2f94d1265ec97f1d1dfbc6cc36edefd Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 1 Sep 2015 00:42:52 +0000 Subject: [PATCH 017/100] Remove unreachable branches. FossilOrigin-Name: fd4da2318cc032d7c355376e440d4a05d7ab8793 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/build.c | 9 ++++++--- src/where.c | 7 ++++--- 4 files changed, 18 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index ab89121f74..a1b5d3fc23 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sbug\sin\serror\sreporting\swhen\sa\sUNIQUE\sindex\son\sexpressions\sfails\sits\nuniqueness\stest. -D 2015-08-31T23:09:42.268 +C Remove\sunreachable\sbranches. +D 2015-09-01T00:42:52.453 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -282,7 +282,7 @@ F src/btmutex.c 45a968cc85afed9b5e6cf55bf1f42f8d18107f79 F src/btree.c f48b3ef91676c06a90a8832987ecef6b94c931ee F src/btree.h 969adc948e89e449220ff0ff724c94bb2a52e9f1 F src/btreeInt.h 8177c9ab90d772d6d2c6c517e05bed774b7c92c0 -F src/build.c f85ede9d6dbf833f8f8d43bd78619c2a112ab161 +F src/build.c c6926497c50c8005c75d4ca476b254978d195a0d F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 F src/complete.c addcd8160b081131005d5bc2d34adf20c1c5c92f F src/ctime.c 5a0b735dc95604766f5dac73973658eef782ee8b @@ -414,7 +414,7 @@ F src/vxworks.h c18586c8edc1bddbc15c004fa16aeb1e1342b4fb F src/wal.c 8cd07f1f99e1a81346db1c9da879bef6c6f97cf6 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 2e14d17f592d176b6dc879c33fbdec4fbccaa2ba -F src/where.c 04e0f87e4d82a85c283176958465bed6ea5434e0 +F src/where.c 567b96aa00e7a13ae379a48e10c2911087159e7d F src/whereInt.h 292d3ac90da4eab1e03ac8452f1add746bcafaa1 F src/wherecode.c b0bf45ca49e62fde68ba2e2ad2939d9cdeb4e409 F src/whereexpr.c 2473e4350e30f9b55d1c6a8f66ca23c689f23f1d @@ -1381,7 +1381,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 487131303980f15dd5e1b6695b4f29efda229eb8 -R 9ef09a7ae54b897d1ff4fd4dfc1065a6 +P 5a2c0e90a1933545b4768d91d8f7c42c8f391019 +R 43c7fa01f20234523219b9eb18641ef8 U drh -Z 8fa9de950d104f084061f9372f8e6439 +Z e43aaed9b864f00cd6aff5655ab5ae6d diff --git a/manifest.uuid b/manifest.uuid index 145029b629..8be0526e03 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5a2c0e90a1933545b4768d91d8f7c42c8f391019 \ No newline at end of file +fd4da2318cc032d7c355376e440d4a05d7ab8793 \ No newline at end of file diff --git a/src/build.c b/src/build.c index 0d5aa34fd1..e05b17b68e 100644 --- a/src/build.c +++ b/src/build.c @@ -1310,7 +1310,8 @@ void sqlite3AddPrimaryKey( nTerm = pList->nExpr; for(i=0; ia[i].pExpr); - if( pCExpr && pCExpr->op==TK_ID ){ + assert( pCExpr!=0 ); + if( pCExpr->op==TK_ID ){ const char *zCName = pCExpr->u.zToken; for(iCol=0; iColnCol; iCol++){ if( sqlite3StrICmp(zCName, pTab->aCol[iCol].zName)==0 ){ @@ -3061,7 +3062,8 @@ Index *sqlite3CreateIndex( */ for(i=0; inExpr; i++){ Expr *pExpr = pList->a[i].pExpr; - if( pExpr && pExpr->op==TK_COLLATE ){ + assert( pExpr!=0 ); + if( pExpr->op==TK_COLLATE ){ nExtra += (1 + sqlite3Strlen30(pExpr->u.zToken)); } } @@ -3258,6 +3260,7 @@ Index *sqlite3CreateIndex( /* Link the new Index structure to its table and to the other ** in-memory database structures. */ + assert( pParse->nErr==0 ); if( db->init.busy ){ Index *p; assert( sqlite3SchemaMutexHeld(db, 0, pIndex->pSchema) ); @@ -3287,7 +3290,7 @@ Index *sqlite3CreateIndex( ** has just been created, it contains no data and the index initialization ** step can be skipped. */ - else if( pParse->nErr==0 && (HasRowid(pTab) || pTblName!=0) ){ + else if( HasRowid(pTab) || pTblName!=0 ){ Vdbe *v; char *zStmt; int iMem = ++pParse->nMem; diff --git a/src/where.c b/src/where.c index 6dc326be0d..68abfdfe1c 100644 --- a/src/where.c +++ b/src/where.c @@ -180,7 +180,7 @@ static WhereTerm *whereScanNext(WhereScan *pScan){ while( pScan->iEquiv<=pScan->nEquiv ){ iCur = pScan->aiCur[pScan->iEquiv-1]; iColumn = pScan->aiColumn[pScan->iEquiv-1]; - if( iColumn==(-2) && pScan->pIdxExpr==0 ) return 0; + assert( iColumn!=(-2) || pScan->pIdxExpr!=0 ); while( (pWC = pScan->pWC)!=0 ){ for(pTerm=pWC->a+k; knTerm; k++, pTerm++){ if( pTerm->leftCursor==iCur @@ -396,6 +396,7 @@ static int indexColumnNotNull(Index *pIdx, int iCol){ }else{ assert( j==(-2) ); return 0; /* Assume an indexed expression can always yield a NULL */ + } } @@ -803,7 +804,7 @@ static sqlite3_index_info *allocateIndexInfo( testcase( pTerm->eOperator & WO_ALL ); if( (pTerm->eOperator & ~(WO_ISNULL|WO_EQUIV|WO_IS))==0 ) continue; if( pTerm->wtFlags & TERM_VNULL ) continue; - if( pTerm->u.leftColumn<(-1) ) continue; + assert( pTerm->u.leftColumn>=(-1) ); nTerm++; } @@ -859,7 +860,7 @@ static sqlite3_index_info *allocateIndexInfo( testcase( pTerm->eOperator & WO_ALL ); if( (pTerm->eOperator & ~(WO_ISNULL|WO_EQUIV|WO_IS))==0 ) continue; if( pTerm->wtFlags & TERM_VNULL ) continue; - if( pTerm->u.leftColumn<(-1) ) continue; + assert( pTerm->u.leftColumn>=(-1) ); pIdxCons[j].iColumn = pTerm->u.leftColumn; pIdxCons[j].iTermOffset = i; op = (u8)pTerm->eOperator & WO_ALL; From a7f910b549e305306165fe99b431891e66c84223 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 1 Sep 2015 13:17:17 +0000 Subject: [PATCH 018/100] Rename SQLITE_FUNC_VARYING to SQLITE_FUNC_SLOCHNG - a more descriptive name for what that bit means. FossilOrigin-Name: ff5137a6dd8cb2a9b629b3a244f52665e9c9ebce --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/resolve.c | 7 ++++--- src/sqliteInt.h | 11 ++++++----- 4 files changed, 18 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index a1b5d3fc23..c69f4462c2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sunreachable\sbranches. -D 2015-09-01T00:42:52.453 +C Rename\sSQLITE_FUNC_VARYING\sto\sSQLITE_FUNC_SLOCHNG\s-\sa\smore\sdescriptive\sname\nfor\swhat\sthat\sbit\smeans. +D 2015-09-01T13:17:17.375 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -335,14 +335,14 @@ F src/pragma.h 631a91c8b0e6ca8f051a1d8a4a0da4150e04620a F src/prepare.c 82e5db1013846a819f198336fed72c44c974e7b1 F src/printf.c 2bc439ff20a4aad0e0ad50a37a67b5eae7d20edc F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 -F src/resolve.c 203146a9adfed08c94cdb4bd59421b066a00e03f +F src/resolve.c 47c00ca0ab24d4b8113c6c05aa07bc6cf6eac9af F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c b52c80f2b1bdb62491f9ce40eea0c5f80c78d105 F src/shell.c 6332ef06db1390ef812cfdff1fc97b4fd76cdd42 F src/sqlite.h.in 378bebc8fe6a88bade25e5f23b7e6123fdc64b00 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h f700e6a9dd1fdcccc9951ab022b366fb66b9e413 -F src/sqliteInt.h fde5d96c3d547084f7c9ba85b87bad3e4a9a613f +F src/sqliteInt.h e7e92112a71aeb214c66ebdea1366560535614d1 F src/sqliteLimit.h 216557999cb45f2e3578ed53ebefe228d779cb46 F src/status.c f266ad8a2892d659b74f0f50cb6a88b6e7c12179 F src/table.c 51b46b2a62d1b3a959633d593b89bab5e2c9155e @@ -1381,7 +1381,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 5a2c0e90a1933545b4768d91d8f7c42c8f391019 -R 43c7fa01f20234523219b9eb18641ef8 +P fd4da2318cc032d7c355376e440d4a05d7ab8793 +R 56bc6d7502c6fcf5fea0759746df8b8f U drh -Z e43aaed9b864f00cd6aff5655ab5ae6d +Z 5fd0e1fb2f0e68b5cb95600541b3c11a diff --git a/manifest.uuid b/manifest.uuid index 8be0526e03..5b8d71a036 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fd4da2318cc032d7c355376e440d4a05d7ab8793 \ No newline at end of file +ff5137a6dd8cb2a9b629b3a244f52665e9c9ebce \ No newline at end of file diff --git a/src/resolve.c b/src/resolve.c index 9522f63cea..22df223948 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -729,15 +729,16 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ return WRC_Prune; } #endif - if( pDef->funcFlags & (SQLITE_FUNC_CONSTANT|SQLITE_FUNC_VARYING) ){ + if( pDef->funcFlags & (SQLITE_FUNC_CONSTANT|SQLITE_FUNC_SLOCHNG) ){ /* For the purposes of the EP_ConstFunc flag, date and time ** functions and other functions that change slowly are considered ** constant because they are constant for the duration of one query */ ExprSetProperty(pExpr,EP_ConstFunc); } if( (pDef->funcFlags & SQLITE_FUNC_CONSTANT)==0 ){ - /* DATETIME functions are considered non-deterministic because of - ** the 'now' capability */ + /* Date/time functions that use 'now', and other functions like + ** sqlite_version() that might change over time cannot be used + ** in an index. */ notValid(pParse, pNC, "non-deterministic functions", NC_IdxExpr); } } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 8bbb913325..69315ed466 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1396,7 +1396,8 @@ struct FuncDestructor { #define SQLITE_FUNC_UNLIKELY 0x0400 /* Built-in unlikely() function */ #define SQLITE_FUNC_CONSTANT 0x0800 /* Constant inputs give a constant output */ #define SQLITE_FUNC_MINMAX 0x1000 /* True for min() and max() aggregates */ -#define SQLITE_FUNC_VARYING 0x2000 /* Function that change in the long term */ +#define SQLITE_FUNC_SLOCHNG 0x2000 /* "Slow Change". Value constant during a + ** single query - might change over time */ /* ** The following three macros, FUNCTION(), LIKEFUNC() and AGGREGATE() are @@ -1414,7 +1415,7 @@ struct FuncDestructor { ** ** DFUNCTION(zName, nArg, iArg, bNC, xFunc) ** Like FUNCTION except it omits the SQLITE_FUNC_CONSTANT flag and -** adds the SQLITE_FUNC_VARYING flag. Used for date & time functions +** adds the SQLITE_FUNC_SLOCHNG flag. Used for date & time functions ** and functions like sqlite_version() that can change, but not during ** a single query. ** @@ -1439,13 +1440,13 @@ struct FuncDestructor { {nArg, SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0, 0} #define DFUNCTION(zName, nArg, iArg, bNC, xFunc) \ - {nArg, SQLITE_FUNC_VARYING|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ + {nArg, SQLITE_FUNC_SLOCHNG|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0, 0} #define FUNCTION2(zName, nArg, iArg, bNC, xFunc, extraFlags) \ {nArg,SQLITE_FUNC_CONSTANT|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL)|extraFlags,\ SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0, 0} #define STR_FUNCTION(zName, nArg, pArg, bNC, xFunc) \ - {nArg, SQLITE_FUNC_VARYING|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ + {nArg, SQLITE_FUNC_SLOCHNG|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ pArg, 0, xFunc, 0, 0, #zName, 0, 0} #define LIKEFUNC(zName, nArg, arg, flags) \ {nArg, SQLITE_FUNC_CONSTANT|SQLITE_UTF8|flags, \ @@ -2131,7 +2132,7 @@ struct Expr { #define EP_MemToken 0x010000 /* Need to sqlite3DbFree() Expr.zToken */ #define EP_NoReduce 0x020000 /* Cannot EXPRDUP_REDUCE this Expr */ #define EP_Unlikely 0x040000 /* unlikely() or likelihood() function */ -#define EP_ConstFunc 0x080000 /* A SQLITE_FUNC_CONSTANT or _VARYING function */ +#define EP_ConstFunc 0x080000 /* A SQLITE_FUNC_CONSTANT or _SLOCHNG function */ #define EP_CanBeNull 0x100000 /* Can be null despite NOT NULL constraint */ #define EP_Subquery 0x200000 /* Tree contains a TK_SELECT operator */ From 3faa07ee0474436575a6cb81d56e80f6c47c3bd9 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 1 Sep 2015 18:08:36 +0000 Subject: [PATCH 019/100] Add tests for fts5 synonyms implemented by adding extra terms to queries. And fixes for the same. FossilOrigin-Name: dbcb73802b88f76be17f09f3eb83ffac542de633 --- ext/fts5/fts5Int.h | 2 + ext/fts5/fts5_expr.c | 92 +++++++++++++++++++++++------ ext/fts5/test/fts5synonym.test | 102 +++++++++++++++++++++++++++------ manifest | 16 +++--- manifest.uuid | 2 +- 5 files changed, 172 insertions(+), 42 deletions(-) diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index e8d997b060..1ad61617d6 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -235,6 +235,8 @@ struct Fts5PoslistReader { int n; /* Size of buffer at a[] in bytes */ int i; /* Current offset in a[] */ + u8 bFlag; /* For client use (any custom purpose) */ + /* Output variables */ u8 bEof; /* Set to true at EOF */ i64 iPos; /* (iCol<<32) + iPos */ diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index bba9932daf..24d63c1cab 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -22,6 +22,8 @@ */ #define FTS5_EOF 0 +#define FTS5_LARGEST_INT64 (0xffffffff|(((i64)0x7fffffff)<<32)) + typedef struct Fts5ExprTerm Fts5ExprTerm; /* @@ -386,26 +388,66 @@ static i64 fts5ExprSynonymRowid(Fts5ExprTerm *pTerm, int bDesc){ static int fts5ExprSynonymPoslist( Fts5ExprTerm *pTerm, i64 iRowid, + int *pbDel, /* OUT: Caller should sqlite3_free(*pa) */ u8 **pa, int *pn ){ Fts5PoslistWriter writer = {0}; Fts5PoslistReader aStatic[4]; Fts5PoslistReader *aIter = aStatic; int nIter = 0; + int rc = SQLITE_OK; Fts5ExprTerm *p; assert( pTerm->pSynonym ); for(p=pTerm; p; p=p->pSynonym){ Fts5IndexIter *pIter = p->pIter; if( sqlite3Fts5IterEof(pIter)==0 && sqlite3Fts5IterRowid(pIter)==iRowid ){ + const u8 *a; + int n; i64 dummy; - int rc = sqlite3Fts5IterPoslist(pIter, (const u8**)pa, pn, &dummy); - return rc; + rc = sqlite3Fts5IterPoslist(pIter, &a, &n, &dummy); + if( rc!=SQLITE_OK ) return rc; + if( sqlite3Fts5PoslistReaderInit(-1, a, n, &aIter[nIter])==0 ){ + nIter++; + } } } - assert( 0 ); - return SQLITE_ERROR; + assert( *pbDel==0 ); + if( nIter==1 ){ + *pa = (u8*)aIter[0].a; + *pn = aIter[0].n; + }else{ + Fts5PoslistWriter writer = {0}; + Fts5Buffer buf = {0,0,0}; + i64 iPrev = -1; + while( 1 ){ + int i; + i64 iMin = FTS5_LARGEST_INT64; + for(i=0; inTerm); /* Initialize a term iterator for each term in the phrase */ for(i=0; inTerm; i++){ Fts5ExprTerm *pTerm = &pPhrase->aTerm[i]; i64 dummy; - int n; - const u8 *a; + int n = 0; + int bFlag = 0; + const u8 *a = 0; if( pTerm->pSynonym ){ - rc = fts5ExprSynonymPoslist(pTerm, pNode->iRowid, (u8**)&a, &n); + rc = fts5ExprSynonymPoslist(pTerm, pNode->iRowid, &bFlag, (u8**)&a, &n); }else{ rc = sqlite3Fts5IterPoslist(pTerm->pIter, &a, &n, &dummy); } - if( rc || sqlite3Fts5PoslistReaderInit(iCol, a, n, &aIter[i]) ){ - goto ismatch_out; - } + if( rc!=SQLITE_OK ) goto ismatch_out; + sqlite3Fts5PoslistReaderInit(iCol, a, n, &aIter[i]); + aIter[i].bFlag = bFlag; + if( aIter[i].bEof ) goto ismatch_out; } while( 1 ){ @@ -495,6 +540,9 @@ static int fts5ExprPhraseIsMatch( ismatch_out: *pbMatch = (pPhrase->poslist.n>0); + for(i=0; inTerm; i++){ + if( aIter[i].bFlag ) sqlite3_free((u8*)aIter[i].a); + } if( aIter!=aStatic ) sqlite3_free(aIter); return rc; } @@ -672,13 +720,25 @@ static int fts5ExprNearAdvanceFirst( /* Find the firstest rowid any synonym points to. */ i64 iRowid = fts5ExprSynonymRowid(pTerm, pExpr->bDesc); - /* Advance each iterator that currently points to iRowid */ + /* Advance each iterator that currently points to iRowid. Or, if iFrom + ** is valid - each iterator that points to a rowid before iFrom. */ for(p=pTerm; p; p=p->pSynonym){ if( sqlite3Fts5IterEof(p->pIter)==0 ){ - bEof = 0; - if( sqlite3Fts5IterRowid(p->pIter)==iRowid ){ - rc = sqlite3Fts5IterNext(p->pIter); + i64 ii = sqlite3Fts5IterRowid(p->pIter); + if( ii==iRowid + || (bFromValid && ii!=iFrom && (ii>iFrom)==pExpr->bDesc) + ){ + if( bFromValid ){ + rc = sqlite3Fts5IterNextFrom(p->pIter, iFrom); + }else{ + rc = sqlite3Fts5IterNext(p->pIter); + } if( rc!=SQLITE_OK ) break; + if( sqlite3Fts5IterEof(p->pIter)==0 ){ + bEof = 0; + } + }else{ + bEof = 0; } } } @@ -807,7 +867,7 @@ static int fts5ExprNearTest( ** phrase is not a match, break out of the loop early. */ for(i=0; rc==SQLITE_OK && inPhrase; i++){ Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; - if( pPhrase->nTerm>1 || pNear->pColset ){ + if( pPhrase->nTerm>1 || pPhrase->aTerm[0].pSynonym || pNear->pColset ){ int bMatch = 0; rc = fts5ExprPhraseIsMatch(pNode, pNear->pColset, pPhrase, &bMatch); if( bMatch==0 ) break; @@ -985,7 +1045,7 @@ static int fts5ExprNearInitAll( &p->pIter ); assert( rc==SQLITE_OK || p->pIter==0 ); - if( p->pIter && 0==sqlite3Fts5IterEof(pTerm->pIter) ){ + if( p->pIter && 0==sqlite3Fts5IterEof(p->pIter) ){ bEof = 0; } } diff --git a/ext/fts5/test/fts5synonym.test b/ext/fts5/test/fts5synonym.test index a607b7e29c..5aa29e003a 100644 --- a/ext/fts5/test/fts5synonym.test +++ b/ext/fts5/test/fts5synonym.test @@ -23,15 +23,15 @@ ifcapable !fts5 { foreach S { {zero 0} - {one 1} - {two 2} + {one 1 i} + {two 2 ii} {three 3 iii} - {four 4} - {five 5} - {six 6} - {seven 7} - {eight 8} - {nine 9} + {four 4 iv} + {five 5 v} + {six 6 vi} + {seven 7 vii} + {eight 8 viii} + {nine 9 ix} } { foreach s $S { set o [list] @@ -186,16 +186,23 @@ do_execsql_test 3.2.0 { one 1 4 three 1 2 two 1 2 } do_execsql_test 3.2.1 { - SELECT rowid FROM ft WHERE ft MATCH 'one two three'; - SELECT rowid FROM ft WHERE ft MATCH 'one + one + two + three'; -} {1 1} -do_execsql_test 3.2.2 { - SELECT rowid FROM ft WHERE ft MATCH 'one two two three'; - SELECT rowid FROM ft WHERE ft MATCH 'one + two + two + three'; + SELECT rowid FROM ft WHERE ft MATCH 'one'; } {1} +do_execsql_test 3.2.2 { + SELECT rowid FROM ft WHERE ft MATCH 'one two three'; +} {1} +do_execsql_test 3.2.3 { + SELECT rowid FROM ft WHERE ft MATCH 'one + one + two + three'; +} {1} +do_execsql_test 3.2.4 { + SELECT rowid FROM ft WHERE ft MATCH 'one two two three'; +} {1} +do_execsql_test 3.2.5 { + SELECT rowid FROM ft WHERE ft MATCH 'one + two + two + three'; +} {} #------------------------------------------------------------------------- -# Check that expressions with synonyms can be parsed. +# Check that expressions with synonyms can be parsed and executed. # reset_db sqlite3_fts5_create_tokenizer db tcl tcl_create @@ -212,11 +219,72 @@ proc tcl_tokenize {tflags text} { foreach {tn expr res} { 1 {abc} {"abc"} - 2 {one} {"one"|"1"} + 2 {one} {"one"|"i"|"1"} 3 {3} {"3"|"iii"|"three"} 4 {3*} {"3"|"iii"|"three" *} } { - do_execsql_test 4.$tn {SELECT fts5_expr($expr, 'tokenize=tcl')} [list $res] + do_execsql_test 4.1.$tn {SELECT fts5_expr($expr, 'tokenize=tcl')} [list $res] +} + +do_execsql_test 4.2.1 { + CREATE VIRTUAL TABLE xx USING fts5(x, tokenize=tcl); + INSERT INTO xx VALUES('one two'); + INSERT INTO xx VALUES('three four'); +} + +do_execsql_test 4.2.2 { + SELECT rowid FROM xx WHERE xx MATCH '2' +} {1} + +do_execsql_test 4.2.3 { + SELECT rowid FROM xx WHERE xx MATCH '3' +} {2} + +do_test 5.0 { + execsql { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, tokenize=tcl) + } + foreach {rowid a b} { + 1 {four v 4 i three} {1 3 five five 4 one} + 2 {5 1 3 4 i} {2 2 v two 4} + 3 {5 i 5 2 four 4 1} {iii ii five two 1} + 4 {ii four 4 one 5 three five} {one 5 1 iii 4 3} + 5 {three i v i four 4 1} {ii five five five iii} + 6 {4 2 ii two 2 iii} {three 1 four 4 iv 1 iv} + 7 {ii ii two three 2 5} {iii i ii iii iii one one} + 8 {2 ii i two 3 three 2} {two iv v iii 3 five} + 9 {i 2 iv 3 five four v} {iii 4 three i three ii 1} + } { + execsql { INSERT INTO t1(rowid, a, b) VALUES($rowid, $a, $b) } + } +} {} + +foreach {tn q res} { + 1 {one} { + 1 {four v 4 [i] three} {[1] 3 five five 4 [one]} + 2 {5 [1] 3 4 [i]} {2 2 v two 4} + 3 {5 [i] 5 2 four 4 [1]} {iii ii five two [1]} + 4 {ii four 4 [one] 5 three five} {[one] 5 [1] iii 4 3} + 5 {three [i] v [i] four 4 [1]} {ii five five five iii} + 6 {4 2 ii two 2 iii} {three [1] four 4 iv [1] iv} + 7 {ii ii two three 2 5} {iii [i] ii iii iii [one] [one]} + 8 {2 ii [i] two 3 three 2} {two iv v iii 3 five} + 9 {[i] 2 iv 3 five four v} {iii 4 three [i] three ii [1]} + } + 2 {five four} { + 1 {[four] [v] [4] i three} {1 3 [five] [five] [4] one} + 2 {[5] 1 3 [4] i} {2 2 [v] two [4]} + 3 {[5] i [5] 2 [four] [4] 1} {iii ii [five] two 1} + 4 {ii [four] [4] one [5] three [five]} {one [5] 1 iii [4] 3} + 5 {three i [v] i [four] [4] 1} {ii [five] [five] [five] iii} + 8 {2 ii i two 3 three 2} {two [iv] [v] iii 3 [five]} + 9 {i 2 [iv] 3 [five] [four] [v]} {iii [4] three i three ii 1} + } +} { + do_execsql_test 5.1.$tn { + SELECT rowid, highlight(t1, 0, '[', ']'), highlight(t1, 1, '[', ']') + FROM t1 WHERE t1 MATCH $q + } $res } finish_test diff --git a/manifest b/manifest index 4d5891206e..4a0414c4b0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Begin\schanges\sto\sallow\ssynonym\ssupport\sby\sadding\smultiple\sterms\sto\sa\squery\s(an\salternative\sto\sadding\smultiple\sterms\sto\sthe\sFTS\sindex). -D 2015-08-31T20:06:06.235 +C Add\stests\sfor\sfts5\ssynonyms\simplemented\sby\sadding\sextra\sterms\sto\squeries.\sAnd\sfixes\sfor\sthe\ssame. +D 2015-09-01T18:08:36.324 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -106,11 +106,11 @@ F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95 F ext/fts5/extract_api_docs.tcl 06583c935f89075ea0b32f85efa5dd7619fcbd03 F ext/fts5/fts5.h 0784692f406588e6c90e13a78e1f36e7e3236e42 -F ext/fts5/fts5Int.h 9c538f0fcc1c3bc2fa12f7199d1326bd2362ce9c +F ext/fts5/fts5Int.h d46f89aeb357fbcf5b268d71b0d2c5000cd27bd9 F ext/fts5/fts5_aux.c 7a307760a9c57c750d043188ec0bad59f5b5ec7e F ext/fts5/fts5_buffer.c 80f9ba4431848cb857e3d2158f5280093dcd8015 F ext/fts5/fts5_config.c 80b61fd2c6844b64a3e72a64572d50a812da9384 -F ext/fts5/fts5_expr.c 1458c3e33c1ec3ad99284f9692edfc49c44afd7c +F ext/fts5/fts5_expr.c 99560f28339b635ba0e0b13f80586c0be58fc680 F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246 F ext/fts5/fts5_index.c 076c4995bf06a6d1559a6e31f9a86b90f2105374 F ext/fts5/fts5_main.c b00834ac543431dc35edbe18018b4befe0c7fd42 @@ -172,7 +172,7 @@ F ext/fts5/test/fts5rank.test 11dcebba31d822f7e99685b4ea2c2ae3ec0b16f1 F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17 F ext/fts5/test/fts5rowid.test 6f9833b23b176dc4aa15b7fc02afeb2b220fd460 -F ext/fts5/test/fts5synonym.test 3343a05f7b1d3fc6fa4559bd3bd0ee979928c30d +F ext/fts5/test/fts5synonym.test a2b0fb9a584417a9c02554aa465ed9084653cdde F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89 F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841 F ext/fts5/test/fts5unicode2.test c1dd890ba32b7609adba78e420faa847abe43b59 @@ -1381,7 +1381,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 98d07d16cab92f1e7001afbe370df3ec6343fc1f -R e2a3883c4d0440bf5b53c5a71ba46260 +P ad7feaed4cd6b1d6e6376bb82d1f5664ddd083f3 +R 7ab2a9e80bd7422fd7466024fe59c669 U dan -Z 7c01c61f2a729597f3365ce7db7a6ce6 +Z 679d87b3074cd417fe8a5b64a65c64bd diff --git a/manifest.uuid b/manifest.uuid index b0ebea4bef..2ddf214da5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ad7feaed4cd6b1d6e6376bb82d1f5664ddd083f3 \ No newline at end of file +dbcb73802b88f76be17f09f3eb83ffac542de633 \ No newline at end of file From d917ad019c446dedc74abf76cd49da512ed0fcbb Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 1 Sep 2015 18:44:33 +0000 Subject: [PATCH 020/100] Fix a problem that occurs when more than 4 synonyms for a term appear within a single row. FossilOrigin-Name: cd359550bdc2bf7be4c294b60130c9fc3f583902 --- ext/fts5/fts5_expr.c | 16 ++++++- ext/fts5/test/fts5synonym.test | 88 ++++++++++++++++++++++++++++++++++ manifest | 14 +++--- manifest.uuid | 2 +- 4 files changed, 111 insertions(+), 9 deletions(-) diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index 24d63c1cab..a44c4e4427 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -395,6 +395,7 @@ static int fts5ExprSynonymPoslist( Fts5PoslistReader aStatic[4]; Fts5PoslistReader *aIter = aStatic; int nIter = 0; + int nAlloc = 4; int rc = SQLITE_OK; Fts5ExprTerm *p; @@ -406,7 +407,18 @@ static int fts5ExprSynonymPoslist( int n; i64 dummy; rc = sqlite3Fts5IterPoslist(pIter, &a, &n, &dummy); - if( rc!=SQLITE_OK ) return rc; + if( rc!=SQLITE_OK ) goto synonym_poslist_out; + if( nIter==nAlloc ){ + int nByte = sizeof(Fts5PoslistReader) * nAlloc * 2; + Fts5PoslistReader *aNew = (Fts5PoslistReader*)sqlite3_malloc(nByte); + if( aNew==0 ){ + rc = SQLITE_NOMEM; + goto synonym_poslist_out; + } + memcpy(aNew, aIter, sizeof(Fts5PoslistReader) * nIter); + nAlloc = nAlloc*2; + aIter = aNew; + } if( sqlite3Fts5PoslistReaderInit(-1, a, n, &aIter[nIter])==0 ){ nIter++; } @@ -447,6 +459,8 @@ static int fts5ExprSynonymPoslist( } } + synonym_poslist_out: + if( aIter!=aStatic ) sqlite3_free(aIter); return rc; } diff --git a/ext/fts5/test/fts5synonym.test b/ext/fts5/test/fts5synonym.test index 5aa29e003a..ddd685c155 100644 --- a/ext/fts5/test/fts5synonym.test +++ b/ext/fts5/test/fts5synonym.test @@ -280,6 +280,17 @@ foreach {tn q res} { 8 {2 ii i two 3 three 2} {two [iv] [v] iii 3 [five]} 9 {i 2 [iv] 3 [five] [four] [v]} {iii [4] three i three ii 1} } + 3 {one OR two OR iii OR 4 OR v} { + 1 {[four] [v] [4] [i] [three]} {[1] [3] [five] [five] [4] [one]} + 2 {[5] [1] [3] [4] [i]} {[2] [2] [v] [two] [4]} + 3 {[5] [i] [5] [2] [four] [4] [1]} {[iii] [ii] [five] [two] [1]} + 4 {[ii] [four] [4] [one] [5] [three] [five]} {[one] [5] [1] [iii] [4] [3]} + 5 {[three] [i] [v] [i] [four] [4] [1]} {[ii] [five] [five] [five] [iii]} + 6 {[4] [2] [ii] [two] [2] [iii]} {[three] [1] [four] [4] [iv] [1] [iv]} + 7 {[ii] [ii] [two] [three] [2] [5]} {[iii] [i] [ii] [iii] [iii] [one] [one]} + 8 {[2] [ii] [i] [two] [3] [three] [2]} {[two] [iv] [v] [iii] [3] [five]} + 9 {[i] [2] [iv] [3] [five] [four] [v]} {[iii] [4] [three] [i] [three] [ii] [1]} + } } { do_execsql_test 5.1.$tn { SELECT rowid, highlight(t1, 0, '[', ']'), highlight(t1, 1, '[', ']') @@ -287,5 +298,82 @@ foreach {tn q res} { } $res } +#------------------------------------------------------------------------- +# Test terms with more than 4 synonyms. +# +reset_db +sqlite3_fts5_create_tokenizer db tcl tcl_create +proc tcl_tokenize {tflags text} { + foreach {w iStart iEnd} [do_tokenize_split $text] { + sqlite3_fts5_token $w $iStart $iEnd + if {$tflags=="query" && [string length $w]==1} { + for {set i 2} {$i<=10} {incr i} { + sqlite3_fts5_token -colo [string repeat $w $i] $iStart $iEnd + } + } + } +} + +do_test 6.0 { + execsql { + CREATE VIRTUAL TABLE t2 USING fts5(a, b, tokenize=tcl) + } + foreach {rowid a b} { + 1 {yyyy vvvvv qq oo yyyyyy vvvv eee} {ffff uu r qq aaaa} + 2 {ww oooooo bbbbb ssssss mm} {ffffff yy iiii rr s ccc qqqqq} + 3 {zzzz llll gggggg cccc uu} {hhhhhh aaaa ppppp rr ee jjjj} + 4 {r f i rrrrrr ww hhh} {aa yyy t x aaaaa ii} + 5 {fffff mm vvvv ooo ffffff kkkk tttt} {cccccc bb e zzz d n} + 6 {iii dddd hh qqqq ddd ooo} {ttt d c b aaaaaa qqqq} + 7 {jjjj rrrr v zzzzz u tt t} {ppppp pp dddd mm hhh uuu} + 8 {gggg rrrrrr kkkk vvvv gggg jjjjjj b} {dddddd jj r w cccc wwwwww ss} + 9 {kkkkk qqq oooo e tttttt mmm} {e ss qqqqqq hhhh llllll gg} + } { + execsql { INSERT INTO t2(rowid, a, b) VALUES($rowid, $a, $b) } + } +} {} + +foreach {tn q res} { + 1 {a} { + 1 {yyyy vvvvv qq oo yyyyyy vvvv eee} {ffff uu r qq [aaaa]} + 3 {zzzz llll gggggg cccc uu} {hhhhhh [aaaa] ppppp rr ee jjjj} + 4 {r f i rrrrrr ww hhh} {[aa] yyy t x [aaaaa] ii} + 6 {iii dddd hh qqqq ddd ooo} {ttt d c b [aaaaaa] qqqq} + } + + 2 {a AND q} { + 1 {yyyy vvvvv [qq] oo yyyyyy vvvv eee} {ffff uu r [qq] [aaaa]} + 6 {iii dddd hh [qqqq] ddd ooo} {ttt d c b [aaaaaa] [qqqq]} + } + + 3 {o OR (q AND a)} { + 1 {yyyy vvvvv [qq] [oo] yyyyyy vvvv eee} {ffff uu r [qq] [aaaa]} + 2 {ww [oooooo] bbbbb ssssss mm} {ffffff yy iiii rr s ccc qqqqq} + 5 {fffff mm vvvv [ooo] ffffff kkkk tttt} {cccccc bb e zzz d n} + 6 {iii dddd hh [qqqq] ddd [ooo]} {ttt d c b [aaaaaa] [qqqq]} + 9 {kkkkk qqq [oooo] e tttttt mmm} {e ss qqqqqq hhhh llllll gg} + } +} { + do_execsql_test 6.1.$tn { + SELECT rowid, highlight(t2, 0, '[', ']'), highlight(t2, 1, '[', ']') + FROM t2 WHERE t2 MATCH $q + } $res +} + +do_execsql_test 6.2.1 { + INSERT INTO t2(rowid, a, b) VALUES(13, + 'x xx xxx xxxx xxxxx xxxxxx xxxxxxx', 'y yy yyy yyyy yyyyy yyyyyy yyyyyyy' + ); + SELECT rowid, highlight(t2, 0, '<', '>'), highlight(t2, 1, '(', ')') + FROM t2 WHERE t2 MATCH 'x OR y' +} { + 1 { vvvvv qq oo vvvv eee} {ffff uu r qq aaaa} + 2 {ww oooooo bbbbb ssssss mm} {ffffff (yy) iiii rr s ccc qqqqq} + 4 {r f i rrrrrr ww hhh} {aa (yyy) t (x) aaaaa ii} + 13 { } + {(y) (yy) (yyy) (yyyy) (yyyyy) (yyyyyy) (yyyyyyy)} +} + + finish_test diff --git a/manifest b/manifest index 4a0414c4b0..55a78b93ad 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stests\sfor\sfts5\ssynonyms\simplemented\sby\sadding\sextra\sterms\sto\squeries.\sAnd\sfixes\sfor\sthe\ssame. -D 2015-09-01T18:08:36.324 +C Fix\sa\sproblem\sthat\soccurs\swhen\smore\sthan\s4\ssynonyms\sfor\sa\sterm\sappear\swithin\sa\ssingle\srow. +D 2015-09-01T18:44:33.194 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -110,7 +110,7 @@ F ext/fts5/fts5Int.h d46f89aeb357fbcf5b268d71b0d2c5000cd27bd9 F ext/fts5/fts5_aux.c 7a307760a9c57c750d043188ec0bad59f5b5ec7e F ext/fts5/fts5_buffer.c 80f9ba4431848cb857e3d2158f5280093dcd8015 F ext/fts5/fts5_config.c 80b61fd2c6844b64a3e72a64572d50a812da9384 -F ext/fts5/fts5_expr.c 99560f28339b635ba0e0b13f80586c0be58fc680 +F ext/fts5/fts5_expr.c 44caa0ccd7e9a392864fda0c14e9b9829a395a84 F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246 F ext/fts5/fts5_index.c 076c4995bf06a6d1559a6e31f9a86b90f2105374 F ext/fts5/fts5_main.c b00834ac543431dc35edbe18018b4befe0c7fd42 @@ -172,7 +172,7 @@ F ext/fts5/test/fts5rank.test 11dcebba31d822f7e99685b4ea2c2ae3ec0b16f1 F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17 F ext/fts5/test/fts5rowid.test 6f9833b23b176dc4aa15b7fc02afeb2b220fd460 -F ext/fts5/test/fts5synonym.test a2b0fb9a584417a9c02554aa465ed9084653cdde +F ext/fts5/test/fts5synonym.test 4321e59c29186d9187cf8ab576e70530176eae49 F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89 F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841 F ext/fts5/test/fts5unicode2.test c1dd890ba32b7609adba78e420faa847abe43b59 @@ -1381,7 +1381,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ad7feaed4cd6b1d6e6376bb82d1f5664ddd083f3 -R 7ab2a9e80bd7422fd7466024fe59c669 +P dbcb73802b88f76be17f09f3eb83ffac542de633 +R 07c5ba5d7a2806d5caa7dc123b3f32ad U dan -Z 679d87b3074cd417fe8a5b64a65c64bd +Z fc32f7df10846658b7b8e62d59624854 diff --git a/manifest.uuid b/manifest.uuid index 2ddf214da5..6c61644fa9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dbcb73802b88f76be17f09f3eb83ffac542de633 \ No newline at end of file +cd359550bdc2bf7be4c294b60130c9fc3f583902 \ No newline at end of file From df998c3d4f05e59411db9abca726ae2cc81d2d5c Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 2 Sep 2015 08:22:41 +0000 Subject: [PATCH 021/100] Fix a problem with fts5 synonyms and the xQueryPhrase() auxiliary function API. FossilOrigin-Name: cf3e45e76d23e10ee06296c3561a341591597a04 --- ext/fts5/fts5Int.h | 2 +- ext/fts5/fts5_expr.c | 156 +++++++++++++++++---------------- ext/fts5/fts5_main.c | 2 +- ext/fts5/test/fts5synonym.test | 26 ++++++ manifest | 18 ++-- manifest.uuid | 2 +- 6 files changed, 118 insertions(+), 88 deletions(-) diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index 1ad61617d6..1da017fb01 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -599,7 +599,7 @@ int sqlite3Fts5ExprPhraseCount(Fts5Expr*); int sqlite3Fts5ExprPhraseSize(Fts5Expr*, int iPhrase); int sqlite3Fts5ExprPoslist(Fts5Expr*, int, const u8 **); -int sqlite3Fts5ExprPhraseExpr(Fts5Config*, Fts5Expr*, int, Fts5Expr**); +int sqlite3Fts5ExprClonePhrase(Fts5Config*, Fts5Expr*, int, Fts5Expr**); /******************************************* ** The fts5_expr.c API above this point is used by the other hand-written diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index a44c4e4427..d4fd6c3634 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -251,79 +251,6 @@ int sqlite3Fts5ExprNew( return sParse.rc; } -/* -** Create a new FTS5 expression by cloning phrase iPhrase of the -** expression passed as the second argument. -*/ -int sqlite3Fts5ExprPhraseExpr( - Fts5Config *pConfig, - Fts5Expr *pExpr, - int iPhrase, - Fts5Expr **ppNew -){ - int rc = SQLITE_OK; /* Return code */ - Fts5ExprPhrase *pOrig; /* The phrase extracted from pExpr */ - Fts5ExprPhrase *pCopy; /* Copy of pOrig */ - Fts5Expr *pNew = 0; /* Expression to return via *ppNew */ - - pOrig = pExpr->apExprPhrase[iPhrase]; - pCopy = (Fts5ExprPhrase*)sqlite3Fts5MallocZero(&rc, - sizeof(Fts5ExprPhrase) + sizeof(Fts5ExprTerm) * pOrig->nTerm - ); - if( pCopy ){ - int i; /* Used to iterate through phrase terms */ - Fts5ExprPhrase **apPhrase; - Fts5ExprNode *pNode; - Fts5ExprNearset *pNear; - - pNew = (Fts5Expr*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Expr)); - apPhrase = (Fts5ExprPhrase**)sqlite3Fts5MallocZero(&rc, - sizeof(Fts5ExprPhrase*) - ); - pNode = (Fts5ExprNode*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5ExprNode)); - pNear = (Fts5ExprNearset*)sqlite3Fts5MallocZero(&rc, - sizeof(Fts5ExprNearset) + sizeof(Fts5ExprPhrase*) - ); - - for(i=0; inTerm; i++){ - pCopy->aTerm[i].zTerm = sqlite3Fts5Strndup(&rc, pOrig->aTerm[i].zTerm,-1); - pCopy->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix; - } - - if( rc==SQLITE_OK ){ - /* All the allocations succeeded. Put the expression object together. */ - pNew->pIndex = pExpr->pIndex; - pNew->pRoot = pNode; - pNew->nPhrase = 1; - pNew->apExprPhrase = apPhrase; - pNew->apExprPhrase[0] = pCopy; - - pNode->eType = (pOrig->nTerm==1 ? FTS5_TERM : FTS5_STRING); - pNode->pNear = pNear; - - pNear->nPhrase = 1; - pNear->apPhrase[0] = pCopy; - - pCopy->nTerm = pOrig->nTerm; - pCopy->pNode = pNode; - }else{ - /* At least one allocation failed. Free them all. */ - for(i=0; inTerm; i++){ - sqlite3_free(pCopy->aTerm[i].zTerm); - } - sqlite3_free(pCopy); - sqlite3_free(pNear); - sqlite3_free(pNode); - sqlite3_free(apPhrase); - sqlite3_free(pNew); - pNew = 0; - } - } - - *ppNew = pNew; - return rc; -} - /* ** Free the expression node object passed as the only argument. */ @@ -1550,8 +1477,8 @@ static int fts5ParseTokenize( int tflags, /* Mask of FTS5_TOKEN_* flags */ const char *pToken, /* Buffer containing token */ int nToken, /* Size of token in bytes */ - int iStart, /* Start offset of token */ - int iEnd /* End offset of token */ + int iUnused1, /* Start offset of token */ + int iUnused2 /* End offset of token */ ){ int rc = SQLITE_OK; const int SZALLOC = 8; @@ -1577,7 +1504,7 @@ static int fts5ParseTokenize( pNew = (Fts5ExprPhrase*)sqlite3_realloc(pPhrase, sizeof(Fts5ExprPhrase) + sizeof(Fts5ExprTerm) * nNew - ); + ); if( pNew==0 ) return SQLITE_NOMEM; if( pPhrase==0 ) memset(pNew, 0, sizeof(Fts5ExprPhrase)); pCtx->pPhrase = pPhrase = pNew; @@ -1675,6 +1602,83 @@ Fts5ExprPhrase *sqlite3Fts5ParseTerm( return sCtx.pPhrase; } +/* +** Create a new FTS5 expression by cloning phrase iPhrase of the +** expression passed as the second argument. +*/ +int sqlite3Fts5ExprClonePhrase( + Fts5Config *pConfig, + Fts5Expr *pExpr, + int iPhrase, + Fts5Expr **ppNew +){ + int rc = SQLITE_OK; /* Return code */ + Fts5ExprPhrase *pOrig; /* The phrase extracted from pExpr */ + Fts5ExprPhrase *pCopy; /* Copy of pOrig */ + int i; /* Used to iterate through phrase terms */ + + Fts5Expr *pNew = 0; /* Expression to return via *ppNew */ + Fts5ExprPhrase **apPhrase; /* pNew->apPhrase */ + Fts5ExprNode *pNode; /* pNew->pRoot */ + Fts5ExprNearset *pNear; /* pNew->pRoot->pNear */ + + TokenCtx sCtx = {0}; /* Context object for fts5ParseTokenize */ + + + pOrig = pExpr->apExprPhrase[iPhrase]; + + pNew = (Fts5Expr*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Expr)); + if( rc==SQLITE_OK ){ + pNew->apExprPhrase = (Fts5ExprPhrase**)sqlite3Fts5MallocZero(&rc, + sizeof(Fts5ExprPhrase*)); + } + if( rc==SQLITE_OK ){ + pNew->pRoot = (Fts5ExprNode*)sqlite3Fts5MallocZero(&rc, + sizeof(Fts5ExprNode)); + } + if( rc==SQLITE_OK ){ + pNew->pRoot->pNear = (Fts5ExprNearset*)sqlite3Fts5MallocZero(&rc, + sizeof(Fts5ExprNearset) + sizeof(Fts5ExprPhrase*)); + } + + for(i=0; rc==SQLITE_OK && inTerm; i++){ + int tflags = 0; + Fts5ExprTerm *p; + for(p=&pOrig->aTerm[i]; p && rc==SQLITE_OK; p=p->pSynonym){ + const char *zTerm = p->zTerm; + rc = fts5ParseTokenize((void*)&sCtx, tflags, zTerm, strlen(zTerm), 0, 0); + tflags = FTS5_TOKEN_COLOCATED; + } + if( rc==SQLITE_OK ){ + sCtx.pPhrase->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix; + } + } + + if( rc==SQLITE_OK ){ + /* All the allocations succeeded. Put the expression object together. */ + pNew->pIndex = pExpr->pIndex; + pNew->nPhrase = 1; + pNew->apExprPhrase[0] = sCtx.pPhrase; + pNew->pRoot->pNear->apPhrase[0] = sCtx.pPhrase; + pNew->pRoot->pNear->nPhrase = 1; + sCtx.pPhrase->pNode = pNew->pRoot; + + if( pOrig->nTerm==1 && pOrig->aTerm[0].pSynonym==0 ){ + pNew->pRoot->eType = FTS5_TERM; + }else{ + pNew->pRoot->eType = FTS5_STRING; + } + }else{ + sqlite3Fts5ExprFree(pNew); + fts5ExprPhraseFree(sCtx.pPhrase); + pNew = 0; + } + + *ppNew = pNew; + return rc; +} + + /* ** Token pTok has appeared in a MATCH expression where the NEAR operator ** is expected. If token pTok does not contain "NEAR", store an error diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index 4704fb8906..f337005d00 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -1860,7 +1860,7 @@ static int fts5ApiQueryPhrase( pNew->iFirstRowid = SMALLEST_INT64; pNew->iLastRowid = LARGEST_INT64; pNew->base.pVtab = (sqlite3_vtab*)pTab; - rc = sqlite3Fts5ExprPhraseExpr(pConf, pCsr->pExpr, iPhrase, &pNew->pExpr); + rc = sqlite3Fts5ExprClonePhrase(pConf, pCsr->pExpr, iPhrase, &pNew->pExpr); } if( rc==SQLITE_OK ){ diff --git a/ext/fts5/test/fts5synonym.test b/ext/fts5/test/fts5synonym.test index ddd685c155..7bc3ffaf90 100644 --- a/ext/fts5/test/fts5synonym.test +++ b/ext/fts5/test/fts5synonym.test @@ -298,6 +298,32 @@ foreach {tn q res} { } $res } +# Test that the xQueryPhrase() API works with synonyms. +# +proc mit {blob} { + set scan(littleEndian) i* + set scan(bigEndian) I* + binary scan $blob $scan($::tcl_platform(byteOrder)) r + return $r +} +db func mit mit +sqlite3_fts5_register_matchinfo db + +foreach {tn q res} { + 1 {one} { + 1 {1 11 7 2 12 6} 2 {2 11 7 0 12 6} + 3 {2 11 7 1 12 6} 4 {1 11 7 2 12 6} + 5 {3 11 7 0 12 6} 6 {0 11 7 2 12 6} + 7 {0 11 7 3 12 6} 8 {1 11 7 0 12 6} + 9 {1 11 7 2 12 6} + } +} { + do_execsql_test 5.2.$tn { + SELECT rowid, mit(matchinfo(t1, 'x')) FROM t1 WHERE t1 MATCH $q + } $res +} + + #------------------------------------------------------------------------- # Test terms with more than 4 synonyms. # diff --git a/manifest b/manifest index 55a78b93ad..c5af116d50 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\sthat\soccurs\swhen\smore\sthan\s4\ssynonyms\sfor\sa\sterm\sappear\swithin\sa\ssingle\srow. -D 2015-09-01T18:44:33.194 +C Fix\sa\sproblem\swith\sfts5\ssynonyms\sand\sthe\sxQueryPhrase()\sauxiliary\sfunction\sAPI. +D 2015-09-02T08:22:41.779 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -106,14 +106,14 @@ F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95 F ext/fts5/extract_api_docs.tcl 06583c935f89075ea0b32f85efa5dd7619fcbd03 F ext/fts5/fts5.h 0784692f406588e6c90e13a78e1f36e7e3236e42 -F ext/fts5/fts5Int.h d46f89aeb357fbcf5b268d71b0d2c5000cd27bd9 +F ext/fts5/fts5Int.h c6f035091fc9fa12a92c7066bf0c266c08cb508b F ext/fts5/fts5_aux.c 7a307760a9c57c750d043188ec0bad59f5b5ec7e F ext/fts5/fts5_buffer.c 80f9ba4431848cb857e3d2158f5280093dcd8015 F ext/fts5/fts5_config.c 80b61fd2c6844b64a3e72a64572d50a812da9384 -F ext/fts5/fts5_expr.c 44caa0ccd7e9a392864fda0c14e9b9829a395a84 +F ext/fts5/fts5_expr.c d92beea335f96f9256cc19203b21fd45826655c9 F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246 F ext/fts5/fts5_index.c 076c4995bf06a6d1559a6e31f9a86b90f2105374 -F ext/fts5/fts5_main.c b00834ac543431dc35edbe18018b4befe0c7fd42 +F ext/fts5/fts5_main.c e9d0892424bb7f0a8b58613d4ff75cb650cf286e F ext/fts5/fts5_storage.c c888defbb961d64c12299b3d1725a24a770b047e F ext/fts5/fts5_tcl.c 6da58d6e8f42a93c4486b5ba9b187a7f995dee37 F ext/fts5/fts5_test_mi.c 80a9e86fb4c5b6b58f8fefac05e9b96d1a6574e1 @@ -172,7 +172,7 @@ F ext/fts5/test/fts5rank.test 11dcebba31d822f7e99685b4ea2c2ae3ec0b16f1 F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17 F ext/fts5/test/fts5rowid.test 6f9833b23b176dc4aa15b7fc02afeb2b220fd460 -F ext/fts5/test/fts5synonym.test 4321e59c29186d9187cf8ab576e70530176eae49 +F ext/fts5/test/fts5synonym.test fd66afccec36bf719ba66a5b6579efe007607b05 F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89 F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841 F ext/fts5/test/fts5unicode2.test c1dd890ba32b7609adba78e420faa847abe43b59 @@ -1381,7 +1381,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P dbcb73802b88f76be17f09f3eb83ffac542de633 -R 07c5ba5d7a2806d5caa7dc123b3f32ad +P cd359550bdc2bf7be4c294b60130c9fc3f583902 +R f0fefb84cb3010de72f87418a092a81a U dan -Z fc32f7df10846658b7b8e62d59624854 +Z b6230653499a35c239ca936f1dd1428a diff --git a/manifest.uuid b/manifest.uuid index 6c61644fa9..7c523ff1d0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cd359550bdc2bf7be4c294b60130c9fc3f583902 \ No newline at end of file +cf3e45e76d23e10ee06296c3561a341591597a04 \ No newline at end of file From 583611df65d799dec0b86cec7627106b9ddf0c81 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 2 Sep 2015 14:17:38 +0000 Subject: [PATCH 022/100] Fix a problem handling OOM conditions within fts5 queries that feature synonyms. FossilOrigin-Name: 11fa980897c6c7be218bbd9c4cd8253272d2c300 --- ext/fts5/fts5_expr.c | 44 +++++++++++++++-------- ext/fts5/fts5_test_mi.c | 13 ++++--- ext/fts5/test/fts5_common.tcl | 33 +++++++++++++++++ ext/fts5/test/fts5fault6.test | 66 ++++++++++++++++++++++++++++++++++ ext/fts5/test/fts5synonym.test | 40 ++++----------------- manifest | 20 +++++------ manifest.uuid | 2 +- 7 files changed, 155 insertions(+), 63 deletions(-) diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index d4fd6c3634..cc24179561 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -1467,6 +1467,7 @@ Fts5ExprNearset *sqlite3Fts5ParseNearset( typedef struct TokenCtx TokenCtx; struct TokenCtx { Fts5ExprPhrase *pPhrase; + int rc; }; /* @@ -1485,17 +1486,23 @@ static int fts5ParseTokenize( TokenCtx *pCtx = (TokenCtx*)pContext; Fts5ExprPhrase *pPhrase = pCtx->pPhrase; + /* If an error has already occurred, this is a no-op */ + if( pCtx->rc!=SQLITE_OK ) return pCtx->rc; + assert( pPhrase==0 || pPhrase->nTerm>0 ); if( pPhrase && (tflags & FTS5_TOKEN_COLOCATED) ){ Fts5ExprTerm *pSyn; int nByte = sizeof(Fts5ExprTerm) + nToken+1; pSyn = (Fts5ExprTerm*)sqlite3_malloc(nByte); - if( pSyn==0 ) return SQLITE_NOMEM; - memset(pSyn, 0, nByte); - pSyn->zTerm = (char*)&pSyn[1]; - memcpy(pSyn->zTerm, pToken, nToken); - pSyn->pSynonym = pPhrase->aTerm[pPhrase->nTerm-1].pSynonym; - pPhrase->aTerm[pPhrase->nTerm-1].pSynonym = pSyn; + if( pSyn==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pSyn, 0, nByte); + pSyn->zTerm = (char*)&pSyn[1]; + memcpy(pSyn->zTerm, pToken, nToken); + pSyn->pSynonym = pPhrase->aTerm[pPhrase->nTerm-1].pSynonym; + pPhrase->aTerm[pPhrase->nTerm-1].pSynonym = pSyn; + } }else{ Fts5ExprTerm *pTerm; if( pPhrase==0 || (pPhrase->nTerm % SZALLOC)==0 ){ @@ -1505,16 +1512,23 @@ static int fts5ParseTokenize( pNew = (Fts5ExprPhrase*)sqlite3_realloc(pPhrase, sizeof(Fts5ExprPhrase) + sizeof(Fts5ExprTerm) * nNew ); - if( pNew==0 ) return SQLITE_NOMEM; - if( pPhrase==0 ) memset(pNew, 0, sizeof(Fts5ExprPhrase)); - pCtx->pPhrase = pPhrase = pNew; - pNew->nTerm = nNew - SZALLOC; + if( pNew==0 ){ + rc = SQLITE_NOMEM; + }else{ + if( pPhrase==0 ) memset(pNew, 0, sizeof(Fts5ExprPhrase)); + pCtx->pPhrase = pPhrase = pNew; + pNew->nTerm = nNew - SZALLOC; + } } - pTerm = &pPhrase->aTerm[pPhrase->nTerm++]; - memset(pTerm, 0, sizeof(Fts5ExprTerm)); - pTerm->zTerm = sqlite3Fts5Strndup(&rc, pToken, nToken); + if( rc==SQLITE_OK ){ + pTerm = &pPhrase->aTerm[pPhrase->nTerm++]; + memset(pTerm, 0, sizeof(Fts5ExprTerm)); + pTerm->zTerm = sqlite3Fts5Strndup(&rc, pToken, nToken); + } } + + pCtx->rc = rc; return rc; } @@ -1573,7 +1587,7 @@ Fts5ExprPhrase *sqlite3Fts5ParseTerm( rc = sqlite3Fts5Tokenize(pConfig, flags, z, n, &sCtx, fts5ParseTokenize); } sqlite3_free(z); - if( rc ){ + if( rc || (rc = sCtx.rc) ){ pParse->rc = rc; fts5ExprPhraseFree(sCtx.pPhrase); sCtx.pPhrase = 0; @@ -1622,7 +1636,7 @@ int sqlite3Fts5ExprClonePhrase( Fts5ExprNode *pNode; /* pNew->pRoot */ Fts5ExprNearset *pNear; /* pNew->pRoot->pNear */ - TokenCtx sCtx = {0}; /* Context object for fts5ParseTokenize */ + TokenCtx sCtx = {0,0}; /* Context object for fts5ParseTokenize */ pOrig = pExpr->apExprPhrase[iPhrase]; diff --git a/ext/fts5/fts5_test_mi.c b/ext/fts5/fts5_test_mi.c index 8ebf5f5077..355f23330d 100644 --- a/ext/fts5/fts5_test_mi.c +++ b/ext/fts5/fts5_test_mi.c @@ -352,7 +352,7 @@ static void fts5MatchinfoFunc( ){ const char *zArg; Fts5MatchinfoCtx *p; - int rc; + int rc = SQLITE_OK; if( nVal>0 ){ zArg = (const char*)sqlite3_value_text(apVal[0]); @@ -363,11 +363,16 @@ static void fts5MatchinfoFunc( p = (Fts5MatchinfoCtx*)pApi->xGetAuxdata(pFts, 0); if( p==0 || sqlite3_stricmp(zArg, p->zArg) ){ p = fts5MatchinfoNew(pApi, pFts, pCtx, zArg); - pApi->xSetAuxdata(pFts, p, sqlite3_free); - if( p==0 ) return; + if( p==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = pApi->xSetAuxdata(pFts, p, sqlite3_free); + } } - rc = fts5MatchinfoIter(pApi, pFts, p, fts5MatchinfoLocalCb); + if( rc==SQLITE_OK ){ + rc = fts5MatchinfoIter(pApi, pFts, p, fts5MatchinfoLocalCb); + } if( rc!=SQLITE_OK ){ sqlite3_result_error_code(pCtx, rc); }else{ diff --git a/ext/fts5/test/fts5_common.tcl b/ext/fts5/test/fts5_common.tcl index 59058a8462..5eba1296ad 100644 --- a/ext/fts5/test/fts5_common.tcl +++ b/ext/fts5/test/fts5_common.tcl @@ -295,3 +295,36 @@ proc NOT {a b} { return $a } +#------------------------------------------------------------------------- +# This command is similar to [split], except that it also provides the +# start and end offsets of each token. For example: +# +# [fts5_tokenize_split "abc d ef"] -> {abc 0 3 d 4 5 ef 6 8} +# + +proc gobble_whitespace {textvar} { + upvar $textvar t + regexp {([ ]*)(.*)} $t -> space t + return [string length $space] +} + +proc gobble_text {textvar wordvar} { + upvar $textvar t + upvar $wordvar w + regexp {([^ ]*)(.*)} $t -> w t + return [string length $w] +} + +proc fts5_tokenize_split {text} { + set token "" + set ret [list] + set iOff [gobble_whitespace text] + while {[set nToken [gobble_text text word]]} { + lappend ret $word $iOff [expr $iOff+$nToken] + incr iOff $nToken + incr iOff [gobble_whitespace text] + } + + set ret +} + diff --git a/ext/fts5/test/fts5fault6.test b/ext/fts5/test/fts5fault6.test index b9657be1cc..b92df3ce79 100644 --- a/ext/fts5/test/fts5fault6.test +++ b/ext/fts5/test/fts5fault6.test @@ -148,5 +148,71 @@ do_faultsim_test 4.1 -faults oom-t* -prep { faultsim_test_result {0 {}} } +#------------------------------------------------------------------------- +# OOM while running a query that includes synonyms and matchinfo(). +# +proc mit {blob} { + set scan(littleEndian) i* + set scan(bigEndian) I* + binary scan $blob $scan($::tcl_platform(byteOrder)) r + return $r +} +proc tcl_tokenize {tflags text} { + foreach {w iStart iEnd} [fts5_tokenize_split $text] { + sqlite3_fts5_token $w $iStart $iEnd + if {$tflags=="query" && [string length $w]==1} { + for {set i 2} {$i < 7} {incr i} { + sqlite3_fts5_token -colo [string repeat $w $i] $iStart $iEnd + } + } + } +} +proc tcl_create {args} { return "tcl_tokenize" } +reset_db +sqlite3_fts5_create_tokenizer db tcl tcl_create +db func mit mit +sqlite3_fts5_register_matchinfo db +do_test 5.0 { + execsql { CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=tcl) } + foreach {rowid text} { + 1 {aaaa cc b aaaaa cc aa} + 2 {aa aa bb a bbb} + 3 {bb aaaaa aaaaa b aaaa aaaaa} + 4 {aa a b aaaa aa} + 5 {aa b ccc aaaaa cc} + 6 {aa aaaaa bbbb cc aaa} + 7 {aaaaa aa aa ccccc bb} + 8 {ccc bbbbb ccccc bbb c} + 9 {cccccc bbbb a aaa cccc c} + } { + execsql { INSERT INTO t1(rowid, a) VALUES($rowid, $text) } + } +} {} + +set res [list {*}{ + 1 {3 24 8 2 12 6} + 5 {2 24 8 2 12 6} + 6 {3 24 8 1 12 6} + 7 {3 24 8 1 12 6} + 9 {2 24 8 3 12 6} +}] +do_execsql_test 5.1 { + SELECT rowid, mit(matchinfo(t1, 'x')) FROM t1 WHERE t1 MATCH 'a AND c' +} $res + +faultsim_save_and_close +do_faultsim_test 5.2 -faults oom* -prep { + faultsim_restore_and_reopen + sqlite3_fts5_create_tokenizer db tcl tcl_create + sqlite3_fts5_register_matchinfo db + db func mit mit +} -body { + db eval { + SELECT rowid, mit(matchinfo(t1, 'x')) FROM t1 WHERE t1 MATCH 'a AND c' + } +} -test { + faultsim_test_result [list 0 $::res] +} + finish_test diff --git a/ext/fts5/test/fts5synonym.test b/ext/fts5/test/fts5synonym.test index 7bc3ffaf90..6acc5617f0 100644 --- a/ext/fts5/test/fts5synonym.test +++ b/ext/fts5/test/fts5synonym.test @@ -40,34 +40,8 @@ foreach S { } } -proc gobble_whitespace {textvar} { - upvar $textvar t - regexp {([ ]*)(.*)} $t -> space t - return [string length $space] -} - -proc gobble_text {textvar wordvar} { - upvar $textvar t - upvar $wordvar w - regexp {([^ ]*)(.*)} $t -> w t - return [string length $w] -} - -proc do_tokenize_split {text} { - set token "" - set ret [list] - set iOff [gobble_whitespace text] - while {[set nToken [gobble_text text word]]} { - lappend ret $word $iOff [expr $iOff+$nToken] - incr iOff $nToken - incr iOff [gobble_whitespace text] - } - - set ret -} - proc tcl_tokenize {tflags text} { - foreach {w iStart iEnd} [do_tokenize_split $text] { + foreach {w iStart iEnd} [fts5_tokenize_split $text] { sqlite3_fts5_token $w $iStart $iEnd } } @@ -95,7 +69,7 @@ do_execsql_test 1.0 { # proc tcl_tokenize {tflags text} { - foreach {w iStart iEnd} [do_tokenize_split $text] { + foreach {w iStart iEnd} [fts5_tokenize_split $text] { sqlite3_fts5_token $w $iStart $iEnd if {$tflags=="document" && [info exists ::syn($w)]} { foreach s $::syn($w) { @@ -137,7 +111,7 @@ reset_db sqlite3_fts5_create_tokenizer db tcl tcl_create proc tcl_tokenize {tflags text} { set bColo 1 - foreach {w iStart iEnd} [do_tokenize_split $text] { + foreach {w iStart iEnd} [fts5_tokenize_split $text] { if {$bColo} { sqlite3_fts5_token -colo $w $iStart $iEnd set bColo 0 @@ -160,7 +134,7 @@ do_execsql_test 3.1.1 { } {} proc tcl_tokenize {tflags text} { - foreach {w iStart iEnd} [do_tokenize_split $text] { + foreach {w iStart iEnd} [fts5_tokenize_split $text] { sqlite3_fts5_token $w $iStart $iEnd } } @@ -172,7 +146,7 @@ do_execsql_test 3.1.2 { reset_db sqlite3_fts5_create_tokenizer db tcl tcl_create proc tcl_tokenize {tflags text} { - foreach {w iStart iEnd} [do_tokenize_split $text] { + foreach {w iStart iEnd} [fts5_tokenize_split $text] { sqlite3_fts5_token $w $iStart $iEnd sqlite3_fts5_token -colo $w $iStart $iEnd } @@ -207,7 +181,7 @@ do_execsql_test 3.2.5 { reset_db sqlite3_fts5_create_tokenizer db tcl tcl_create proc tcl_tokenize {tflags text} { - foreach {w iStart iEnd} [do_tokenize_split $text] { + foreach {w iStart iEnd} [fts5_tokenize_split $text] { sqlite3_fts5_token $w $iStart $iEnd if {$tflags=="query" && [info exists ::syn($w)]} { foreach s $::syn($w) { @@ -330,7 +304,7 @@ foreach {tn q res} { reset_db sqlite3_fts5_create_tokenizer db tcl tcl_create proc tcl_tokenize {tflags text} { - foreach {w iStart iEnd} [do_tokenize_split $text] { + foreach {w iStart iEnd} [fts5_tokenize_split $text] { sqlite3_fts5_token $w $iStart $iEnd if {$tflags=="query" && [string length $w]==1} { for {set i 2} {$i<=10} {incr i} { diff --git a/manifest b/manifest index c5af116d50..01f188aef6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\swith\sfts5\ssynonyms\sand\sthe\sxQueryPhrase()\sauxiliary\sfunction\sAPI. -D 2015-09-02T08:22:41.779 +C Fix\sa\sproblem\shandling\sOOM\sconditions\swithin\sfts5\squeries\sthat\sfeature\ssynonyms. +D 2015-09-02T14:17:38.670 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -110,20 +110,20 @@ F ext/fts5/fts5Int.h c6f035091fc9fa12a92c7066bf0c266c08cb508b F ext/fts5/fts5_aux.c 7a307760a9c57c750d043188ec0bad59f5b5ec7e F ext/fts5/fts5_buffer.c 80f9ba4431848cb857e3d2158f5280093dcd8015 F ext/fts5/fts5_config.c 80b61fd2c6844b64a3e72a64572d50a812da9384 -F ext/fts5/fts5_expr.c d92beea335f96f9256cc19203b21fd45826655c9 +F ext/fts5/fts5_expr.c 5ae9bae1a36583a1e85f742f972c65bffd7e9d3b F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246 F ext/fts5/fts5_index.c 076c4995bf06a6d1559a6e31f9a86b90f2105374 F ext/fts5/fts5_main.c e9d0892424bb7f0a8b58613d4ff75cb650cf286e F ext/fts5/fts5_storage.c c888defbb961d64c12299b3d1725a24a770b047e F ext/fts5/fts5_tcl.c 6da58d6e8f42a93c4486b5ba9b187a7f995dee37 -F ext/fts5/fts5_test_mi.c 80a9e86fb4c5b6b58f8fefac05e9b96d1a6574e1 +F ext/fts5/fts5_test_mi.c e96be827aa8f571031e65e481251dc1981d608bf F ext/fts5/fts5_tokenize.c 710541513ecf3fe6d9365326fc85aee6efe97229 F ext/fts5/fts5_unicode2.c 78273fbd588d1d9bd0a7e4e0ccc9207348bae33c F ext/fts5/fts5_varint.c 3f86ce09cab152e3d45490d7586b7ed2e40c13f1 F ext/fts5/fts5_vocab.c 4622e0b7d84a488a1585aaa56eb214ee67a988bc F ext/fts5/fts5parse.y 833db1101b78c0c47686ab1b84918e38c36e9452 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba -F ext/fts5/test/fts5_common.tcl 3338968de1880ca12b0451ae8f9b8b12d14e0ba7 +F ext/fts5/test/fts5_common.tcl b6e6a40ef5d069c8e86ca4fbad491e1195485dbc F ext/fts5/test/fts5aa.test caa44c528f7270aa4e325c4f2c28d355c1e8c307 F ext/fts5/test/fts5ab.test 6fe3a56731d15978afbb74ae51b355fc9310f2ad F ext/fts5/test/fts5ac.test 9737992d08c56bfd4803e933744d2d764e23795c @@ -156,7 +156,7 @@ F ext/fts5/test/fts5fault2.test 28c36c843bb39ae855ba79827417ecc37f114341 F ext/fts5/test/fts5fault3.test d6e9577d4312e331a913c72931bf131704efc8f3 F ext/fts5/test/fts5fault4.test 762991d526ee67c2b374351a17248097ea38bee7 F ext/fts5/test/fts5fault5.test 54da9fd4c3434a1d4f6abdcb6469299d91cf5875 -F ext/fts5/test/fts5fault6.test 234dc6355f8d3f8b5be2763f30699d770247c215 +F ext/fts5/test/fts5fault6.test 233a29d4a086cd09cfae262bbb15c709a438d010 F ext/fts5/test/fts5full.test 6f6143af0c6700501d9fd597189dfab1555bb741 F ext/fts5/test/fts5hash.test 42eb066f667e9a389a63437cb7038c51974d4fc6 F ext/fts5/test/fts5integrity.test 29f41d2c7126c6122fbb5d54e556506456876145 @@ -172,7 +172,7 @@ F ext/fts5/test/fts5rank.test 11dcebba31d822f7e99685b4ea2c2ae3ec0b16f1 F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17 F ext/fts5/test/fts5rowid.test 6f9833b23b176dc4aa15b7fc02afeb2b220fd460 -F ext/fts5/test/fts5synonym.test fd66afccec36bf719ba66a5b6579efe007607b05 +F ext/fts5/test/fts5synonym.test dd3c9016345e1bbd7384ec25bbcd3de1f1700475 F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89 F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841 F ext/fts5/test/fts5unicode2.test c1dd890ba32b7609adba78e420faa847abe43b59 @@ -1381,7 +1381,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P cd359550bdc2bf7be4c294b60130c9fc3f583902 -R f0fefb84cb3010de72f87418a092a81a +P cf3e45e76d23e10ee06296c3561a341591597a04 +R 0e89c166acdebcc1b36d66580652787d U dan -Z b6230653499a35c239ca936f1dd1428a +Z 84847a00fa2f61f76712fa9a6aa07879 diff --git a/manifest.uuid b/manifest.uuid index 7c523ff1d0..985c544ce9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cf3e45e76d23e10ee06296c3561a341591597a04 \ No newline at end of file +11fa980897c6c7be218bbd9c4cd8253272d2c300 \ No newline at end of file From bea34fc53af825712a983dd3f1a717fcb9e6c03d Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 2 Sep 2015 17:34:22 +0000 Subject: [PATCH 023/100] Fix a problem with fts5 synonyms and phrase queries. Also fix an OOM handling bug in fts5. FossilOrigin-Name: a4c35fa2c94fe34b376670244fe72303c99868c1 --- ext/fts5/fts5_expr.c | 61 +++++++++++++++++++++++----------- ext/fts5/test/fts5fault6.test | 58 ++++++++++++++++++++++++++++++-- ext/fts5/test/fts5synonym.test | 19 +++++++++++ main.mk | 46 +++++++++++++------------ manifest | 18 +++++----- manifest.uuid | 2 +- 6 files changed, 151 insertions(+), 53 deletions(-) diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index cc24179561..8796ba5e2d 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -288,8 +288,8 @@ static int fts5ExprColsetTest(Fts5ExprColset *pColset, int iCol){ ** Argument pTerm must be a synonym iterator. Return the current rowid ** that it points to. */ -static i64 fts5ExprSynonymRowid(Fts5ExprTerm *pTerm, int bDesc){ - i64 iRet; +static i64 fts5ExprSynonymRowid(Fts5ExprTerm *pTerm, int bDesc, int *pbEof){ + i64 iRet = 0; int bRetValid = 0; Fts5ExprTerm *p; @@ -305,7 +305,7 @@ static i64 fts5ExprSynonymRowid(Fts5ExprTerm *pTerm, int bDesc){ } } - assert( bRetValid ); + if( pbEof && bRetValid==0 ) *pbEof = 1; return iRet; } @@ -346,9 +346,9 @@ static int fts5ExprSynonymPoslist( nAlloc = nAlloc*2; aIter = aNew; } - if( sqlite3Fts5PoslistReaderInit(-1, a, n, &aIter[nIter])==0 ){ - nIter++; - } + sqlite3Fts5PoslistReaderInit(-1, a, n, &aIter[nIter]); + assert( aIter[nIter].bEof==0 ); + nIter++; } } @@ -659,7 +659,7 @@ static int fts5ExprNearAdvanceFirst( Fts5ExprTerm *p; /* Find the firstest rowid any synonym points to. */ - i64 iRowid = fts5ExprSynonymRowid(pTerm, pExpr->bDesc); + i64 iRowid = fts5ExprSynonymRowid(pTerm, pExpr->bDesc, 0); /* Advance each iterator that currently points to iRowid. Or, if iFrom ** is valid - each iterator that points to a rowid before iFrom. */ @@ -738,6 +738,35 @@ static int fts5ExprAdvanceto( return 0; } +static int fts5ExprSynonymAdvanceto( + Fts5ExprTerm *pTerm, /* Term iterator to advance */ + int bDesc, /* True if iterator is "rowid DESC" */ + i64 *piLast, /* IN/OUT: Lastest rowid seen so far */ + int *pRc, /* OUT: Error code */ + int *pbEof /* OUT: Set to true if EOF */ +){ + int rc = SQLITE_OK; + i64 iLast = *piLast; + Fts5ExprTerm *p; + + for(p=pTerm; rc==SQLITE_OK && p; p=p->pSynonym){ + if( sqlite3Fts5IterEof(p->pIter)==0 ){ + i64 iRowid = sqlite3Fts5IterRowid(p->pIter); + if( (bDesc==0 && iLast>iRowid) || (bDesc && iLastpIter, iLast); + } + } + } + + if( rc!=SQLITE_OK ){ + *pbEof = 1; + }else{ + *piLast = fts5ExprSynonymRowid(pTerm, bDesc, pbEof); + } + + return rc; +} + /* ** IN/OUT parameter (*pa) points to a position list n bytes in size. If ** the position list contains entries for column iCol, then (*pa) is set @@ -906,7 +935,7 @@ static int fts5ExprNearNextMatch( ** the maximum rowid. Or, if the iterator is "ORDER BY rowid DESC", then it ** means the minimum rowid. */ if( pLeft->aTerm[0].pSynonym ){ - iLast = fts5ExprSynonymRowid(&pLeft->aTerm[0], bDesc); + iLast = fts5ExprSynonymRowid(&pLeft->aTerm[0], bDesc, 0); }else{ iLast = sqlite3Fts5IterRowid(pLeft->aTerm[0].pIter); } @@ -920,21 +949,13 @@ static int fts5ExprNearNextMatch( if( pTerm->pSynonym ){ Fts5ExprTerm *p; int bEof = 1; - i64 iRowid = fts5ExprSynonymRowid(pTerm, bDesc); + i64 iRowid = fts5ExprSynonymRowid(pTerm, bDesc, 0); if( iRowid==iLast ) continue; - for(p=pTerm; p; p=p->pSynonym){ - Fts5IndexIter *pIter = p->pIter; - int dummy; - if( 0==sqlite3Fts5IterEof(pIter) - && 0==fts5ExprAdvanceto(pIter, bDesc, &iLast, &rc, &dummy)==0 - ){ - bEof = 0; - } - } - if( bEof || rc ){ - pNode->bEof = 1; + bMatch = 0; + if( fts5ExprSynonymAdvanceto(pTerm,bDesc,&iLast,&rc,&pNode->bEof) ){ return rc; } + }else{ Fts5IndexIter *pIter = pPhrase->aTerm[j].pIter; i64 iRowid = sqlite3Fts5IterRowid(pIter); diff --git a/ext/fts5/test/fts5fault6.test b/ext/fts5/test/fts5fault6.test index b92df3ce79..b739e87418 100644 --- a/ext/fts5/test/fts5fault6.test +++ b/ext/fts5/test/fts5fault6.test @@ -22,6 +22,8 @@ ifcapable !fts5 { return } +if 1 { + #------------------------------------------------------------------------- # OOM while rebuilding an FTS5 table. # @@ -148,8 +150,14 @@ do_faultsim_test 4.1 -faults oom-t* -prep { faultsim_test_result {0 {}} } +} + #------------------------------------------------------------------------- -# OOM while running a query that includes synonyms and matchinfo(). +# +# 5.2.* OOM while running a query that includes synonyms and matchinfo(). +# +# 5.3.* OOM while running a query that returns a row containing instances +# of more than 4 synonyms for a single term. # proc mit {blob} { set scan(littleEndian) i* @@ -174,6 +182,7 @@ db func mit mit sqlite3_fts5_register_matchinfo db do_test 5.0 { execsql { CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=tcl) } + execsql { INSERT INTO t1(t1, rank) VALUES('pgsz', 32) } foreach {rowid text} { 1 {aaaa cc b aaaaa cc aa} 2 {aa aa bb a bbb} @@ -184,6 +193,37 @@ do_test 5.0 { 7 {aaaaa aa aa ccccc bb} 8 {ccc bbbbb ccccc bbb c} 9 {cccccc bbbb a aaa cccc c} + + 20 {ddd f ddd eeeee fff ffff eeee ddd fff eeeee dddddd eeee} + 21 {fffff eee dddd fffff dd ee ee eeeee eee eeeeee ee dd e} + 22 {fffff d eeee dddd fffff dddddd ffff ddddd eeeee ee eee dddd ddddd} + 23 {ddddd fff ddd eeeee ffff eeee ddd ff ff ffffff eeeeee dddd ffffff} + 24 {eee dd ee dddd dddd eeeeee e eee fff ffff} + 25 {ddddd ffffff dddddd fff ddd ddddd ddd f eeee fff dddd f} + 26 {f ffff fff fff eeeeee dddd d dddddd ddddd eee ff eeeee} + 27 {eee fff dddddd eeeee eeeee dddd ddddd ffff f eeeee eee dddddd ddddd d} + 28 {dd ddddd d ddd d fff d dddd ee dddd ee ddd dddddd dddddd} + 29 {eeee dddd ee dddd eeee dddd dd fffff f ddd eeeee ddd ee} + 30 {ff ffffff eeeeee eeeee eee ffffff ff ffff f fffff eeeee} + 31 {fffff eeeeee dddd eeee eeee eeeeee eee fffff d ddddd ffffff ffff dddddd} + 32 {dddddd fffff ee eeeeee eeee ee fff dddd fff eeee ffffff eeeeee ffffff} + 33 {ddddd eeee dd ffff dddddd fff eeee ddddd ffff eeee ddd} + 34 {ee dddd ddddd dddddd eeee eeeeee f dd ee dddddd ffffff} + 35 {ee dddd dd eeeeee ddddd eee d eeeeee dddddd eee dddd fffff} + 36 {eee ffffff ffffff e fffff eeeee ff dddddd dddddd fff} + 37 {eeeee fffff dddddd dddd ffffff fff f dd ee dd dd eeeee} + 38 {eeeeee ee d ff eeeeee eeeeee eee eeeee ee ffffff dddd eeee dddddd ee} + 39 {eeeeee ddd fffff e dddd ee eee eee ffffff ee f d dddd} + 40 {ffffff dddddd eee ee ffffff eee eeee ddddd ee eeeeee f} + 41 {ddd ddd fff fffff ee fffff f fff ddddd fffff} + 42 {dddd ee ff d f ffffff fff ffffff ff dd dddddd f eeee} + 43 {d dd fff fffff d f fff e dddd ee ee} + 44 {ff ffff eee ddd d dd ffff dddd d eeee d eeeeee} + 45 {eeee f eeeee ee e ffff f ddd e fff} + 46 {ffff d ffff eeee ffff eeeee f ffff ddddd eee} + 47 {dd dd dddddd ddddd fffff dddddd ddd ddddd eeeeee ffff eeee eee ee} + 48 {ffff ffff e dddd ffffff dd dd dddd f fffff} + 49 {ffffff d dddddd ffff eeeee f ffff ffff d dd fffff eeeee} } { execsql { INSERT INTO t1(rowid, a) VALUES($rowid, $text) } } @@ -196,9 +236,12 @@ set res [list {*}{ 7 {3 24 8 1 12 6} 9 {2 24 8 3 12 6} }] -do_execsql_test 5.1 { +do_execsql_test 5.1.1 { SELECT rowid, mit(matchinfo(t1, 'x')) FROM t1 WHERE t1 MATCH 'a AND c' } $res +do_execsql_test 5.1.2 { + SELECT count(*) FROM t1 WHERE t1 MATCH 'd e f' +} 29 faultsim_save_and_close do_faultsim_test 5.2 -faults oom* -prep { @@ -214,5 +257,16 @@ do_faultsim_test 5.2 -faults oom* -prep { faultsim_test_result [list 0 $::res] } +do_faultsim_test 5.3 -faults oom* -prep { + faultsim_restore_and_reopen + sqlite3_fts5_create_tokenizer db tcl tcl_create +} -body { + db eval { + SELECT count(*) FROM t1 WHERE t1 MATCH 'd AND e AND f' + } +} -test { + faultsim_test_result {0 29} +} + finish_test diff --git a/ext/fts5/test/fts5synonym.test b/ext/fts5/test/fts5synonym.test index 6acc5617f0..33d20a014e 100644 --- a/ext/fts5/test/fts5synonym.test +++ b/ext/fts5/test/fts5synonym.test @@ -93,6 +93,7 @@ foreach {tn expr res} { 2 "eight OR 8 OR 5" {2 3} 3 "10" {} 4 "1*" {1} + 5 "1 + 2" {1} } { do_execsql_test 2.1.$tn { SELECT rowid FROM ft WHERE ft MATCH $expr @@ -233,6 +234,7 @@ do_test 5.0 { } } {} + foreach {tn q res} { 1 {one} { 1 {four v 4 [i] three} {[1] 3 five five 4 [one]} @@ -265,6 +267,23 @@ foreach {tn q res} { 8 {[2] [ii] [i] [two] [3] [three] [2]} {[two] [iv] [v] [iii] [3] [five]} 9 {[i] [2] [iv] [3] [five] [four] [v]} {[iii] [4] [three] [i] [three] [ii] [1]} } + + 4 {5 + 1} { + 2 {[5 1] 3 4 i} {2 2 v two 4} + 3 {[5 i] 5 2 four 4 1} {iii ii five two 1} + 4 {ii four 4 one 5 three five} {one [5 1] iii 4 3} + 5 {three i [v i] four 4 1} {ii five five five iii} + } + + 5 {one + two + three} { + 7 {ii ii two three 2 5} {iii [i ii iii] iii one one} + 8 {2 ii [i two 3] three 2} {two iv v iii 3 five} + } + + 6 {"v v"} { + 1 {four v 4 i three} {1 3 [five five] 4 one} + 5 {three i v i four 4 1} {ii [five five five] iii} + } } { do_execsql_test 5.1.$tn { SELECT rowid, highlight(t1, 0, '[', ']'), highlight(t1, 1, '[', ']') diff --git a/main.mk b/main.mk index 9988fe2775..828a824eee 100644 --- a/main.mk +++ b/main.mk @@ -47,6 +47,7 @@ TCCX = $(TCC) $(OPTS) -I. -I$(TOP)/src -I$(TOP) TCCX += -I$(TOP)/ext/rtree -I$(TOP)/ext/icu -I$(TOP)/ext/fts3 TCCX += -I$(TOP)/ext/async -I$(TOP)/ext/userauth +TCCX += -I$(TOP)/ext/fts5 # Object files for the SQLite library. # @@ -230,6 +231,29 @@ SRC += \ $(TOP)/ext/rbu/sqlite3rbu.h +# FTS5 things +# +FTS5_HDR = \ + $(TOP)/ext/fts5/fts5.h \ + $(TOP)/ext/fts5/fts5Int.h \ + fts5parse.h + +FTS5_SRC = \ + $(TOP)/ext/fts5/fts5_aux.c \ + $(TOP)/ext/fts5/fts5_buffer.c \ + $(TOP)/ext/fts5/fts5_main.c \ + $(TOP)/ext/fts5/fts5_config.c \ + $(TOP)/ext/fts5/fts5_expr.c \ + $(TOP)/ext/fts5/fts5_hash.c \ + $(TOP)/ext/fts5/fts5_index.c \ + fts5parse.c \ + $(TOP)/ext/fts5/fts5_storage.c \ + $(TOP)/ext/fts5/fts5_tokenize.c \ + $(TOP)/ext/fts5/fts5_unicode2.c \ + $(TOP)/ext/fts5/fts5_varint.c \ + $(TOP)/ext/fts5/fts5_vocab.c \ + + # Generated source code files # SRC += \ @@ -636,25 +660,6 @@ fts3_write.o: $(TOP)/ext/fts3/fts3_write.c $(HDR) $(EXTHDR) rtree.o: $(TOP)/ext/rtree/rtree.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/rtree/rtree.c -# FTS5 things -# -FTS5_SRC = \ - $(TOP)/ext/fts5/fts5.h \ - $(TOP)/ext/fts5/fts5Int.h \ - $(TOP)/ext/fts5/fts5_aux.c \ - $(TOP)/ext/fts5/fts5_buffer.c \ - $(TOP)/ext/fts5/fts5_main.c \ - $(TOP)/ext/fts5/fts5_config.c \ - $(TOP)/ext/fts5/fts5_expr.c \ - $(TOP)/ext/fts5/fts5_hash.c \ - $(TOP)/ext/fts5/fts5_index.c \ - fts5parse.c fts5parse.h \ - $(TOP)/ext/fts5/fts5_storage.c \ - $(TOP)/ext/fts5/fts5_tokenize.c \ - $(TOP)/ext/fts5/fts5_unicode2.c \ - $(TOP)/ext/fts5/fts5_varint.c \ - $(TOP)/ext/fts5/fts5_vocab.c \ - fts5parse.c: $(TOP)/ext/fts5/fts5parse.y lemon cp $(TOP)/ext/fts5/fts5parse.y . rm -f fts5parse.h @@ -662,11 +667,10 @@ fts5parse.c: $(TOP)/ext/fts5/fts5parse.y lemon fts5parse.h: fts5parse.c -fts5.c: $(FTS5_SRC) +fts5.c: $(FTS5_SRC) $(FTS5_HDR) tclsh $(TOP)/ext/fts5/tool/mkfts5c.tcl cp $(TOP)/ext/fts5/fts5.h . - userauth.o: $(TOP)/ext/userauth/userauth.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/userauth/userauth.c diff --git a/manifest b/manifest index 01f188aef6..30f63e1524 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\shandling\sOOM\sconditions\swithin\sfts5\squeries\sthat\sfeature\ssynonyms. -D 2015-09-02T14:17:38.670 +C Fix\sa\sproblem\swith\sfts5\ssynonyms\sand\sphrase\squeries.\sAlso\sfix\san\sOOM\shandling\sbug\sin\sfts5. +D 2015-09-02T17:34:22.663 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -110,7 +110,7 @@ F ext/fts5/fts5Int.h c6f035091fc9fa12a92c7066bf0c266c08cb508b F ext/fts5/fts5_aux.c 7a307760a9c57c750d043188ec0bad59f5b5ec7e F ext/fts5/fts5_buffer.c 80f9ba4431848cb857e3d2158f5280093dcd8015 F ext/fts5/fts5_config.c 80b61fd2c6844b64a3e72a64572d50a812da9384 -F ext/fts5/fts5_expr.c 5ae9bae1a36583a1e85f742f972c65bffd7e9d3b +F ext/fts5/fts5_expr.c 408384b2f540dcf72a363e1cde4c3e9896e8d41d F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246 F ext/fts5/fts5_index.c 076c4995bf06a6d1559a6e31f9a86b90f2105374 F ext/fts5/fts5_main.c e9d0892424bb7f0a8b58613d4ff75cb650cf286e @@ -156,7 +156,7 @@ F ext/fts5/test/fts5fault2.test 28c36c843bb39ae855ba79827417ecc37f114341 F ext/fts5/test/fts5fault3.test d6e9577d4312e331a913c72931bf131704efc8f3 F ext/fts5/test/fts5fault4.test 762991d526ee67c2b374351a17248097ea38bee7 F ext/fts5/test/fts5fault5.test 54da9fd4c3434a1d4f6abdcb6469299d91cf5875 -F ext/fts5/test/fts5fault6.test 233a29d4a086cd09cfae262bbb15c709a438d010 +F ext/fts5/test/fts5fault6.test 93e9d43ab1e05ae5efa30cfde17649b3bb6a82c3 F ext/fts5/test/fts5full.test 6f6143af0c6700501d9fd597189dfab1555bb741 F ext/fts5/test/fts5hash.test 42eb066f667e9a389a63437cb7038c51974d4fc6 F ext/fts5/test/fts5integrity.test 29f41d2c7126c6122fbb5d54e556506456876145 @@ -172,7 +172,7 @@ F ext/fts5/test/fts5rank.test 11dcebba31d822f7e99685b4ea2c2ae3ec0b16f1 F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17 F ext/fts5/test/fts5rowid.test 6f9833b23b176dc4aa15b7fc02afeb2b220fd460 -F ext/fts5/test/fts5synonym.test dd3c9016345e1bbd7384ec25bbcd3de1f1700475 +F ext/fts5/test/fts5synonym.test 88efc3de51a2c46aff984d74f06c833e4def0493 F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89 F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841 F ext/fts5/test/fts5unicode2.test c1dd890ba32b7609adba78e420faa847abe43b59 @@ -259,7 +259,7 @@ F ext/userauth/userauth.c 5fa3bdb492f481bbc1709fc83c91ebd13460c69e F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk 04840e8277ab5159af16172eafd214dae7cffff5 +F main.mk 8a8468564b2e387116614fcda2a6d4f5f754b955 F mkopcodec.awk c2ff431854d702cdd2d779c9c0d1f58fa16fa4ea F mkopcodeh.awk 0e7f04a8eb90f92259e47d80110e4e98d7ce337a F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 @@ -1381,7 +1381,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P cf3e45e76d23e10ee06296c3561a341591597a04 -R 0e89c166acdebcc1b36d66580652787d +P 11fa980897c6c7be218bbd9c4cd8253272d2c300 +R 6df2cc151a0891be96a1f7bf13aeaf04 U dan -Z 84847a00fa2f61f76712fa9a6aa07879 +Z 72b6e5381768fc37d01c74eff21e6a40 diff --git a/manifest.uuid b/manifest.uuid index 985c544ce9..2675a32e17 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -11fa980897c6c7be218bbd9c4cd8253272d2c300 \ No newline at end of file +a4c35fa2c94fe34b376670244fe72303c99868c1 \ No newline at end of file From 50ea76e748403b4f7af07137dc593f100e6a68f4 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 2 Sep 2015 18:56:01 +0000 Subject: [PATCH 024/100] Fix an issue with fts5 synonyms and NEAR(...) queries. FossilOrigin-Name: f2e590700d8eb30da8ac3a92f6ca92a2bbe9c1fe --- ext/fts5/fts5_expr.c | 16 ++++++++-------- ext/fts5/test/fts5synonym.test | 16 +++++++++++++++- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 31 insertions(+), 17 deletions(-) diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index 8796ba5e2d..7cdb5e7632 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -742,12 +742,12 @@ static int fts5ExprSynonymAdvanceto( Fts5ExprTerm *pTerm, /* Term iterator to advance */ int bDesc, /* True if iterator is "rowid DESC" */ i64 *piLast, /* IN/OUT: Lastest rowid seen so far */ - int *pRc, /* OUT: Error code */ - int *pbEof /* OUT: Set to true if EOF */ + int *pRc /* OUT: Error code */ ){ int rc = SQLITE_OK; i64 iLast = *piLast; Fts5ExprTerm *p; + int bEof = 0; for(p=pTerm; rc==SQLITE_OK && p; p=p->pSynonym){ if( sqlite3Fts5IterEof(p->pIter)==0 ){ @@ -759,12 +759,12 @@ static int fts5ExprSynonymAdvanceto( } if( rc!=SQLITE_OK ){ - *pbEof = 1; + *pRc = rc; + bEof = 1; }else{ - *piLast = fts5ExprSynonymRowid(pTerm, bDesc, pbEof); + *piLast = fts5ExprSynonymRowid(pTerm, bDesc, &bEof); } - - return rc; + return bEof; } /* @@ -952,10 +952,10 @@ static int fts5ExprNearNextMatch( i64 iRowid = fts5ExprSynonymRowid(pTerm, bDesc, 0); if( iRowid==iLast ) continue; bMatch = 0; - if( fts5ExprSynonymAdvanceto(pTerm,bDesc,&iLast,&rc,&pNode->bEof) ){ + if( fts5ExprSynonymAdvanceto(pTerm, bDesc, &iLast, &rc) ){ + pNode->bEof = 1; return rc; } - }else{ Fts5IndexIter *pIter = pPhrase->aTerm[j].pIter; i64 iRowid = sqlite3Fts5IterRowid(pIter); diff --git a/ext/fts5/test/fts5synonym.test b/ext/fts5/test/fts5synonym.test index 33d20a014e..a0fb43eca6 100644 --- a/ext/fts5/test/fts5synonym.test +++ b/ext/fts5/test/fts5synonym.test @@ -333,7 +333,16 @@ proc tcl_tokenize {tflags text} { } } -do_test 6.0 { +do_execsql_test 6.0.1 { + CREATE VIRTUAL TABLE t1 USING fts5(x, tokenize=tcl); + INSERT INTO t1 VALUES('yy xx qq'); + INSERT INTO t1 VALUES('yy xx xx'); +} +do_execsql_test 6.0.2 { + SELECT * FROM t1 WHERE t1 MATCH 'NEAR(y q)'; +} {{yy xx qq}} + +do_test 6.0.3 { execsql { CREATE VIRTUAL TABLE t2 USING fts5(a, b, tokenize=tcl) } @@ -372,6 +381,11 @@ foreach {tn q res} { 6 {iii dddd hh [qqqq] ddd [ooo]} {ttt d c b [aaaaaa] [qqqq]} 9 {kkkkk qqq [oooo] e tttttt mmm} {e ss qqqqqq hhhh llllll gg} } + + 4 {NEAR(q y, 20)} { + 1 {[yyyy] vvvvv [qq] oo [yyyyyy] vvvv eee} {ffff uu r qq aaaa} + 2 {ww oooooo bbbbb ssssss mm} {ffffff [yy] iiii rr s ccc [qqqqq]} + } } { do_execsql_test 6.1.$tn { SELECT rowid, highlight(t2, 0, '[', ']'), highlight(t2, 1, '[', ']') diff --git a/manifest b/manifest index 30f63e1524..672b113552 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\swith\sfts5\ssynonyms\sand\sphrase\squeries.\sAlso\sfix\san\sOOM\shandling\sbug\sin\sfts5. -D 2015-09-02T17:34:22.663 +C Fix\san\sissue\swith\sfts5\ssynonyms\sand\sNEAR(...)\squeries. +D 2015-09-02T18:56:01.484 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -110,7 +110,7 @@ F ext/fts5/fts5Int.h c6f035091fc9fa12a92c7066bf0c266c08cb508b F ext/fts5/fts5_aux.c 7a307760a9c57c750d043188ec0bad59f5b5ec7e F ext/fts5/fts5_buffer.c 80f9ba4431848cb857e3d2158f5280093dcd8015 F ext/fts5/fts5_config.c 80b61fd2c6844b64a3e72a64572d50a812da9384 -F ext/fts5/fts5_expr.c 408384b2f540dcf72a363e1cde4c3e9896e8d41d +F ext/fts5/fts5_expr.c 1a6dbdab7a1035210b04f9ddccc019fccefe77d5 F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246 F ext/fts5/fts5_index.c 076c4995bf06a6d1559a6e31f9a86b90f2105374 F ext/fts5/fts5_main.c e9d0892424bb7f0a8b58613d4ff75cb650cf286e @@ -172,7 +172,7 @@ F ext/fts5/test/fts5rank.test 11dcebba31d822f7e99685b4ea2c2ae3ec0b16f1 F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17 F ext/fts5/test/fts5rowid.test 6f9833b23b176dc4aa15b7fc02afeb2b220fd460 -F ext/fts5/test/fts5synonym.test 88efc3de51a2c46aff984d74f06c833e4def0493 +F ext/fts5/test/fts5synonym.test 4bc555107abb16df3873d23d1f735eeab4d24010 F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89 F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841 F ext/fts5/test/fts5unicode2.test c1dd890ba32b7609adba78e420faa847abe43b59 @@ -1381,7 +1381,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 11fa980897c6c7be218bbd9c4cd8253272d2c300 -R 6df2cc151a0891be96a1f7bf13aeaf04 +P a4c35fa2c94fe34b376670244fe72303c99868c1 +R 95a88bfdbba18ffe1c3b172dc2ad91be U dan -Z 72b6e5381768fc37d01c74eff21e6a40 +Z d85f21bea0d3a94f93f876bfbe641651 diff --git a/manifest.uuid b/manifest.uuid index 2675a32e17..14089c65e5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a4c35fa2c94fe34b376670244fe72303c99868c1 \ No newline at end of file +f2e590700d8eb30da8ac3a92f6ca92a2bbe9c1fe \ No newline at end of file From 9c671b741c220ebb7c4667c22eede207e801f100 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 2 Sep 2015 19:48:55 +0000 Subject: [PATCH 025/100] Further tests to raise coverage of fts5 synonym code to 100%. Fix a dropped error code in the same. FossilOrigin-Name: bdedd838bb3028c586bcc9f643852ce1364adb49 --- ext/fts5/fts5_expr.c | 2 +- ext/fts5/fts5_tokenize.c | 2 +- ext/fts5/test/fts5aa.test | 10 ++++++++++ ext/fts5/test/fts5fault6.test | 31 ++++++++++++++++++++++++++++--- ext/fts5/test/fts5synonym.test | 12 +++++++++++- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- 7 files changed, 62 insertions(+), 17 deletions(-) diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index 7cdb5e7632..91753652d1 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -662,7 +662,7 @@ static int fts5ExprNearAdvanceFirst( i64 iRowid = fts5ExprSynonymRowid(pTerm, pExpr->bDesc, 0); /* Advance each iterator that currently points to iRowid. Or, if iFrom - ** is valid - each iterator that points to a rowid before iFrom. */ + ** is valid - each iterator that points to a rowid before iFrom. */ for(p=pTerm; p; p=p->pSynonym){ if( sqlite3Fts5IterEof(p->pIter)==0 ){ i64 ii = sqlite3Fts5IterRowid(p->pIter); diff --git a/ext/fts5/fts5_tokenize.c b/ext/fts5/fts5_tokenize.c index 6b1129c24d..29a4579b44 100644 --- a/ext/fts5/fts5_tokenize.c +++ b/ext/fts5/fts5_tokenize.c @@ -1229,7 +1229,7 @@ int sqlite3Fts5TokenizerInit(fts5_api *pApi){ ); } - return SQLITE_OK; + return rc; } diff --git a/ext/fts5/test/fts5aa.test b/ext/fts5/test/fts5aa.test index 22aa1b0f35..832a972cfc 100644 --- a/ext/fts5/test/fts5aa.test +++ b/ext/fts5/test/fts5aa.test @@ -506,6 +506,16 @@ do_execsql_test 18.3 { SELECT t1.rowid, t2.rowid FROM t2, t1 WHERE t2 MATCH t1.a AND t1.rowid = t2.c } {1 1} +#-------------------------------------------------------------------- +# fts5 table in the temp schema. +# +reset_db +do_execsql_test 19.0 { + CREATE VIRTUAL TABLE temp.t1 USING fts5(x); + INSERT INTO t1 VALUES('x y z'); + INSERT INTO t1 VALUES('w x 1'); + SELECT rowid FROM t1 WHERE t1 MATCH 'x'; +} {1 2} finish_test diff --git a/ext/fts5/test/fts5fault6.test b/ext/fts5/test/fts5fault6.test index b739e87418..c79cf7ab70 100644 --- a/ext/fts5/test/fts5fault6.test +++ b/ext/fts5/test/fts5fault6.test @@ -22,7 +22,6 @@ ifcapable !fts5 { return } -if 1 { #------------------------------------------------------------------------- # OOM while rebuilding an FTS5 table. @@ -150,8 +149,6 @@ do_faultsim_test 4.1 -faults oom-t* -prep { faultsim_test_result {0 {}} } -} - #------------------------------------------------------------------------- # # 5.2.* OOM while running a query that includes synonyms and matchinfo(). @@ -224,6 +221,8 @@ do_test 5.0 { 47 {dd dd dddddd ddddd fffff dddddd ddd ddddd eeeeee ffff eeee eee ee} 48 {ffff ffff e dddd ffffff dd dd dddd f fffff} 49 {ffffff d dddddd ffff eeeee f ffff ffff d dd fffff eeeee} + + 50 {x e} } { execsql { INSERT INTO t1(rowid, a) VALUES($rowid, $text) } } @@ -268,5 +267,31 @@ do_faultsim_test 5.3 -faults oom* -prep { faultsim_test_result {0 29} } +do_faultsim_test 5.4 -faults oom* -prep { + faultsim_restore_and_reopen + sqlite3_fts5_create_tokenizer db tcl tcl_create +} -body { + db eval { + SELECT count(*) FROM t1 WHERE t1 MATCH 'x + e' + } +} -test { + faultsim_test_result {0 1} +} + +#------------------------------------------------------------------------- +catch { db close } +breakpoint +do_faultsim_test 6 -faults oom* -prep { + sqlite_orig db test.db + sqlite3_db_config_lookaside db 0 0 0 +} -body { + load_static_extension db fts5 +} -test { + faultsim_test_result {0 {}} {1 {initialization of fts5 failed: }} + if {$testrc==0} { + db eval { CREATE VIRTUAL TABLE temp.t1 USING fts5(x) } + } + db close +} finish_test diff --git a/ext/fts5/test/fts5synonym.test b/ext/fts5/test/fts5synonym.test index a0fb43eca6..3c44bd7d17 100644 --- a/ext/fts5/test/fts5synonym.test +++ b/ext/fts5/test/fts5synonym.test @@ -387,10 +387,20 @@ foreach {tn q res} { 2 {ww oooooo bbbbb ssssss mm} {ffffff [yy] iiii rr s ccc [qqqqq]} } } { - do_execsql_test 6.1.$tn { + do_execsql_test 6.1.$tn.asc { SELECT rowid, highlight(t2, 0, '[', ']'), highlight(t2, 1, '[', ']') FROM t2 WHERE t2 MATCH $q } $res + + set res2 [list] + foreach {rowid a b} $res { + set res2 [concat [list $rowid $a $b] $res2] + } + + do_execsql_test 6.1.$tn.desc { + SELECT rowid, highlight(t2, 0, '[', ']'), highlight(t2, 1, '[', ']') + FROM t2 WHERE t2 MATCH $q ORDER BY rowid DESC + } $res2 } do_execsql_test 6.2.1 { diff --git a/manifest b/manifest index 672b113552..e7ac7255f4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sissue\swith\sfts5\ssynonyms\sand\sNEAR(...)\squeries. -D 2015-09-02T18:56:01.484 +C Further\stests\sto\sraise\scoverage\sof\sfts5\ssynonym\scode\sto\s100%.\sFix\sa\sdropped\serror\scode\sin\sthe\ssame. +D 2015-09-02T19:48:55.424 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -110,21 +110,21 @@ F ext/fts5/fts5Int.h c6f035091fc9fa12a92c7066bf0c266c08cb508b F ext/fts5/fts5_aux.c 7a307760a9c57c750d043188ec0bad59f5b5ec7e F ext/fts5/fts5_buffer.c 80f9ba4431848cb857e3d2158f5280093dcd8015 F ext/fts5/fts5_config.c 80b61fd2c6844b64a3e72a64572d50a812da9384 -F ext/fts5/fts5_expr.c 1a6dbdab7a1035210b04f9ddccc019fccefe77d5 +F ext/fts5/fts5_expr.c 45a1d1865b648ca0797f98a5f3a50a0f3a282fbb F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246 F ext/fts5/fts5_index.c 076c4995bf06a6d1559a6e31f9a86b90f2105374 F ext/fts5/fts5_main.c e9d0892424bb7f0a8b58613d4ff75cb650cf286e F ext/fts5/fts5_storage.c c888defbb961d64c12299b3d1725a24a770b047e F ext/fts5/fts5_tcl.c 6da58d6e8f42a93c4486b5ba9b187a7f995dee37 F ext/fts5/fts5_test_mi.c e96be827aa8f571031e65e481251dc1981d608bf -F ext/fts5/fts5_tokenize.c 710541513ecf3fe6d9365326fc85aee6efe97229 +F ext/fts5/fts5_tokenize.c f380f46f341af9c9a9908e1aade685ba1eaa157a F ext/fts5/fts5_unicode2.c 78273fbd588d1d9bd0a7e4e0ccc9207348bae33c F ext/fts5/fts5_varint.c 3f86ce09cab152e3d45490d7586b7ed2e40c13f1 F ext/fts5/fts5_vocab.c 4622e0b7d84a488a1585aaa56eb214ee67a988bc F ext/fts5/fts5parse.y 833db1101b78c0c47686ab1b84918e38c36e9452 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba F ext/fts5/test/fts5_common.tcl b6e6a40ef5d069c8e86ca4fbad491e1195485dbc -F ext/fts5/test/fts5aa.test caa44c528f7270aa4e325c4f2c28d355c1e8c307 +F ext/fts5/test/fts5aa.test 86def30ff5e76eb6db66f0833c4aefe0b765d590 F ext/fts5/test/fts5ab.test 6fe3a56731d15978afbb74ae51b355fc9310f2ad F ext/fts5/test/fts5ac.test 9737992d08c56bfd4803e933744d2d764e23795c F ext/fts5/test/fts5ad.test b2edee8b7de0c21d2c88f8a18c195034aad6952d @@ -156,7 +156,7 @@ F ext/fts5/test/fts5fault2.test 28c36c843bb39ae855ba79827417ecc37f114341 F ext/fts5/test/fts5fault3.test d6e9577d4312e331a913c72931bf131704efc8f3 F ext/fts5/test/fts5fault4.test 762991d526ee67c2b374351a17248097ea38bee7 F ext/fts5/test/fts5fault5.test 54da9fd4c3434a1d4f6abdcb6469299d91cf5875 -F ext/fts5/test/fts5fault6.test 93e9d43ab1e05ae5efa30cfde17649b3bb6a82c3 +F ext/fts5/test/fts5fault6.test 97bce1a36b7a64e3203fea504ae8e5cfd5ada423 F ext/fts5/test/fts5full.test 6f6143af0c6700501d9fd597189dfab1555bb741 F ext/fts5/test/fts5hash.test 42eb066f667e9a389a63437cb7038c51974d4fc6 F ext/fts5/test/fts5integrity.test 29f41d2c7126c6122fbb5d54e556506456876145 @@ -172,7 +172,7 @@ F ext/fts5/test/fts5rank.test 11dcebba31d822f7e99685b4ea2c2ae3ec0b16f1 F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17 F ext/fts5/test/fts5rowid.test 6f9833b23b176dc4aa15b7fc02afeb2b220fd460 -F ext/fts5/test/fts5synonym.test 4bc555107abb16df3873d23d1f735eeab4d24010 +F ext/fts5/test/fts5synonym.test 4b6387dcc12fc4e7216db06402375326ae4a28aa F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89 F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841 F ext/fts5/test/fts5unicode2.test c1dd890ba32b7609adba78e420faa847abe43b59 @@ -1381,7 +1381,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P a4c35fa2c94fe34b376670244fe72303c99868c1 -R 95a88bfdbba18ffe1c3b172dc2ad91be +P f2e590700d8eb30da8ac3a92f6ca92a2bbe9c1fe +R 5fe3995e4cfcebadaed779600452a235 U dan -Z d85f21bea0d3a94f93f876bfbe641651 +Z 535c1d64e133dff7ac9edd44481e77b4 diff --git a/manifest.uuid b/manifest.uuid index 14089c65e5..6008a07af2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f2e590700d8eb30da8ac3a92f6ca92a2bbe9c1fe \ No newline at end of file +bdedd838bb3028c586bcc9f643852ce1364adb49 \ No newline at end of file From 2a6ecb3141436fe7c0f2b11d80a0e1d1ae9de4c6 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 3 Sep 2015 10:27:02 +0000 Subject: [PATCH 026/100] Fix a memory leak in fts5_expr.c. FossilOrigin-Name: 399932a181437d84cd57234e76c4b65da1e4ac5f --- ext/fts5/fts5_expr.c | 1 + ext/fts5/test/fts5synonym.test | 22 ++++++++++++++++++++++ manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 31 insertions(+), 8 deletions(-) diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index 91753652d1..ea5fd8a09e 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -344,6 +344,7 @@ static int fts5ExprSynonymPoslist( } memcpy(aNew, aIter, sizeof(Fts5PoslistReader) * nIter); nAlloc = nAlloc*2; + if( aIter!=aStatic ) sqlite3_free(aIter); aIter = aNew; } sqlite3Fts5PoslistReaderInit(-1, a, n, &aIter[nIter]); diff --git a/ext/fts5/test/fts5synonym.test b/ext/fts5/test/fts5synonym.test index 3c44bd7d17..81073dec8b 100644 --- a/ext/fts5/test/fts5synonym.test +++ b/ext/fts5/test/fts5synonym.test @@ -417,6 +417,28 @@ do_execsql_test 6.2.1 { {(y) (yy) (yyy) (yyyy) (yyyyy) (yyyyyy) (yyyyyyy)} } +#------------------------------------------------------------------------- +# Test that the xColumnSize() API is not confused by colocated tokens. +# +reset_db +sqlite3_fts5_create_tokenizer db tcl tcl_create +fts5_aux_test_functions db +proc tcl_tokenize {tflags text} { + foreach {w iStart iEnd} [fts5_tokenize_split $text] { + sqlite3_fts5_token $w $iStart $iEnd + if {[string length $w]==1} { + for {set i 2} {$i<=10} {incr i} { + sqlite3_fts5_token -colo [string repeat $w $i] $iStart $iEnd + } + } + } +} +do_execsql_test 7.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, columnsize=0, tokenize=tcl); + INSERT INTO t1 VALUES('0 2 3', '4 5 6 7'); + INSERT INTO t1 VALUES('8 9', '0 0 0 0 0 0 0 0 0 0'); + SELECT fts5_test_columnsize(t1) FROM t1 WHERE t1 MATCH '000 AND 00 AND 0'; +} {{3 4} {2 10}} finish_test diff --git a/manifest b/manifest index e7ac7255f4..efbe08818c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\stests\sto\sraise\scoverage\sof\sfts5\ssynonym\scode\sto\s100%.\sFix\sa\sdropped\serror\scode\sin\sthe\ssame. -D 2015-09-02T19:48:55.424 +C Fix\sa\smemory\sleak\sin\sfts5_expr.c. +D 2015-09-03T10:27:02.747 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -110,7 +110,7 @@ F ext/fts5/fts5Int.h c6f035091fc9fa12a92c7066bf0c266c08cb508b F ext/fts5/fts5_aux.c 7a307760a9c57c750d043188ec0bad59f5b5ec7e F ext/fts5/fts5_buffer.c 80f9ba4431848cb857e3d2158f5280093dcd8015 F ext/fts5/fts5_config.c 80b61fd2c6844b64a3e72a64572d50a812da9384 -F ext/fts5/fts5_expr.c 45a1d1865b648ca0797f98a5f3a50a0f3a282fbb +F ext/fts5/fts5_expr.c 0c36c1db8eccdeb006e3c8d1499d05015f6e11a6 F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246 F ext/fts5/fts5_index.c 076c4995bf06a6d1559a6e31f9a86b90f2105374 F ext/fts5/fts5_main.c e9d0892424bb7f0a8b58613d4ff75cb650cf286e @@ -172,7 +172,7 @@ F ext/fts5/test/fts5rank.test 11dcebba31d822f7e99685b4ea2c2ae3ec0b16f1 F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17 F ext/fts5/test/fts5rowid.test 6f9833b23b176dc4aa15b7fc02afeb2b220fd460 -F ext/fts5/test/fts5synonym.test 4b6387dcc12fc4e7216db06402375326ae4a28aa +F ext/fts5/test/fts5synonym.test e9ef0e6d59e6050020e0a805c37edde623bf289e F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89 F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841 F ext/fts5/test/fts5unicode2.test c1dd890ba32b7609adba78e420faa847abe43b59 @@ -1381,7 +1381,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f2e590700d8eb30da8ac3a92f6ca92a2bbe9c1fe -R 5fe3995e4cfcebadaed779600452a235 +P bdedd838bb3028c586bcc9f643852ce1364adb49 +R 780ceea4c7fd3021fab9ec34a27ddefc U dan -Z 535c1d64e133dff7ac9edd44481e77b4 +Z ba334b6166e1647cafa078f408fcb3d8 diff --git a/manifest.uuid b/manifest.uuid index 6008a07af2..f12f376ddc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bdedd838bb3028c586bcc9f643852ce1364adb49 \ No newline at end of file +399932a181437d84cd57234e76c4b65da1e4ac5f \ No newline at end of file From 9a9e3531f7ba6d6b1ccad3bdde65cd3483c0fe7b Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 3 Sep 2015 11:00:50 +0000 Subject: [PATCH 027/100] Fix the fts5 integrity-check so that it works with columnsize=0 tables. FossilOrigin-Name: 11b887b15eaee57ea2577c763e70494f1e251275 --- ext/fts5/fts5_storage.c | 19 ++++++++----- ext/fts5/test/fts5columnsize.test | 13 +++++++++ ext/fts5/test/fts5fault7.test | 45 +++++++++++++++++++++++++++++++ ext/fts5/test/fts5synonym.test | 20 ++++++++++++-- manifest | 17 ++++++------ manifest.uuid | 2 +- 6 files changed, 99 insertions(+), 17 deletions(-) create mode 100644 ext/fts5/test/fts5fault7.test diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c index 6ff81d428d..c91e7eeba3 100644 --- a/ext/fts5/fts5_storage.c +++ b/ext/fts5/fts5_storage.c @@ -891,7 +891,9 @@ int sqlite3Fts5StorageIntegrity(Fts5Storage *p){ int i; ctx.iRowid = sqlite3_column_int64(pScan, 0); ctx.szCol = 0; - rc = sqlite3Fts5StorageDocsize(p, ctx.iRowid, aColSize); + if( pConfig->bColumnsize ){ + rc = sqlite3Fts5StorageDocsize(p, ctx.iRowid, aColSize); + } for(i=0; rc==SQLITE_OK && inCol; i++){ if( pConfig->abUnindexed[i] ) continue; ctx.iCol = i; @@ -903,7 +905,9 @@ int sqlite3Fts5StorageIntegrity(Fts5Storage *p){ (void*)&ctx, fts5StorageIntegrityCallback ); - if( ctx.szCol!=aColSize[i] ) rc = FTS5_CORRUPT; + if( pConfig->bColumnsize && ctx.szCol!=aColSize[i] ){ + rc = FTS5_CORRUPT; + } aTotalSize[i] += ctx.szCol; } if( rc!=SQLITE_OK ) break; @@ -928,7 +932,7 @@ int sqlite3Fts5StorageIntegrity(Fts5Storage *p){ rc = fts5StorageCount(p, "content", &nRow); if( rc==SQLITE_OK && nRow!=p->nTotalRow ) rc = FTS5_CORRUPT; } - if( rc==SQLITE_OK ){ + if( rc==SQLITE_OK && pConfig->bColumnsize ){ i64 nRow; rc = fts5StorageCount(p, "docsize", &nRow); if( rc==SQLITE_OK && nRow!=p->nTotalRow ) rc = FTS5_CORRUPT; @@ -1012,9 +1016,12 @@ static int fts5StorageDecodeSizeArray( ** otherwise. */ int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol){ - int nCol = p->pConfig->nCol; - sqlite3_stmt *pLookup = 0; - int rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP_DOCSIZE, &pLookup, 0); + int nCol = p->pConfig->nCol; /* Number of user columns in table */ + sqlite3_stmt *pLookup = 0; /* Statement to query %_docsize */ + int rc; /* Return Code */ + + assert( p->pConfig->bColumnsize ); + rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP_DOCSIZE, &pLookup, 0); if( rc==SQLITE_OK ){ int bCorrupt = 1; sqlite3_bind_int64(pLookup, 1, iRowid); diff --git a/ext/fts5/test/fts5columnsize.test b/ext/fts5/test/fts5columnsize.test index ed0edd677e..dec9b58d3d 100644 --- a/ext/fts5/test/fts5columnsize.test +++ b/ext/fts5/test/fts5columnsize.test @@ -134,5 +134,18 @@ do_execsql_test 3.2.1 { 1 {-1 0 -1} 2 {-1 0 -1} } +#------------------------------------------------------------------------- +# Test the integrity-check +# +do_execsql_test 4.1.1 { + CREATE VIRTUAL TABLE t5 USING fts5(x, columnsize=0); + INSERT INTO t5 VALUES('1 2 3 4'); + INSERT INTO t5 VALUES('2 4 6 8'); +} + +breakpoint +do_execsql_test 4.1.2 { + INSERT INTO t5(t5) VALUES('integrity-check'); +} finish_test diff --git a/ext/fts5/test/fts5fault7.test b/ext/fts5/test/fts5fault7.test new file mode 100644 index 0000000000..381f9ea6ad --- /dev/null +++ b/ext/fts5/test/fts5fault7.test @@ -0,0 +1,45 @@ +# 2015 September 3 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# +# This file is focused on OOM errors. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +source $testdir/malloc_common.tcl +set testprefix fts5fault2 + +# If SQLITE_ENABLE_FTS3 is defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +#------------------------------------------------------------------------- +# Test fault-injection on a query that uses xColumnSize() on columnsize=0 +# table. +# +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x, columnsize=0); + INSERT INTO t1 VALUES('a b c d e f g'); + INSERT INTO t1 VALUES('a b c d'); + INSERT INTO t1 VALUES('a b c d e f g h i j'); +} + + +fts5_aux_test_functions db +do_faultsim_test 1 -faults oom* -body { + execsql { SELECT fts5_test_columnsize(t1) FROM t1 WHERE t1 MATCH 'b' } +} -test { + faultsim_test_result {0 {7 4 10}} {1 SQLITE_NOMEM} +} + +finish_test + diff --git a/ext/fts5/test/fts5synonym.test b/ext/fts5/test/fts5synonym.test index 81073dec8b..008d2fc543 100644 --- a/ext/fts5/test/fts5synonym.test +++ b/ext/fts5/test/fts5synonym.test @@ -433,12 +433,28 @@ proc tcl_tokenize {tflags text} { } } } -do_execsql_test 7.0 { - CREATE VIRTUAL TABLE t1 USING fts5(a, b, columnsize=0, tokenize=tcl); + +do_execsql_test 7.0.1 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, columnsize=1, tokenize=tcl); INSERT INTO t1 VALUES('0 2 3', '4 5 6 7'); INSERT INTO t1 VALUES('8 9', '0 0 0 0 0 0 0 0 0 0'); SELECT fts5_test_columnsize(t1) FROM t1 WHERE t1 MATCH '000 AND 00 AND 0'; } {{3 4} {2 10}} +do_execsql_test 7.0.2 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} + +do_execsql_test 7.1.1 { + CREATE VIRTUAL TABLE t2 USING fts5(a, b, columnsize=0, tokenize=tcl); + INSERT INTO t2 VALUES('0 2 3', '4 5 6 7'); + INSERT INTO t2 VALUES('8 9', '0 0 0 0 0 0 0 0 0 0'); + SELECT fts5_test_columnsize(t2) FROM t2 WHERE t2 MATCH '000 AND 00 AND 0'; +} {{3 4} {2 10}} + +do_execsql_test 7.1.2 { + INSERT INTO t2(t2) VALUES('integrity-check'); +} + finish_test diff --git a/manifest b/manifest index efbe08818c..5ea1f7c040 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\smemory\sleak\sin\sfts5_expr.c. -D 2015-09-03T10:27:02.747 +C Fix\sthe\sfts5\sintegrity-check\sso\sthat\sit\sworks\swith\scolumnsize=0\stables. +D 2015-09-03T11:00:50.518 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -114,7 +114,7 @@ F ext/fts5/fts5_expr.c 0c36c1db8eccdeb006e3c8d1499d05015f6e11a6 F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246 F ext/fts5/fts5_index.c 076c4995bf06a6d1559a6e31f9a86b90f2105374 F ext/fts5/fts5_main.c e9d0892424bb7f0a8b58613d4ff75cb650cf286e -F ext/fts5/fts5_storage.c c888defbb961d64c12299b3d1725a24a770b047e +F ext/fts5/fts5_storage.c 4b883f592ffdc6bcefba6fa03580228379a1333f F ext/fts5/fts5_tcl.c 6da58d6e8f42a93c4486b5ba9b187a7f995dee37 F ext/fts5/fts5_test_mi.c e96be827aa8f571031e65e481251dc1981d608bf F ext/fts5/fts5_tokenize.c f380f46f341af9c9a9908e1aade685ba1eaa157a @@ -141,7 +141,7 @@ F ext/fts5/test/fts5auto.test caa5bcf917db11944655a2a9bd38c67c520376ca F ext/fts5/test/fts5aux.test 8c687c948cc98e9a94be014df7d518acc1b3b74f F ext/fts5/test/fts5auxdata.test 141a7cbffcceb1bd2799b4b29c183ff8780d586e F ext/fts5/test/fts5bigpl.test 04ee0d7eebbebf17c31f5a0b5c5f9494eac3a0cb -F ext/fts5/test/fts5columnsize.test 97dc6bd66c91009d00407aa078dd5e9e8eb22f99 +F ext/fts5/test/fts5columnsize.test a8cfef21ffa1c264b9f670a7d94eeaccb5341c07 F ext/fts5/test/fts5config.test ad2ff42ddc856aed2d05bf89dc1c578c8a39ea3b F ext/fts5/test/fts5content.test 9a952c95518a14182dc3b59e3c8fa71cda82a4e1 F ext/fts5/test/fts5corrupt.test 928c9c91d40690d301f943a7ed0ffc19e0d0e7b6 @@ -157,6 +157,7 @@ F ext/fts5/test/fts5fault3.test d6e9577d4312e331a913c72931bf131704efc8f3 F ext/fts5/test/fts5fault4.test 762991d526ee67c2b374351a17248097ea38bee7 F ext/fts5/test/fts5fault5.test 54da9fd4c3434a1d4f6abdcb6469299d91cf5875 F ext/fts5/test/fts5fault6.test 97bce1a36b7a64e3203fea504ae8e5cfd5ada423 +F ext/fts5/test/fts5fault7.test f4a9b796f8b20c78ec7cf9f4e11144d15d7c3fd4 F ext/fts5/test/fts5full.test 6f6143af0c6700501d9fd597189dfab1555bb741 F ext/fts5/test/fts5hash.test 42eb066f667e9a389a63437cb7038c51974d4fc6 F ext/fts5/test/fts5integrity.test 29f41d2c7126c6122fbb5d54e556506456876145 @@ -172,7 +173,7 @@ F ext/fts5/test/fts5rank.test 11dcebba31d822f7e99685b4ea2c2ae3ec0b16f1 F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17 F ext/fts5/test/fts5rowid.test 6f9833b23b176dc4aa15b7fc02afeb2b220fd460 -F ext/fts5/test/fts5synonym.test e9ef0e6d59e6050020e0a805c37edde623bf289e +F ext/fts5/test/fts5synonym.test cf88c0a56d5ea9591e3939ef1f6e294f7f2d0671 F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89 F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841 F ext/fts5/test/fts5unicode2.test c1dd890ba32b7609adba78e420faa847abe43b59 @@ -1381,7 +1382,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P bdedd838bb3028c586bcc9f643852ce1364adb49 -R 780ceea4c7fd3021fab9ec34a27ddefc +P 399932a181437d84cd57234e76c4b65da1e4ac5f +R 71ce956d6d1c9e126c469db6cf4c4e29 U dan -Z ba334b6166e1647cafa078f408fcb3d8 +Z 54ee3fa6162283338bf35bd792c52ef7 diff --git a/manifest.uuid b/manifest.uuid index f12f376ddc..5fc6e92798 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -399932a181437d84cd57234e76c4b65da1e4ac5f \ No newline at end of file +11b887b15eaee57ea2577c763e70494f1e251275 \ No newline at end of file From 5d144875bc3f6390f67575e0710a08cfe1124ae1 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 3 Sep 2015 11:17:52 +0000 Subject: [PATCH 028/100] Remove dead code from fts5_index.c. FossilOrigin-Name: 8a0a9b01e74072ee52fe393311ad591208fbbf7c --- ext/fts5/fts5_index.c | 295 +++++++----------------------------------- manifest | 12 +- manifest.uuid | 2 +- 3 files changed, 51 insertions(+), 258 deletions(-) diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index a7ab7c4c05..06a5d56bbc 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -293,7 +293,6 @@ typedef struct Fts5Data Fts5Data; typedef struct Fts5DlidxIter Fts5DlidxIter; typedef struct Fts5DlidxLvl Fts5DlidxLvl; typedef struct Fts5DlidxWriter Fts5DlidxWriter; -typedef struct Fts5NodeIter Fts5NodeIter; typedef struct Fts5PageWriter Fts5PageWriter; typedef struct Fts5SegIter Fts5SegIter; typedef struct Fts5DoclistIter Fts5DoclistIter; @@ -526,24 +525,6 @@ struct Fts5IndexIter { }; -/* -** Object for iterating through the conents of a single internal node in -** memory. -*/ -struct Fts5NodeIter { - /* Internal. Set and managed by fts5NodeIterXXX() functions. Except, - ** the EOF test for the iterator is (Fts5NodeIter.aData==0). */ - const u8 *aData; - int nData; - int iOff; - - /* Output variables */ - Fts5Buffer term; - int nEmpty; - int iChild; - int bDlidx; -}; - /* ** An instance of the following type is used to iterate through the contents ** of a doclist-index record. @@ -617,6 +598,7 @@ static void *fts5IdxMalloc(Fts5Index *p, int nByte){ ** ** res = *pLeft - *pRight */ +#ifdef SQLITE_DEBUG static int fts5BufferCompareBlob( Fts5Buffer *pLeft, /* Left hand side of comparison */ const u8 *pRight, int nRight /* Right hand side of comparison */ @@ -625,7 +607,7 @@ static int fts5BufferCompareBlob( int res = memcmp(pLeft->p, pRight, nCmp); return (res==0 ? (pLeft->n - nRight) : res); } - +#endif /* ** Compare the contents of the two buffers using memcmp(). If one buffer @@ -1228,62 +1210,6 @@ static void fts5StructurePromote( } -/* -** If the pIter->iOff offset currently points to an entry indicating one -** or more term-less nodes, advance past it and set pIter->nEmpty to -** the number of empty child nodes. -*/ -static void fts5NodeIterGobbleNEmpty(Fts5NodeIter *pIter){ - if( pIter->iOffnData && 0==(pIter->aData[pIter->iOff] & 0xfe) ){ - pIter->bDlidx = pIter->aData[pIter->iOff] & 0x01; - pIter->iOff++; - pIter->iOff += fts5GetVarint32(&pIter->aData[pIter->iOff], pIter->nEmpty); - }else{ - pIter->nEmpty = 0; - pIter->bDlidx = 0; - } -} - -/* -** Advance to the next entry within the node. -*/ -static void fts5NodeIterNext(int *pRc, Fts5NodeIter *pIter){ - if( pIter->iOff>=pIter->nData ){ - pIter->aData = 0; - pIter->iChild += pIter->nEmpty; - }else{ - int nPre, nNew; - pIter->iOff += fts5GetVarint32(&pIter->aData[pIter->iOff], nPre); - pIter->iOff += fts5GetVarint32(&pIter->aData[pIter->iOff], nNew); - pIter->term.n = nPre-2; - fts5BufferAppendBlob(pRc, &pIter->term, nNew, pIter->aData+pIter->iOff); - pIter->iOff += nNew; - pIter->iChild += (1 + pIter->nEmpty); - fts5NodeIterGobbleNEmpty(pIter); - if( *pRc ) pIter->aData = 0; - } -} - - -/* -** Initialize the iterator object pIter to iterate through the internal -** segment node in pData. -*/ -static void fts5NodeIterInit(const u8 *aData, int nData, Fts5NodeIter *pIter){ - memset(pIter, 0, sizeof(*pIter)); - pIter->aData = aData; - pIter->nData = nData; - pIter->iOff = fts5GetVarint32(aData, pIter->iChild); - fts5NodeIterGobbleNEmpty(pIter); -} - -/* -** Free any memory allocated by the iterator object. -*/ -static void fts5NodeIterFree(Fts5NodeIter *pIter){ - fts5BufferFree(&pIter->term); -} - /* ** Advance the iterator passed as the only argument. If the end of the ** doclist-index page is reached, return non-zero. @@ -2041,119 +1967,6 @@ static void fts5SegIterLoadDlidx(Fts5Index *p, Fts5SegIter *pIter){ pIter->pDlidx = fts5DlidxIterInit(p, bRev, iSeg, pIter->iTermLeafPgno); } -#ifdef SQLITE_DEBUG -static void fts5AssertNodeSeekOk( - Fts5Buffer *pNode, - const u8 *pTerm, int nTerm, /* Term to search for */ - int iExpectPg, - int bExpectDlidx -){ - int bDlidx; - int iPg; - int rc = SQLITE_OK; - Fts5NodeIter node; - - fts5NodeIterInit(pNode->p, pNode->n, &node); - assert( node.term.n==0 ); - iPg = node.iChild; - bDlidx = node.bDlidx; - for(fts5NodeIterNext(&rc, &node); - node.aData && fts5BufferCompareBlob(&node.term, pTerm, nTerm)<=0; - fts5NodeIterNext(&rc, &node) - ){ - iPg = node.iChild; - bDlidx = node.bDlidx; - } - fts5NodeIterFree(&node); - - assert( rc!=SQLITE_OK || iPg==iExpectPg ); - assert( rc!=SQLITE_OK || bDlidx==bExpectDlidx ); -} -#else -#define fts5AssertNodeSeekOk(v,w,x,y,z) -#endif - -/* -** Argument pNode is an internal b-tree node. This function searches -** within the node for the largest term that is smaller than or equal -** to (pTerm/nTerm). -** -** It returns the associated page number. Or, if (pTerm/nTerm) is smaller -** than all terms within the node, the leftmost child page number. -** -** Before returning, (*pbDlidx) is set to true if the last term on the -** returned child page number has a doclist-index. Or left as is otherwise. -*/ -static int fts5NodeSeek( - Fts5Buffer *pNode, /* Node to search */ - const u8 *pTerm, int nTerm, /* Term to search for */ - int *pbDlidx /* OUT: True if dlidx flag is set */ -){ - int iPg; - u8 *pPtr = pNode->p; - u8 *pEnd = &pPtr[pNode->n]; - int nMatch = 0; /* Number of bytes of pTerm already matched */ - - assert( *pbDlidx==0 ); - - pPtr += fts5GetVarint32(pPtr, iPg); - while( pPtr=pEnd ) break; - } - - /* Read the next "term" pointer. Set nKeep to the number of bytes to - ** keep from the previous term, and nNew to the number of bytes of - ** new data that will be appended to it. */ - nKeep = (int)*pPtr++; - nNew = (int)*pPtr++; - if( (nKeep | nNew) & 0x0080 ){ - pPtr -= 2; - pPtr += fts5GetVarint32(pPtr, nKeep); - pPtr += fts5GetVarint32(pPtr, nNew); - } - nKeep -= 2; - - /* Compare (pTerm/nTerm) to the current term on the node (the one described - ** by nKeep/nNew). If the node term is larger, break out of the while() - ** loop. - ** - ** Otherwise, if (pTerm/nTerm) is larger or the two terms are equal, - ** leave variable nMatch set to the size of the largest prefix common to - ** both terms in bytes. */ - if( nKeep==nMatch ){ - int nTst = MIN(nNew, nTerm-nMatch); - int i; - for(i=0; i pTerm[nMatch]) ) break; - }else if( nKeep=4 ){ - iRowidOff = fts5GetU16(&a[0]); - iTermOff = fts5GetU16(&a[2]); - }else{ - sqlite3Fts5BufferSet(&rc, &s, 8, (const u8*)"corrupt"); - goto decode_out; - } - - if( iRowidOff ){ - iOff = iRowidOff; - }else if( iTermOff ){ - iOff = iTermOff; - }else{ - iOff = n; - } - fts5DecodePoslist(&rc, &s, &a[4], iOff-4); - - assert( iRowidOff==0 || iOff==iRowidOff ); - if( iRowidOff ){ - iOff += fts5DecodeDoclist(&rc, &s, &a[iOff], n-iOff); - } - - assert( iTermOff==0 || iOff==iTermOff ); - while( iOff=4 ){ + iRowidOff = fts5GetU16(&a[0]); + iTermOff = fts5GetU16(&a[2]); }else{ - Fts5NodeIter ss; - for(fts5NodeIterInit(a, n, &ss); ss.aData; fts5NodeIterNext(&rc, &ss)){ - if( ss.term.n==0 ){ - sqlite3Fts5BufferAppendPrintf(&rc, &s, " left=%d", ss.iChild); - }else{ - sqlite3Fts5BufferAppendPrintf(&rc,&s, " \"%.*s\"", - ss.term.n, ss.term.p - ); - } - if( ss.nEmpty ){ - sqlite3Fts5BufferAppendPrintf(&rc, &s, " empty=%d%s", ss.nEmpty, - ss.bDlidx ? "*" : "" - ); - } - } - fts5NodeIterFree(&ss); + sqlite3Fts5BufferSet(&rc, &s, 8, (const u8*)"corrupt"); + goto decode_out; } + + if( iRowidOff ){ + iOff = iRowidOff; + }else if( iTermOff ){ + iOff = iTermOff; + }else{ + iOff = n; + } + fts5DecodePoslist(&rc, &s, &a[4], iOff-4); + + assert( iRowidOff==0 || iOff==iRowidOff ); + if( iRowidOff ){ + iOff += fts5DecodeDoclist(&rc, &s, &a[iOff], n-iOff); + } + + assert( iTermOff==0 || iOff==iTermOff ); + while( iOff Date: Thu, 3 Sep 2015 14:22:27 +0000 Subject: [PATCH 029/100] Remove some more code from fts5_index.c by consolidating similar functions. FossilOrigin-Name: 59ae30b97b40faa363c55aa2664dead9eaeeddc0 --- ext/fts5/fts5Int.h | 4 +- ext/fts5/fts5_index.c | 128 +++++++++++++++------------------------- ext/fts5/fts5_storage.c | 17 +----- manifest | 16 ++--- manifest.uuid | 2 +- 5 files changed, 58 insertions(+), 109 deletions(-) diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index 1da017fb01..837ecb1ccd 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -384,9 +384,9 @@ int sqlite3Fts5IndexErrcode(Fts5Index*); void sqlite3Fts5IndexReset(Fts5Index*); /* -** Get or set the "averages" record. +** Get or set the "averages" values. */ -int sqlite3Fts5IndexGetAverages(Fts5Index *p, Fts5Buffer *pBuf); +int sqlite3Fts5IndexGetAverages(Fts5Index *p, i64 *pnRow, i64 *anSize); int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8*, int); /* diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 06a5d56bbc..7bb744773c 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -554,23 +554,6 @@ struct Fts5DlidxIter { Fts5DlidxLvl aLvl[1]; }; - - -/* -** The first argument passed to this macro is a pointer to an Fts5Buffer -** object. -*/ -#define fts5BufferSize(pBuf,n) { \ - if( pBuf->nSpacep, n); \ - if( pNew==0 ){ \ - sqlite3_free(pBuf->p); \ - } \ - pBuf->nSpace = n; \ - pBuf->p = pNew; \ - } \ -} - static void fts5PutU16(u8 *aOut, u16 iVal){ aOut[0] = (iVal>>8); aOut[1] = (iVal&0xFF); @@ -647,11 +630,14 @@ static void fts5CloseReader(Fts5Index *p){ } } -static Fts5Data *fts5DataReadOrBuffer( - Fts5Index *p, - Fts5Buffer *pBuf, - i64 iRowid -){ + +/* +** Retrieve a record from the %_data table. +** +** If an error occurs, NULL is returned and an error left in the +** Fts5Index object. +*/ +static Fts5Data *fts5DataRead(Fts5Index *p, i64 iRowid){ Fts5Data *pRet = 0; if( p->rc==SQLITE_OK ){ int rc = SQLITE_OK; @@ -671,8 +657,8 @@ static Fts5Data *fts5DataReadOrBuffer( if( rc==SQLITE_ABORT ) rc = SQLITE_OK; } - /* If the blob handle is not yet open, open and seek it. Otherwise, use - ** the blob_reopen() API to reseek the existing blob handle. */ + /* If the blob handle is not open at this point, open it and seek + ** to the requested entry. */ if( p->pReader==0 && rc==SQLITE_OK ){ Fts5Config *pConfig = p->pConfig; rc = sqlite3_blob_open(pConfig->db, @@ -690,22 +676,13 @@ static Fts5Data *fts5DataReadOrBuffer( if( rc==SQLITE_OK ){ u8 *aOut = 0; /* Read blob data into this buffer */ int nByte = sqlite3_blob_bytes(p->pReader); - if( pBuf ){ - fts5BufferSize(pBuf, MAX(nByte, p->pConfig->pgsz) + 20); - pBuf->n = nByte; - aOut = pBuf->p; - if( aOut==0 ){ - rc = SQLITE_NOMEM; - } + int nAlloc = sizeof(Fts5Data) + nByte + FTS5_DATA_PADDING; + pRet = (Fts5Data*)sqlite3_malloc(nAlloc); + if( pRet ){ + pRet->n = nByte; + aOut = pRet->p = (u8*)&pRet[1]; }else{ - int nSpace = nByte + FTS5_DATA_PADDING; - pRet = (Fts5Data*)sqlite3_malloc(nSpace+sizeof(Fts5Data)); - if( pRet ){ - pRet->n = nByte; - aOut = pRet->p = (u8*)&pRet[1]; - }else{ - rc = SQLITE_NOMEM; - } + rc = SQLITE_NOMEM; } if( rc==SQLITE_OK ){ @@ -720,33 +697,10 @@ static Fts5Data *fts5DataReadOrBuffer( p->nRead++; } - return pRet; -} - -/* -** Retrieve a record from the %_data table. -** -** If an error occurs, NULL is returned and an error left in the -** Fts5Index object. -*/ -static Fts5Data *fts5DataRead(Fts5Index *p, i64 iRowid){ - Fts5Data *pRet = fts5DataReadOrBuffer(p, 0, iRowid); assert( (pRet==0)==(p->rc!=SQLITE_OK) ); return pRet; } -/* -** Read a record from the %_data table into the buffer supplied as the -** second argument. -** -** If an error occurs, an error is left in the Fts5Index object. If an -** error has already occurred when this function is called, it is a -** no-op. -*/ -static void fts5DataBuffer(Fts5Index *p, Fts5Buffer *pBuf, i64 iRowid){ - (void)fts5DataReadOrBuffer(p, pBuf, iRowid); -} - /* ** Release a reference to data record returned by an earlier call to ** fts5DataRead(). @@ -1015,19 +969,18 @@ static Fts5Structure *fts5StructureRead(Fts5Index *p){ Fts5Config *pConfig = p->pConfig; Fts5Structure *pRet = 0; /* Object to return */ int iCookie; /* Configuration cookie */ + Fts5Data *pData; Fts5Buffer buf = {0, 0, 0}; - fts5DataBuffer(p, &buf, FTS5_STRUCTURE_ROWID); - if( buf.p==0 ) return 0; - assert( buf.nSpace>=(buf.n + FTS5_DATA_ZERO_PADDING) ); - memset(&buf.p[buf.n], 0, FTS5_DATA_ZERO_PADDING); - p->rc = fts5StructureDecode(buf.p, buf.n, &iCookie, &pRet); - + pData = fts5DataRead(p, FTS5_STRUCTURE_ROWID); + if( p->rc ) return 0; + memset(&pData->p[pData->n], 0, FTS5_DATA_PADDING); + p->rc = fts5StructureDecode(pData->p, pData->n, &iCookie, &pRet); if( p->rc==SQLITE_OK && pConfig->iCookie!=iCookie ){ p->rc = sqlite3Fts5ConfigLoad(pConfig, iCookie); } - fts5BufferFree(&buf); + fts5DataRelease(pData); if( p->rc!=SQLITE_OK ){ fts5StructureRelease(pRet); pRet = 0; @@ -2490,13 +2443,13 @@ static void fts5SegIterNextFrom( } } - while( p->rc==SQLITE_OK ){ + do{ if( bMove ) fts5SegIterNext(p, pIter, 0); if( pIter->pLeaf==0 ) break; if( bRev==0 && pIter->iRowid>=iMatch ) break; if( bRev!=0 && pIter->iRowid<=iMatch ) break; bMove = 1; - } + }while( p->rc==SQLITE_OK ); } @@ -4272,13 +4225,9 @@ int sqlite3Fts5IndexRollback(Fts5Index *p){ */ int sqlite3Fts5IndexReinit(Fts5Index *p){ Fts5Structure s; - - assert( p->rc==SQLITE_OK ); - p->rc = sqlite3Fts5IndexSetAverages(p, (const u8*)"", 0); - memset(&s, 0, sizeof(Fts5Structure)); + fts5DataWrite(p, FTS5_AVERAGES_ROWID, (const u8*)"", 0); fts5StructureWrite(p, &s); - return fts5IndexReturn(p); } @@ -4600,13 +4549,28 @@ void sqlite3Fts5IterClose(Fts5IndexIter *pIter){ } /* -** Read the "averages" record into the buffer supplied as the second -** argument. Return SQLITE_OK if successful, or an SQLite error code -** if an error occurs. +** Read and decode the "averages" record from the database. +** +** Parameter anSize must point to an array of size nCol, where nCol is +** the number of user defined columns in the FTS table. */ -int sqlite3Fts5IndexGetAverages(Fts5Index *p, Fts5Buffer *pBuf){ - assert( p->rc==SQLITE_OK ); - fts5DataReadOrBuffer(p, pBuf, FTS5_AVERAGES_ROWID); +int sqlite3Fts5IndexGetAverages(Fts5Index *p, i64 *pnRow, i64 *anSize){ + int nCol = p->pConfig->nCol; + Fts5Data *pData; + + *pnRow = 0; + memset(anSize, 0, sizeof(i64) * nCol); + pData = fts5DataRead(p, FTS5_AVERAGES_ROWID); + if( p->rc==SQLITE_OK && pData->n ){ + int i = 0; + int iCol; + i += fts5GetVarint(&pData->p[i], (u64*)pnRow); + for(iCol=0; in && iColp[i], (u64*)&anSize[iCol]); + } + } + + fts5DataRelease(pData); return fts5IndexReturn(p); } diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c index c91e7eeba3..65c6d48a36 100644 --- a/ext/fts5/fts5_storage.c +++ b/ext/fts5/fts5_storage.c @@ -455,22 +455,7 @@ static int fts5StorageInsertDocsize( static int fts5StorageLoadTotals(Fts5Storage *p, int bCache){ int rc = SQLITE_OK; if( p->bTotalsValid==0 ){ - int nCol = p->pConfig->nCol; - Fts5Buffer buf; - memset(&buf, 0, sizeof(buf)); - - memset(p->aTotalSize, 0, sizeof(i64) * nCol); - p->nTotalRow = 0; - rc = sqlite3Fts5IndexGetAverages(p->pIndex, &buf); - if( rc==SQLITE_OK && buf.n ){ - int i = 0; - int iCol; - i += fts5GetVarint(&buf.p[i], (u64*)&p->nTotalRow); - for(iCol=0; iaTotalSize[iCol]); - } - } - sqlite3_free(buf.p); + rc = sqlite3Fts5IndexGetAverages(p->pIndex, &p->nTotalRow, p->aTotalSize); p->bTotalsValid = bCache; } return rc; diff --git a/manifest b/manifest index 73cbe4ee86..0e7dfca7b2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sdead\scode\sfrom\sfts5_index.c. -D 2015-09-03T11:17:52.904 +C Remove\ssome\smore\scode\sfrom\sfts5_index.c\sby\sconsolidating\ssimilar\sfunctions. +D 2015-09-03T14:22:27.810 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -106,15 +106,15 @@ F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95 F ext/fts5/extract_api_docs.tcl 06583c935f89075ea0b32f85efa5dd7619fcbd03 F ext/fts5/fts5.h 0784692f406588e6c90e13a78e1f36e7e3236e42 -F ext/fts5/fts5Int.h c6f035091fc9fa12a92c7066bf0c266c08cb508b +F ext/fts5/fts5Int.h f65d41f66accad0a289d6bd66b13c07d2932f9be F ext/fts5/fts5_aux.c 7a307760a9c57c750d043188ec0bad59f5b5ec7e F ext/fts5/fts5_buffer.c 80f9ba4431848cb857e3d2158f5280093dcd8015 F ext/fts5/fts5_config.c 80b61fd2c6844b64a3e72a64572d50a812da9384 F ext/fts5/fts5_expr.c 0c36c1db8eccdeb006e3c8d1499d05015f6e11a6 F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246 -F ext/fts5/fts5_index.c df98f39c0f6e9d06e144dde5a0751ab14ee9d0fd +F ext/fts5/fts5_index.c febb68173333ae3248eb15928a18b21112d45135 F ext/fts5/fts5_main.c e9d0892424bb7f0a8b58613d4ff75cb650cf286e -F ext/fts5/fts5_storage.c 4b883f592ffdc6bcefba6fa03580228379a1333f +F ext/fts5/fts5_storage.c 120f7b143688b5b7710dacbd48cff211609b8059 F ext/fts5/fts5_tcl.c 6da58d6e8f42a93c4486b5ba9b187a7f995dee37 F ext/fts5/fts5_test_mi.c e96be827aa8f571031e65e481251dc1981d608bf F ext/fts5/fts5_tokenize.c f380f46f341af9c9a9908e1aade685ba1eaa157a @@ -1382,7 +1382,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 11b887b15eaee57ea2577c763e70494f1e251275 -R 97c9a99cb5cabc51c42bc2a29210372f +P 8a0a9b01e74072ee52fe393311ad591208fbbf7c +R e26c002d3fbf39ced0ec480a25b8c1b8 U dan -Z 9499a3a8b1a7472d387dbda943ee05af +Z cd2601a80f5a8baa4e1506c2cddafc82 diff --git a/manifest.uuid b/manifest.uuid index f57a7a34cb..03cc15587f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8a0a9b01e74072ee52fe393311ad591208fbbf7c \ No newline at end of file +59ae30b97b40faa363c55aa2664dead9eaeeddc0 \ No newline at end of file From ea623630e2f057f531f895c4501264a93b02f472 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 3 Sep 2015 15:37:26 +0000 Subject: [PATCH 030/100] Add tests to improve coverage of fts5_varint.c. FossilOrigin-Name: 89f24f31a8f7d7cb0a66ee53523881f566dcb035 --- ext/fts5/test/fts5aa.test | 20 ++++++++++++++++++++ manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/ext/fts5/test/fts5aa.test b/ext/fts5/test/fts5aa.test index 832a972cfc..5f3a9d9e1d 100644 --- a/ext/fts5/test/fts5aa.test +++ b/ext/fts5/test/fts5aa.test @@ -516,6 +516,26 @@ do_execsql_test 19.0 { INSERT INTO t1 VALUES('w x 1'); SELECT rowid FROM t1 WHERE t1 MATCH 'x'; } {1 2} + +#-------------------------------------------------------------------- +# Test that 6 and 7 byte varints can be read. +# +reset_db +do_execsql_test 20.0 { + CREATE VIRTUAL TABLE temp.tmp USING fts5(x); +} +set ::ids [list \ + 0 [expr 1<<36] [expr 2<<36] [expr 1<<43] [expr 2<<43] +] +do_test 20.1 { + foreach id $::ids { + execsql { INSERT INTO tmp(rowid, x) VALUES($id, 'x y z') } + } + execsql { SELECT rowid FROM tmp WHERE tmp MATCH 'y' } +} $::ids + + + finish_test diff --git a/manifest b/manifest index 0e7dfca7b2..c46ab77428 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\ssome\smore\scode\sfrom\sfts5_index.c\sby\sconsolidating\ssimilar\sfunctions. -D 2015-09-03T14:22:27.810 +C Add\stests\sto\simprove\scoverage\sof\sfts5_varint.c. +D 2015-09-03T15:37:26.095 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -124,7 +124,7 @@ F ext/fts5/fts5_vocab.c 4622e0b7d84a488a1585aaa56eb214ee67a988bc F ext/fts5/fts5parse.y 833db1101b78c0c47686ab1b84918e38c36e9452 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba F ext/fts5/test/fts5_common.tcl b6e6a40ef5d069c8e86ca4fbad491e1195485dbc -F ext/fts5/test/fts5aa.test 86def30ff5e76eb6db66f0833c4aefe0b765d590 +F ext/fts5/test/fts5aa.test be961d10b0eff5e973c37b3b7a32a18cda87a77c F ext/fts5/test/fts5ab.test 6fe3a56731d15978afbb74ae51b355fc9310f2ad F ext/fts5/test/fts5ac.test 9737992d08c56bfd4803e933744d2d764e23795c F ext/fts5/test/fts5ad.test b2edee8b7de0c21d2c88f8a18c195034aad6952d @@ -1382,7 +1382,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 8a0a9b01e74072ee52fe393311ad591208fbbf7c -R e26c002d3fbf39ced0ec480a25b8c1b8 +P 59ae30b97b40faa363c55aa2664dead9eaeeddc0 +R e54371f57da08ab0c80326ea4baedd7e U dan -Z cd2601a80f5a8baa4e1506c2cddafc82 +Z 416751bdad2bf139cde3272ad789aa1a diff --git a/manifest.uuid b/manifest.uuid index 03cc15587f..3c272efde7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -59ae30b97b40faa363c55aa2664dead9eaeeddc0 \ No newline at end of file +89f24f31a8f7d7cb0a66ee53523881f566dcb035 \ No newline at end of file From c98a4cc8b6a2fd2a54d6281279f46ccb14ec2e81 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 3 Sep 2015 17:54:32 +0000 Subject: [PATCH 031/100] Change the Pager.hasBeenUsed flag into Pager.hasHeldSharedLock in order to take it off the critical path in sqlite3PagerAcquire(). FossilOrigin-Name: b79096be7cb02aae2f303db33a8bf19e69204374 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pager.c | 11 ++++++----- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 5e842820c7..9427db7679 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\ssqlite3VdbeLoadString()\sand\ssqlite3VdbeMultiLoad()\sroutines\sto\shelp\nwith\scode\sgeneration,\sespecially\son\sPRAGMAs.\s\sRename\ssqlite3VdbeAddGoto()\nto\sjust\ssqlite3VdbeGoto(). -D 2015-09-03T13:46:12.406 +C Change\sthe\sPager.hasBeenUsed\sflag\sinto\sPager.hasHeldSharedLock\sin\sorder\sto\ntake\sit\soff\sthe\scritical\spath\sin\ssqlite3PagerAcquire(). +D 2015-09-03T17:54:32.014 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -324,7 +324,7 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c 76f493ed71c4154338049dee1bf6e47f69c74a55 F src/os_win.c 40b3af7a47eb1107d0d69e592bec345a3b7b798a F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca -F src/pager.c aa916ca28606ccf4b6877dfc2b643ccbca86589f +F src/pager.c 2a9d77ac483a684c4187ec9ca786b551d3adfda5 F src/pager.h 6d435f563b3f7fcae4b84433b76a6ac2730036e2 F src/parse.y f599aa5e871a493330d567ced93de696f61f48f7 F src/pcache.c cde06aa50962595e412d497e22fd2e07878ba1f0 @@ -1380,7 +1380,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P d7f4bdf594e93c848f46901637861c8eed4c34df -R 87d27a58ce6d4a6cd03f169f2370f2fe +P 847387ec8e6fef283899578fb232b2c23b00ee5b +R 530c1d1d05feba269ca8969115207173 U drh -Z bdbaccc5e83ea81c60c1b2c883af5b3e +Z e28985417ac4a7094ed6187f7e395585 diff --git a/manifest.uuid b/manifest.uuid index 3428834616..eaef64ee94 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -847387ec8e6fef283899578fb232b2c23b00ee5b \ No newline at end of file +b79096be7cb02aae2f303db33a8bf19e69204374 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 060edb8d1d..f4e27f36ca 100644 --- a/src/pager.c +++ b/src/pager.c @@ -647,7 +647,7 @@ struct Pager { u8 doNotSpill; /* Do not spill the cache when non-zero */ u8 subjInMemory; /* True to use in-memory sub-journals */ u8 bUseFetch; /* True to use xFetch() */ - u8 hasBeenUsed; /* True if any content previously read */ + u8 hasHeldSharedLock; /* True if a shared lock has ever been held */ Pgno dbSize; /* Number of pages in the database */ Pgno dbOrigSize; /* dbSize before the current transaction */ Pgno dbFileSize; /* Number of pages in the database file */ @@ -5097,10 +5097,10 @@ int sqlite3PagerSharedLock(Pager *pPager){ ); } - if( !pPager->tempFile && pPager->hasBeenUsed ){ + if( !pPager->tempFile && pPager->hasHeldSharedLock ){ /* The shared-lock has just been acquired then check to ** see if the database has been modified. If the database has changed, - ** flush the cache. The pPager->hasBeenUsed flag prevents this from + ** flush the cache. The hasHeldSharedLock flag prevents this from ** occurring on the very first access to a file, in order to save a ** single unnecessary sqlite3OsRead() call at the start-up. ** @@ -5170,6 +5170,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ assert( pPager->eState==PAGER_OPEN ); }else{ pPager->eState = PAGER_READER; + pPager->hasHeldSharedLock = 1; } return rc; } @@ -5267,7 +5268,7 @@ int sqlite3PagerAcquire( if( pgno==0 ){ return SQLITE_CORRUPT_BKPT; } - pPager->hasBeenUsed = 1; + assert( pPager->hasHeldSharedLock==1 ); /* If the pager is in the error state, return an error immediately. ** Otherwise, request the page from the PCache layer. */ @@ -5422,7 +5423,7 @@ DbPage *sqlite3PagerLookup(Pager *pPager, Pgno pgno){ assert( pgno!=0 ); assert( pPager->pPCache!=0 ); pPage = sqlite3PcacheFetch(pPager->pPCache, pgno, 0); - assert( pPage==0 || pPager->hasBeenUsed ); + assert( pPage==0 || pPager->hasHeldSharedLock ); if( pPage==0 ) return 0; return sqlite3PcacheFetchFinish(pPager->pPCache, pgno, pPage); } From ea8b93ee2041a1f4e22132cb937eff137127f4b5 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 3 Sep 2015 18:05:09 +0000 Subject: [PATCH 032/100] Add documentation for fts5 synonym support. FossilOrigin-Name: 58aa1f435959852df74f1bca8e0bdbc4f47c256a --- ext/fts5/extract_api_docs.tcl | 8 +- ext/fts5/fts5.h | 150 ++++++++++++++++++++++++++++++++-- manifest | 14 ++-- manifest.uuid | 2 +- 4 files changed, 157 insertions(+), 17 deletions(-) diff --git a/ext/fts5/extract_api_docs.tcl b/ext/fts5/extract_api_docs.tcl index afb2699be5..2320d70b7d 100644 --- a/ext/fts5/extract_api_docs.tcl +++ b/ext/fts5/extract_api_docs.tcl @@ -108,13 +108,15 @@ proc get_tokenizer_docs {data} { append res "
$line

\n" continue } + if {[regexp {SYNONYM SUPPORT} $line]} { + set line "

Synonym Support

" + } if {[string trim $line] == ""} { append res "

\n" } else { append res "$line\n" } } - append res "\n" set res } @@ -208,6 +210,10 @@ proc main {data} { fts5_tokenizer { output [get_fts5_struct $data "typedef struct Fts5Tokenizer" "^\};"] + output [get_fts5_struct $data \ + "Flags that may be passed as the third argument to xTokenize()" \ + "#define FTS5_TOKEN_COLOCATED" + ] } fts5_extension { diff --git a/ext/fts5/fts5.h b/ext/fts5/fts5.h index c9eb91d4cc..6872918e94 100644 --- a/ext/fts5/fts5.h +++ b/ext/fts5/fts5.h @@ -278,18 +278,46 @@ struct Fts5ExtensionApi { ** ** xTokenize: ** This function is expected to tokenize the nText byte string indicated -** by argument pText. pText may not be nul-terminated. The first argument -** passed to this function is a pointer to an Fts5Tokenizer object returned -** by an earlier call to xCreate(). +** by argument pText. pText may or may not be nul-terminated. The first +** argument passed to this function is a pointer to an Fts5Tokenizer object +** returned by an earlier call to xCreate(). +** +** The second argument indicates the reason that FTS5 is requesting +** tokenization of the supplied text. This is always one of the following +** four values: +** +**

  • FTS5_TOKENIZE_DOCUMENT - A document is being inserted into +** or removed from the FTS table. The tokenizer is being invoked to +** determine the set of tokens to add to (or delete from) the +** FTS index. +** +**
  • FTS5_TOKENIZE_QUERY - A MATCH query is being executed +** against the FTS index. The tokenizer is being called to tokenize +** a bareword or quoted string specified as part of the query. +** +**
  • (FTS5_TOKENIZE_QUERY | FTS5_TOKENIZE_PREFIX) - Same as +** FTS5_TOKENIZE_QUERY, except that the bareword or quoted string is +** followed by a "*" character, indicating that the last token +** returned by the tokenizer will be treated as a token prefix. +** +**
  • FTS5_TOKENIZE_AUX - The tokenizer is being invoked to +** satisfy an fts5_api.xTokenize() request made by an auxiliary +** function. Or an fts5_api.xColumnSize() request made by the same +** on a columnsize=0 database. +**
** ** For each token in the input string, the supplied callback xToken() must ** be invoked. The first argument to it should be a copy of the pointer -** passed as the second argument to xTokenize(). The next two arguments -** are a pointer to a buffer containing the token text, and the size of -** the token in bytes. The 4th and 5th arguments are the byte offsets of -** the first byte of and first byte immediately following the text from +** passed as the second argument to xTokenize(). The third and fourth +** arguments are a pointer to a buffer containing the token text, and the +** size of the token in bytes. The 4th and 5th arguments are the byte offsets +** of the first byte of and first byte immediately following the text from ** which the token is derived within the input. ** +** The second argument passed to the xToken() callback ("tflags") should +** normally be set to 0. The exception is if the tokenizer supports +** synonyms. In this case see the discussion below for details. +** ** FTS5 assumes the xToken() callback is invoked for each token in the ** order that they occur within the input text. ** @@ -301,6 +329,112 @@ struct Fts5ExtensionApi { ** may abandon the tokenization and return any error code other than ** SQLITE_OK or SQLITE_DONE. ** +** SYNONYM SUPPORT +** +** Custom tokenizers may also support synonyms. Consider a case in which a +** user wishes to query for a phrase such as "first place". Using the +** built-in tokenizers, the FTS5 query 'first + place' will match instances +** of "first place" within the document set, but not alternative forms +** such as "1st place". In some applications, it would be better to match +** all instances of "first place" or "1st place" regardless of which form +** the user specified in the MATCH query text. +** +** There are several ways to approach this in FTS5: +** +**
  1. By mapping all synonyms to a single token. In this case, the +** In the above example, this means that the tokenizer returns the +** same token for inputs "first" and "1st". Say that token is in +** fact "first", so that when the user inserts the document "I won +** 1st place" entries are added to the index for tokens "i", "won", +** "first" and "place". If the user then queries for '1st + place', +** the tokenizer substitutes "first" for "1st" and the query works +** as expected. +** +**
  2. By adding multiple synonyms for a single term to the FTS index. +** In this case, when tokenizing query text, the tokenizer may +** provide multiple synonyms for a single term within the document. +** FTS5 then queries the index for each synonym individually. For +** example, faced with the query: +** +** +** ... MATCH 'first place' +** +** the tokenizer offers both "1st" and "first" as synonyms for the +** first token in the MATCH query and FTS5 effectively runs a query +** similar to: +** +** +** ... MATCH '(first OR 1st) place' +** +** except that, for the purposes of auxiliary functions, the query +** still appears to contain just two phrases - "(first OR 1st)" +** being treated as a single phrase. +** +**
  3. By adding multiple synonyms for a single term to the FTS index. +** Using this method, when tokenizing document text, the tokenizer +** provides multiple synonyms for each token. So that when a +** document such as "I won first place" is tokenized, entries are +** added to the FTS index for "i", "won", "first", "1st" and +** "place". +** +** This way, even if the tokenizer does not provide synonyms +** when tokenizing query text (it should not - to do would be +** inefficient), it doesn't matter if the user queries for +** 'first + place' or '1st + place', as there are entires in the +** FTS index corresponding to both forms of the first token. +**
+** +** Whether is is parsing document or query text, any call to xToken that +** specifies a tflags argument with the FTS5_TOKEN_COLOCATED bit +** is considered to supply a synonym for the previous token. For example, +** when parsing the document "I won first place", a tokenizer that supports +** synonyms would call xToken() 5 times, as follows: +** +** +** xToken(pCtx, 0, "i", 1, 0, 1); +** xToken(pCtx, 0, "won", 3, 2, 5); +** xToken(pCtx, 0, "first", 5, 6, 11); +** xToken(pCtx, FTS5_TOKEN_COLOCATED, "1st", 3, 6, 11); +** xToken(pCtx, 0, "place", 5, 12, 17); +** +** +** It is an error to specify the FTS5_TOKEN_COLOCATED flag the first time +** xToken() is called. Multiple synonyms may be specified for a single token +** by making multiple calls to xToken(FTS5_TOKEN_COLOCATED) in sequence. +** There is no limit to the number of synonyms that may be provided for a +** single token. +** +** In many cases, method (1) above is the best approach. It does not add +** extra data to the FTS index or require FTS5 to query for multiple terms, +** so it is efficient in terms of disk space and query speed. However, it +** does not support prefix queries very well. If, as suggested above, the +** token "first" is subsituted for "1st" by the tokenizer, then the query: +** +** +** ... MATCH '1s*' +** +** will not match documents that contain the token "1st" (as the tokenizer +** will probably not map "1s" to any prefix of "first"). +** +** For full prefix support, method (3) may be preferred. In this case, +** because the index contains entries for both "first" and "1st", prefix +** queries such as 'fi*' or '1s*' will match correctly. However, because +** extra entries are added to the FTS index, this method uses more space +** within the database. +** +** Method (2) offers a midpoint between (1) and (3). Using this method, +** a query such as '1s*' will match documents that contain the literal +** token "1st", but not "first" (assuming the tokenizer is not able to +** provide synonyms for prefixes). However, a non-prefix query like '1st' +** will match against "1st" and "first". This method does not require +** extra disk space, as no extra entries are added to the FTS index. +** On the other hand, it may require more CPU cycles to run MATCH queries, +** as separate queries of the FTS index are required for each synonym. +** +** When using methods (2) or (3), it is important that the tokenizer only +** provide synonyms when tokenizing document text (method (2)) or query +** text (method (3)), not both. Doing so will not cause any errors, but is +** inefficient. */ typedef struct Fts5Tokenizer Fts5Tokenizer; typedef struct fts5_tokenizer fts5_tokenizer; @@ -309,7 +443,7 @@ struct fts5_tokenizer { void (*xDelete)(Fts5Tokenizer*); int (*xTokenize)(Fts5Tokenizer*, void *pCtx, - int flags, + int flags, /* Mask of FTS5_TOKENIZE_* flags */ const char *pText, int nText, int (*xToken)( void *pCtx, /* Copy of 2nd argument to xTokenize() */ diff --git a/manifest b/manifest index c46ab77428..2777d36021 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stests\sto\simprove\scoverage\sof\sfts5_varint.c. -D 2015-09-03T15:37:26.095 +C Add\sdocumentation\sfor\sfts5\ssynonym\ssupport. +D 2015-09-03T18:05:09.505 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in e2218eb228374422969de7b1680eda6864affcef F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -104,8 +104,8 @@ F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7 F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95 -F ext/fts5/extract_api_docs.tcl 06583c935f89075ea0b32f85efa5dd7619fcbd03 -F ext/fts5/fts5.h 0784692f406588e6c90e13a78e1f36e7e3236e42 +F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0 +F ext/fts5/fts5.h f04659e0df5af83731b102189a32280f74f4a6bc F ext/fts5/fts5Int.h f65d41f66accad0a289d6bd66b13c07d2932f9be F ext/fts5/fts5_aux.c 7a307760a9c57c750d043188ec0bad59f5b5ec7e F ext/fts5/fts5_buffer.c 80f9ba4431848cb857e3d2158f5280093dcd8015 @@ -1382,7 +1382,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 59ae30b97b40faa363c55aa2664dead9eaeeddc0 -R e54371f57da08ab0c80326ea4baedd7e +P 89f24f31a8f7d7cb0a66ee53523881f566dcb035 +R 763079caf8ff69ca16b541c2d3f4f0ad U dan -Z 416751bdad2bf139cde3272ad789aa1a +Z 726cf8268b191707cc8a3f300ac29036 diff --git a/manifest.uuid b/manifest.uuid index 3c272efde7..fc803bdc2c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -89f24f31a8f7d7cb0a66ee53523881f566dcb035 \ No newline at end of file +58aa1f435959852df74f1bca8e0bdbc4f47c256a \ No newline at end of file From 98c5f8795a050e61bf82e4d52f8a627d84f5afb7 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 3 Sep 2015 18:20:10 +0000 Subject: [PATCH 033/100] A simple optimization and size reduction in sqlite3PagerAcquire(). FossilOrigin-Name: 618d8dd4ff44cce10cc4688a2134715ff66cc562 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pager.c | 8 ++++---- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 9427db7679..a5189ddba4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\sPager.hasBeenUsed\sflag\sinto\sPager.hasHeldSharedLock\sin\sorder\sto\ntake\sit\soff\sthe\scritical\spath\sin\ssqlite3PagerAcquire(). -D 2015-09-03T17:54:32.014 +C A\ssimple\soptimization\sand\ssize\sreduction\sin\ssqlite3PagerAcquire(). +D 2015-09-03T18:20:10.962 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -324,7 +324,7 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c 76f493ed71c4154338049dee1bf6e47f69c74a55 F src/os_win.c 40b3af7a47eb1107d0d69e592bec345a3b7b798a F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca -F src/pager.c 2a9d77ac483a684c4187ec9ca786b551d3adfda5 +F src/pager.c 748ff3e183e0a311328af902c0819c6a8e3f22ca F src/pager.h 6d435f563b3f7fcae4b84433b76a6ac2730036e2 F src/parse.y f599aa5e871a493330d567ced93de696f61f48f7 F src/pcache.c cde06aa50962595e412d497e22fd2e07878ba1f0 @@ -1380,7 +1380,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 847387ec8e6fef283899578fb232b2c23b00ee5b -R 530c1d1d05feba269ca8969115207173 +P b79096be7cb02aae2f303db33a8bf19e69204374 +R d1173d7149bc899e819d5c3a3bb552ec U drh -Z e28985417ac4a7094ed6187f7e395585 +Z 442809bedf9761a63ee7744b080363fa diff --git a/manifest.uuid b/manifest.uuid index eaef64ee94..67e347a346 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b79096be7cb02aae2f303db33a8bf19e69204374 \ No newline at end of file +618d8dd4ff44cce10cc4688a2134715ff66cc562 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index f4e27f36ca..dc25b90be6 100644 --- a/src/pager.c +++ b/src/pager.c @@ -5254,20 +5254,20 @@ int sqlite3PagerAcquire( ** page 1 if there is no write-transaction open or the ACQUIRE_READONLY ** flag was specified by the caller. And so long as the db is not a ** temporary or in-memory database. */ - const int bMmapOk = (pgno!=1 && USEFETCH(pPager) + const int bMmapOk = (pgno>1 && USEFETCH(pPager) && (pPager->eState==PAGER_READER || (flags & PAGER_GET_READONLY)) #ifdef SQLITE_HAS_CODEC && pPager->xCodec==0 #endif ); + if( pgno<=1 && pgno==0 ){ + return SQLITE_CORRUPT_BKPT; + } assert( pPager->eState>=PAGER_READER ); assert( assert_pager_state(pPager) ); assert( noContent==0 || bMmapOk==0 ); - if( pgno==0 ){ - return SQLITE_CORRUPT_BKPT; - } assert( pPager->hasHeldSharedLock==1 ); /* If the pager is in the error state, return an error immediately. From f8efe64649d4d4a646007bf8a76897a0b20ce95e Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 3 Sep 2015 19:48:02 +0000 Subject: [PATCH 034/100] Rearrange code in fts5_expr.c so that synonym support does not slow down the common case. FossilOrigin-Name: 801882817f1d895aef1426f9a7196bd140b807c3 --- ext/fts5/fts5_expr.c | 11 +++++++++-- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index ea5fd8a09e..b68634694f 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -1181,10 +1181,17 @@ static int fts5ExprNodeNext( }; case FTS5_TERM: { - rc = fts5ExprNearAdvanceFirst(pExpr, pNode, bFromValid, iFrom); - if( pNode->bEof==0 ){ + Fts5IndexIter *pIter = pNode->pNear->apPhrase[0]->aTerm[0].pIter; + if( bFromValid ){ + rc = sqlite3Fts5IterNextFrom(pIter, iFrom); + }else{ + rc = sqlite3Fts5IterNext(pIter); + } + if( rc==SQLITE_OK && sqlite3Fts5IterEof(pIter)==0 ){ assert( rc==SQLITE_OK ); rc = fts5ExprTokenTest(pExpr, pNode); + }else{ + pNode->bEof = 1; } return rc; }; diff --git a/manifest b/manifest index b7ec71865d..20d7c879fc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\senhancements\sfrom\strunk. -D 2015-09-03T18:57:52.309 +C Rearrange\scode\sin\sfts5_expr.c\sso\sthat\ssynonym\ssupport\sdoes\snot\sslow\sdown\sthe\scommon\scase. +D 2015-09-03T19:48:02.456 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -110,7 +110,7 @@ F ext/fts5/fts5Int.h f65d41f66accad0a289d6bd66b13c07d2932f9be F ext/fts5/fts5_aux.c 7a307760a9c57c750d043188ec0bad59f5b5ec7e F ext/fts5/fts5_buffer.c 80f9ba4431848cb857e3d2158f5280093dcd8015 F ext/fts5/fts5_config.c 80b61fd2c6844b64a3e72a64572d50a812da9384 -F ext/fts5/fts5_expr.c 0c36c1db8eccdeb006e3c8d1499d05015f6e11a6 +F ext/fts5/fts5_expr.c 1c24e1a2ffb286bfe37e537a43b7fadabfe993d4 F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246 F ext/fts5/fts5_index.c febb68173333ae3248eb15928a18b21112d45135 F ext/fts5/fts5_main.c e9d0892424bb7f0a8b58613d4ff75cb650cf286e @@ -1382,7 +1382,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 58aa1f435959852df74f1bca8e0bdbc4f47c256a 618d8dd4ff44cce10cc4688a2134715ff66cc562 -R 38e68c75949f5efa3d52c01a80d7f0aa +P 4b49fe996989fe42d2bb1c24e7193fef09c5fc50 +R 0fe79065145fa2e8621202f9c4e41627 U dan -Z 9f85937304cd0ef1147cd11d7113913e +Z fe72ec30c1ce282b0a02c9b8970f9360 diff --git a/manifest.uuid b/manifest.uuid index d9cd6ed680..005faca754 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4b49fe996989fe42d2bb1c24e7193fef09c5fc50 \ No newline at end of file +801882817f1d895aef1426f9a7196bd140b807c3 \ No newline at end of file From 95a0b371566fcb1219c0277682a03b7a2a3cea30 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 3 Sep 2015 20:43:55 +0000 Subject: [PATCH 035/100] Change the pcache module to keep track of the total number of references to all pages rather than the number of pages references, for a performance improvement and size reduction. FossilOrigin-Name: f00a9e1e998c4bd249a45444dc2d71a7e4903b8b --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/pager.c | 6 +++++- src/pcache.c | 27 ++++++++++++++------------- src/pcache1.c | 5 ++++- 5 files changed, 32 insertions(+), 24 deletions(-) diff --git a/manifest b/manifest index a5189ddba4..febdf5011b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C A\ssimple\soptimization\sand\ssize\sreduction\sin\ssqlite3PagerAcquire(). -D 2015-09-03T18:20:10.962 +C Change\sthe\spcache\smodule\sto\skeep\strack\sof\sthe\stotal\snumber\sof\sreferences\sto\nall\spages\srather\sthan\sthe\snumber\sof\spages\sreferences,\sfor\sa\sperformance\nimprovement\sand\ssize\sreduction. +D 2015-09-03T20:43:55.204 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -324,12 +324,12 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c 76f493ed71c4154338049dee1bf6e47f69c74a55 F src/os_win.c 40b3af7a47eb1107d0d69e592bec345a3b7b798a F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca -F src/pager.c 748ff3e183e0a311328af902c0819c6a8e3f22ca +F src/pager.c 4784012f80b2197c61ff6eaf4f5c7026d93253fd F src/pager.h 6d435f563b3f7fcae4b84433b76a6ac2730036e2 F src/parse.y f599aa5e871a493330d567ced93de696f61f48f7 -F src/pcache.c cde06aa50962595e412d497e22fd2e07878ba1f0 +F src/pcache.c 24be750c79272e0ca7b6e007bc94999700f3e5ef F src/pcache.h 9968603796240cdf83da7e7bef76edf90619cea9 -F src/pcache1.c b31af9dbc83b9c68e87d681b8453a9605f28e734 +F src/pcache1.c e1529369c047ac645e6a28196f25b7e936c46b82 F src/pragma.c d71b813e67bf03f3116b9dd5164fbfd81ec673a2 F src/pragma.h 631a91c8b0e6ca8f051a1d8a4a0da4150e04620a F src/prepare.c 82e5db1013846a819f198336fed72c44c974e7b1 @@ -1380,7 +1380,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P b79096be7cb02aae2f303db33a8bf19e69204374 -R d1173d7149bc899e819d5c3a3bb552ec +P 618d8dd4ff44cce10cc4688a2134715ff66cc562 +R 7304361de7c7fa82f683343fecc699dd U drh -Z 442809bedf9761a63ee7744b080363fa +Z 8da3824804c6c5593515c7564fca1c5b diff --git a/manifest.uuid b/manifest.uuid index 67e347a346..4550487b63 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -618d8dd4ff44cce10cc4688a2134715ff66cc562 \ No newline at end of file +f00a9e1e998c4bd249a45444dc2d71a7e4903b8b \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index dc25b90be6..2f7c330d60 100644 --- a/src/pager.c +++ b/src/pager.c @@ -5261,6 +5261,10 @@ int sqlite3PagerAcquire( #endif ); + /* Optimization note: Adding the "pgno<=1" term before "pgno==0" here + ** allows the compiler optimizer to reuse the results of the "pgno>1" + ** test in the previous statement, and avoid testing pgno==0 in the + ** common case where pgno is large. */ if( pgno<=1 && pgno==0 ){ return SQLITE_CORRUPT_BKPT; } @@ -6390,7 +6394,7 @@ u8 sqlite3PagerIsreadonly(Pager *pPager){ #ifdef SQLITE_DEBUG /* -** Return the number of references to the pager. +** Return the sum of the reference counts for all pages held by pPager. */ int sqlite3PagerRefcount(Pager *pPager){ return sqlite3PcacheRefCount(pPager->pPCache); diff --git a/src/pcache.c b/src/pcache.c index 58c05ac2a4..e39262cb8c 100644 --- a/src/pcache.c +++ b/src/pcache.c @@ -19,7 +19,7 @@ struct PCache { PgHdr *pDirty, *pDirtyTail; /* List of dirty pages in LRU order */ PgHdr *pSynced; /* Last synced page in dirty page list */ - int nRef; /* Number of referenced pages */ + int nRefSum; /* Sum of ref counts over all pages */ int szCache; /* Configured cache size */ int szPage; /* Size of every page in this cache */ int szExtra; /* Size of extra space for each page */ @@ -184,7 +184,7 @@ int sqlite3PcacheOpen( ** are no outstanding page references when this function is called. */ int sqlite3PcacheSetPageSize(PCache *pCache, int szPage){ - assert( pCache->nRef==0 && pCache->pDirty==0 ); + assert( pCache->nRefSum==0 && pCache->pDirty==0 ); if( pCache->szPage ){ sqlite3_pcache *pNew; pNew = sqlite3GlobalConfig.pcache2.xCreate( @@ -351,9 +351,7 @@ PgHdr *sqlite3PcacheFetchFinish( if( !pPgHdr->pPage ){ return pcacheFetchFinishWithInit(pCache, pgno, pPage); } - if( 0==pPgHdr->nRef ){ - pCache->nRef++; - } + pCache->nRefSum++; pPgHdr->nRef++; return pPgHdr; } @@ -364,9 +362,8 @@ PgHdr *sqlite3PcacheFetchFinish( */ void SQLITE_NOINLINE sqlite3PcacheRelease(PgHdr *p){ assert( p->nRef>0 ); - p->nRef--; - if( p->nRef==0 ){ - p->pCache->nRef--; + p->pCache->nRefSum--; + if( (--p->nRef)==0 ){ if( p->flags&PGHDR_CLEAN ){ pcacheUnpin(p); }else if( p->pDirtyPrev!=0 ){ @@ -382,6 +379,7 @@ void SQLITE_NOINLINE sqlite3PcacheRelease(PgHdr *p){ void sqlite3PcacheRef(PgHdr *p){ assert(p->nRef>0); p->nRef++; + p->pCache->nRefSum++; } /* @@ -394,7 +392,7 @@ void sqlite3PcacheDrop(PgHdr *p){ if( p->flags&PGHDR_DIRTY ){ pcacheManageDirtyList(p, PCACHE_DIRTYLIST_REMOVE); } - p->pCache->nRef--; + p->pCache->nRefSum--; sqlite3GlobalConfig.pcache2.xUnpin(p->pCache->pCache, p->pPage, 1); } @@ -490,11 +488,11 @@ void sqlite3PcacheTruncate(PCache *pCache, Pgno pgno){ sqlite3PcacheMakeClean(p); } } - if( pgno==0 && pCache->nRef ){ + if( pgno==0 && pCache->nRefSum ){ sqlite3_pcache_page *pPage1; pPage1 = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache,1,0); if( ALWAYS(pPage1) ){ /* Page 1 is always available in cache, because - ** pCache->nRef>0 */ + ** pCache->nRefSum>0 */ memset(pPage1->pBuf, 0, pCache->szPage); pgno = 1; } @@ -600,10 +598,13 @@ PgHdr *sqlite3PcacheDirtyList(PCache *pCache){ } /* -** Return the total number of referenced pages held by the cache. +** Return the total number of references to all pages held by the cache. +** +** This is not the total number of pages referenced, but the sum of the +** reference count for all pages. */ int sqlite3PcacheRefCount(PCache *pCache){ - return pCache->nRef; + return pCache->nRefSum; } /* diff --git a/src/pcache1.c b/src/pcache1.c index 18df8c87d3..c92238ef06 100644 --- a/src/pcache1.c +++ b/src/pcache1.c @@ -961,7 +961,10 @@ static PgHdr1 *pcache1FetchNoMutex( pPage = pCache->apHash[iKey % pCache->nHash]; while( pPage && pPage->iKey!=iKey ){ pPage = pPage->pNext; } - /* Step 2: Abort if no existing page is found and createFlag is 0 */ + /* Step 2: If the page was found in the hash table, then return it. + ** If the page was not in the hash table and createFlag is 0, abort. + ** Otherwise (page not in hash and createFlag!=0) continue with + ** subsequent steps to try to create the page. */ if( pPage ){ if( !pPage->isPinned ){ return pcache1PinPage(pPage); From 92af02c9391639469f4372f59ac08e912da0cdb1 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 4 Sep 2015 04:31:56 +0000 Subject: [PATCH 036/100] Simplification of the LRU list handling in pcache1. FossilOrigin-Name: 05a3a2cd140587265b5427d23c93c5be1f39e199 --- manifest | 12 +++---- manifest.uuid | 2 +- src/pcache1.c | 91 ++++++++++++++++++++++++++------------------------- 3 files changed, 53 insertions(+), 52 deletions(-) diff --git a/manifest b/manifest index febdf5011b..bfd4ab3109 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\spcache\smodule\sto\skeep\strack\sof\sthe\stotal\snumber\sof\sreferences\sto\nall\spages\srather\sthan\sthe\snumber\sof\spages\sreferences,\sfor\sa\sperformance\nimprovement\sand\ssize\sreduction. -D 2015-09-03T20:43:55.204 +C Simplification\sof\sthe\sLRU\slist\shandling\sin\spcache1. +D 2015-09-04T04:31:56.886 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -329,7 +329,7 @@ F src/pager.h 6d435f563b3f7fcae4b84433b76a6ac2730036e2 F src/parse.y f599aa5e871a493330d567ced93de696f61f48f7 F src/pcache.c 24be750c79272e0ca7b6e007bc94999700f3e5ef F src/pcache.h 9968603796240cdf83da7e7bef76edf90619cea9 -F src/pcache1.c e1529369c047ac645e6a28196f25b7e936c46b82 +F src/pcache1.c bf2afe64a3dedb8643c8dcbd94a145cc80ab2a67 F src/pragma.c d71b813e67bf03f3116b9dd5164fbfd81ec673a2 F src/pragma.h 631a91c8b0e6ca8f051a1d8a4a0da4150e04620a F src/prepare.c 82e5db1013846a819f198336fed72c44c974e7b1 @@ -1380,7 +1380,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 618d8dd4ff44cce10cc4688a2134715ff66cc562 -R 7304361de7c7fa82f683343fecc699dd +P f00a9e1e998c4bd249a45444dc2d71a7e4903b8b +R 231caff1c142ba151d728fb853afd78c U drh -Z 8da3824804c6c5593515c7564fca1c5b +Z ce45274979109dc4b492f06dc4bc7386 diff --git a/manifest.uuid b/manifest.uuid index 4550487b63..562662b19b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f00a9e1e998c4bd249a45444dc2d71a7e4903b8b \ No newline at end of file +05a3a2cd140587265b5427d23c93c5be1f39e199 \ No newline at end of file diff --git a/src/pcache1.c b/src/pcache1.c index c92238ef06..2fed0bdff1 100644 --- a/src/pcache1.c +++ b/src/pcache1.c @@ -87,6 +87,24 @@ typedef struct PgHdr1 PgHdr1; typedef struct PgFreeslot PgFreeslot; typedef struct PGroup PGroup; +/* +** Each cache entry is represented by an instance of the following +** structure. Unless SQLITE_PCACHE_SEPARATE_HEADER is defined, a buffer of +** PgHdr1.pCache->szPage bytes is allocated directly before this structure +** in memory. +*/ +struct PgHdr1 { + sqlite3_pcache_page page; /* Base class. Must be first. pBuf & pExtra */ + unsigned int iKey; /* Key value (page number) */ + u8 isPinned; /* Page in use, not on the LRU list */ + u8 isBulkLocal; /* This page from bulk local storage */ + u8 isAnchor; /* This is the PGroup.lru element */ + PgHdr1 *pNext; /* Next in hash table chain */ + PCache1 *pCache; /* Cache that currently owns this page */ + PgHdr1 *pLruNext; /* Next in LRU list of unpinned pages */ + PgHdr1 *pLruPrev; /* Previous in LRU list of unpinned pages */ +}; + /* Each page cache (or PCache) belongs to a PGroup. A PGroup is a set ** of one or more PCaches that are able to recycle each other's unpinned ** pages when they are under memory pressure. A PGroup is an instance of @@ -115,7 +133,7 @@ struct PGroup { unsigned int nMinPage; /* Sum of nMin for purgeable caches */ unsigned int mxPinned; /* nMaxpage + 10 - nMinPage */ unsigned int nCurrentPage; /* Number of purgeable pages allocated */ - PgHdr1 *pLruHead, *pLruTail; /* LRU list of unpinned pages */ + PgHdr1 lru; /* The beginning and end of the LRU list */ }; /* Each page cache is an instance of the following object. Every @@ -153,23 +171,6 @@ struct PCache1 { void *pBulk; /* Bulk memory used by pcache-local */ }; -/* -** Each cache entry is represented by an instance of the following -** structure. Unless SQLITE_PCACHE_SEPARATE_HEADER is defined, a buffer of -** PgHdr1.pCache->szPage bytes is allocated directly before this structure -** in memory. -*/ -struct PgHdr1 { - sqlite3_pcache_page page; - unsigned int iKey; /* Key value (page number) */ - u8 isPinned; /* Page in use, not on the LRU list */ - u8 isBulkLocal; /* This page from bulk local storage */ - PgHdr1 *pNext; /* Next in hash table chain */ - PCache1 *pCache; /* Cache that currently owns this page */ - PgHdr1 *pLruNext; /* Next in LRU list of unpinned pages */ - PgHdr1 *pLruPrev; /* Previous in LRU list of unpinned pages */ -}; - /* ** Free slots in the allocator used to divide up the global page cache ** buffer provided using the SQLITE_CONFIG_PAGECACHE mechanism. @@ -230,6 +231,7 @@ static SQLITE_WSD struct PCacheGlobal { /******************************************************************************/ /******** Page Allocation/SQLITE_CONFIG_PCACHE Related Functions **************/ + /* ** This function is called during initialization if a static buffer is ** supplied to use for the page-cache by passing the SQLITE_CONFIG_PAGECACHE @@ -289,6 +291,7 @@ static int pcache1InitBulk(PCache1 *pCache){ pX->page.pBuf = zBulk; pX->page.pExtra = &pX[1]; pX->isBulkLocal = 1; + pX->isAnchor = 0; pX->pNext = pCache->pFree; pCache->pFree = pX; zBulk += pCache->szAlloc; @@ -431,6 +434,7 @@ static PgHdr1 *pcache1AllocPage(PCache1 *pCache, int benignMalloc){ p->page.pBuf = pPg; p->page.pExtra = &p[1]; p->isBulkLocal = 0; + p->isAnchor = 0; } if( pCache->bPurgeable ){ pCache->pGroup->nCurrentPage++; @@ -557,22 +561,16 @@ static PgHdr1 *pcache1PinPage(PgHdr1 *pPage){ assert( pPage!=0 ); assert( pPage->isPinned==0 ); pCache = pPage->pCache; - assert( pPage->pLruNext || pPage==pCache->pGroup->pLruTail ); - assert( pPage->pLruPrev || pPage==pCache->pGroup->pLruHead ); + assert( pPage->pLruNext ); + assert( pPage->pLruPrev ); assert( sqlite3_mutex_held(pCache->pGroup->mutex) ); - if( pPage->pLruPrev ){ - pPage->pLruPrev->pLruNext = pPage->pLruNext; - }else{ - pCache->pGroup->pLruHead = pPage->pLruNext; - } - if( pPage->pLruNext ){ - pPage->pLruNext->pLruPrev = pPage->pLruPrev; - }else{ - pCache->pGroup->pLruTail = pPage->pLruPrev; - } + pPage->pLruPrev->pLruNext = pPage->pLruNext; + pPage->pLruNext->pLruPrev = pPage->pLruPrev; pPage->pLruNext = 0; pPage->pLruPrev = 0; pPage->isPinned = 1; + assert( pPage->isAnchor==0 ); + assert( pCache->pGroup->lru.isAnchor==1 ); pCache->nRecyclable--; return pPage; } @@ -605,9 +603,11 @@ static void pcache1RemoveFromHash(PgHdr1 *pPage, int freeFlag){ */ static void pcache1EnforceMaxPage(PCache1 *pCache){ PGroup *pGroup = pCache->pGroup; + PgHdr1 *p; assert( sqlite3_mutex_held(pGroup->mutex) ); - while( pGroup->nCurrentPage>pGroup->nMaxPage && pGroup->pLruTail ){ - PgHdr1 *p = pGroup->pLruTail; + while( pGroup->nCurrentPage>pGroup->nMaxPage + && (p=pGroup->lru.pLruPrev)->isAnchor==0 + ){ assert( p->pCache->pGroup==pGroup ); assert( p->isPinned==0 ); pcache1PinPage(p); @@ -741,6 +741,10 @@ static sqlite3_pcache *pcache1Create(int szPage, int szExtra, int bPurgeable){ }else{ pGroup = &pcache1.grp; } + if( pGroup->lru.isAnchor==0 ){ + pGroup->lru.isAnchor = 1; + pGroup->lru.pLruPrev = pGroup->lru.pLruNext = &pGroup->lru; + } pCache->pGroup = pGroup; pCache->szPage = szPage; pCache->szExtra = szExtra; @@ -848,11 +852,11 @@ static SQLITE_NOINLINE PgHdr1 *pcache1FetchStage2( /* Step 4. Try to recycle a page. */ if( pCache->bPurgeable - && pGroup->pLruTail + && !pGroup->lru.pLruPrev->isAnchor && ((pCache->nPage+1>=pCache->nMax) || pcache1UnderMemoryPressure(pCache)) ){ PCache1 *pOther; - pPage = pGroup->pLruTail; + pPage = pGroup->lru.pLruPrev; assert( pPage->isPinned==0 ); pcache1RemoveFromHash(pPage, 0); pcache1PinPage(pPage); @@ -1041,21 +1045,16 @@ static void pcache1Unpin( ** part of the PGroup LRU list. */ assert( pPage->pLruPrev==0 && pPage->pLruNext==0 ); - assert( pGroup->pLruHead!=pPage && pGroup->pLruTail!=pPage ); assert( pPage->isPinned==1 ); if( reuseUnlikely || pGroup->nCurrentPage>pGroup->nMaxPage ){ pcache1RemoveFromHash(pPage, 1); }else{ /* Add the page to the PGroup LRU list. */ - if( pGroup->pLruHead ){ - pGroup->pLruHead->pLruPrev = pPage; - pPage->pLruNext = pGroup->pLruHead; - pGroup->pLruHead = pPage; - }else{ - pGroup->pLruTail = pPage; - pGroup->pLruHead = pPage; - } + PgHdr1 **ppFirst = &pGroup->lru.pLruNext; + pPage->pLruPrev = &pGroup->lru; + (pPage->pLruNext = *ppFirst)->pLruPrev = pPage; + *ppFirst = pPage; pCache->nRecyclable++; pPage->isPinned = 0; } @@ -1193,7 +1192,9 @@ int sqlite3PcacheReleaseMemory(int nReq){ if( sqlite3GlobalConfig.nPage==0 ){ PgHdr1 *p; pcache1EnterMutex(&pcache1.grp); - while( (nReq<0 || nFreeisAnchor==0 + ){ nFree += pcache1MemSize(p->page.pBuf); #ifdef SQLITE_PCACHE_SEPARATE_HEADER nFree += sqlite3MemSize(p); @@ -1221,7 +1222,7 @@ void sqlite3PcacheStats( ){ PgHdr1 *p; int nRecyclable = 0; - for(p=pcache1.grp.pLruHead; p; p=p->pLruNext){ + for(p=pcache1.grp.lru.pLruNext; !p->isAnchor; p=p->pLruNext){ assert( p->isPinned==0 ); nRecyclable++; } From 4dfe98a8c436251bd5abaafb1846d8d49ed540e1 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 4 Sep 2015 11:13:00 +0000 Subject: [PATCH 037/100] Enhance showfts5.tcl so that it can optionally display the number of terms in each segment. FossilOrigin-Name: d648ddd93de039820f5abe064c7bc1318cd9d6b1 --- ext/fts5/fts5_index.c | 6 ++-- ext/fts5/test/fts5aa.test | 2 +- ext/fts5/test/fts5ah.test | 4 +-- ext/fts5/tool/showfts5.tcl | 58 ++++++++++++++++++++++++++++++++++---- manifest | 18 ++++++------ manifest.uuid | 2 +- 6 files changed, 69 insertions(+), 21 deletions(-) diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 7bb744773c..5bf4feba93 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -5104,13 +5104,13 @@ static void fts5DebugRowid(int *pRc, Fts5Buffer *pBuf, i64 iKey){ if( iSegid==0 ){ if( iKey==FTS5_AVERAGES_ROWID ){ - sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "(averages) "); + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "{averages} "); }else{ - sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "(structure)"); + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "{structure}"); } } else{ - sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "(%ssegid=%d h=%d pgno=%d)", + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "{%ssegid=%d h=%d pgno=%d}", bDlidx ? "dlidx " : "", iSegid, iHeight, iPgno ); } diff --git a/ext/fts5/test/fts5aa.test b/ext/fts5/test/fts5aa.test index 5f3a9d9e1d..e0ff6a6132 100644 --- a/ext/fts5/test/fts5aa.test +++ b/ext/fts5/test/fts5aa.test @@ -51,7 +51,7 @@ do_execsql_test 2.1 { do_test 2.2 { execsql { SELECT fts5_decode(id, block) FROM t1_data WHERE id==10 } -} {/{\(structure\) {lvl=0 nMerge=0 nSeg=1 {id=[0123456789]* h=0 leaves=1..1}}}/} +} {/{{structure} {lvl=0 nMerge=0 nSeg=1 {id=[0123456789]* h=0 leaves=1..1}}}/} foreach w {a b c d e f} { do_execsql_test 2.3.$w.asc { diff --git a/ext/fts5/test/fts5ah.test b/ext/fts5/test/fts5ah.test index 3c8ad253d1..6d7e39f793 100644 --- a/ext/fts5/test/fts5ah.test +++ b/ext/fts5/test/fts5ah.test @@ -90,13 +90,13 @@ foreach {tn q res} " do_test 1.6.$tn.1 { set n [execsql_reads $q] - puts -nonewline "(n=$n nReadX=$nReadX)" + #puts -nonewline "(n=$n nReadX=$nReadX)" expr {$n < ($nReadX / 8)} } {1} do_test 1.6.$tn.2 { set n [execsql_reads "$q ORDER BY rowid DESC"] - puts -nonewline "(n=$n nReadX=$nReadX)" + #puts -nonewline "(n=$n nReadX=$nReadX)" expr {$n < ($nReadX / 8)} } {1} diff --git a/ext/fts5/tool/showfts5.tcl b/ext/fts5/tool/showfts5.tcl index 846902b3be..d9af5f38eb 100644 --- a/ext/fts5/tool/showfts5.tcl +++ b/ext/fts5/tool/showfts5.tcl @@ -5,14 +5,52 @@ # Process command line arguments. # proc usage {} { - puts stderr "usage: $::argv0 database table" + puts stderr "usage: $::argv0 ?OPTIONS? database table" + puts stderr "" + puts stderr " -nterm (count number of terms in each segment)" puts stderr "" exit 1 } -if {[llength $argv]!=2} usage -set database [lindex $argv 0] -set tbl [lindex $argv 1] +set O(nterm) 0 + +if {[llength $argv]<2} usage +foreach a [lrange $argv 0 end-2] { + switch -- $a { + -nterm { + set O(nterm) 1 + } + + default { + usage + } + } +} + +set database [lindex $argv end-1] +set tbl [lindex $argv end] + + +#------------------------------------------------------------------------- +# Count the number of terms in each segment of fts5 table $tbl. Store the +# counts in the array variable in the parent context named by parameter +# $arrayname, indexed by segment-id. Example: +# +# count_terms fts_tbl A +# foreach {k v} [array get A] { puts "segid=$k nTerm=$v" } +# +proc count_terms {tbl arrayname} { + upvar A $arrayname + array unset A + db eval "SELECT fts5_decode(rowid, block) AS d FROM ${tbl}_data" { + set desc [lindex $d 0] + if {[regexp {^segid=([0-9]*)} $desc -> id]} { + foreach i [lrange $d 1 end] { + if {[string match {term=*} $i]} { incr A($id) } + } + } + } +} #------------------------------------------------------------------------- @@ -21,11 +59,21 @@ set tbl [lindex $argv 1] sqlite3 db $database catch { load_static_extension db fts5 } +if {$O(nterm)} { count_terms $tbl A } + db eval "SELECT fts5_decode(rowid, block) AS d FROM ${tbl}_data WHERE id=10" { foreach lvl [lrange $d 1 end] { puts [lrange $lvl 0 2] + foreach seg [lrange $lvl 3 end] { - puts " $seg" + if {$::O(nterm)} { + regexp {^id=([0-9]*)} $seg -> id + set nTerm 0 + catch { set nTerm $A($id) } + puts [format " % -28s nTerm=%d" $seg $nTerm] + } else { + puts [format " % -28s" $seg] + } } } } diff --git a/manifest b/manifest index e7ca9c9ceb..b900ddf1c9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Modify\sthe\sfts5\scustom\stokenizer\sinterface\sto\spermit\ssynonym\ssupport.\sThe\sfts5_api.iVersion\svalue\sis\snow\sset\sto\s2.\sExisting\sfts5\scustom\stokenizers\s(if\sthere\sare\ssuch\sthings)\swill\sneed\sto\sbe\supdated\sto\suse\sthe\snew\sapi\sversion. -D 2015-09-04T10:31:51.624 +C Enhance\sshowfts5.tcl\sso\sthat\sit\scan\soptionally\sdisplay\sthe\snumber\sof\sterms\sin\seach\ssegment. +D 2015-09-04T11:13:00.822 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -112,7 +112,7 @@ F ext/fts5/fts5_buffer.c 80f9ba4431848cb857e3d2158f5280093dcd8015 F ext/fts5/fts5_config.c 80b61fd2c6844b64a3e72a64572d50a812da9384 F ext/fts5/fts5_expr.c 1c24e1a2ffb286bfe37e537a43b7fadabfe993d4 F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246 -F ext/fts5/fts5_index.c febb68173333ae3248eb15928a18b21112d45135 +F ext/fts5/fts5_index.c 950e37028cc81ae21534819e79c73aea7efa6c8e F ext/fts5/fts5_main.c e9d0892424bb7f0a8b58613d4ff75cb650cf286e F ext/fts5/fts5_storage.c 120f7b143688b5b7710dacbd48cff211609b8059 F ext/fts5/fts5_tcl.c 6da58d6e8f42a93c4486b5ba9b187a7f995dee37 @@ -124,14 +124,14 @@ F ext/fts5/fts5_vocab.c 4622e0b7d84a488a1585aaa56eb214ee67a988bc F ext/fts5/fts5parse.y 833db1101b78c0c47686ab1b84918e38c36e9452 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba F ext/fts5/test/fts5_common.tcl b6e6a40ef5d069c8e86ca4fbad491e1195485dbc -F ext/fts5/test/fts5aa.test be961d10b0eff5e973c37b3b7a32a18cda87a77c +F ext/fts5/test/fts5aa.test f558e1e5ccffa75d69e9a4814245d468ec6b6608 F ext/fts5/test/fts5ab.test 6fe3a56731d15978afbb74ae51b355fc9310f2ad F ext/fts5/test/fts5ac.test 9737992d08c56bfd4803e933744d2d764e23795c F ext/fts5/test/fts5ad.test b2edee8b7de0c21d2c88f8a18c195034aad6952d F ext/fts5/test/fts5ae.test 0a9984fc3479f89f8c63d9848d6ed0c465dfcebe F ext/fts5/test/fts5af.test c2501ec2b61d6b179c305f5d2b8782ab3d4f832a F ext/fts5/test/fts5ag.test ec3e119b728196620a31507ef503c455a7a73505 -F ext/fts5/test/fts5ah.test b9e78fa986a7bd564ebadfb244de02c84d7ac3ae +F ext/fts5/test/fts5ah.test e592c4978622dbc4de552cd0f9395df60ac5d54c F ext/fts5/test/fts5ai.test f20e53bbf0c55bc596f1fd47f2740dae028b8f37 F ext/fts5/test/fts5aj.test 05b569f5c16ea3098fb1984eec5cf50dbdaae5d8 F ext/fts5/test/fts5ak.test 7b8c5df96df599293f920b7e5521ebc79f647592 @@ -183,7 +183,7 @@ F ext/fts5/test/fts5version.test 205beb2a67d9496af64df959e6a19238f69b83e8 F ext/fts5/test/fts5vocab.test cdf97b9678484e9bad5062edf9c9106e5c3b0c5c F ext/fts5/tool/loadfts5.tcl 95edf0b6b92a09f9ed85595038b1108127987556 F ext/fts5/tool/mkfts5c.tcl 5745072c7de346e18c7f491e4c3281fe8a1cfe51 -F ext/fts5/tool/showfts5.tcl fb62e8eae6d862afdd22f367e286fb886d5e1ab6 +F ext/fts5/tool/showfts5.tcl 9eaf6c3df352f98a2ab5ce1921dd94128ab1381d F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43 F ext/icu/icu.c b2732aef0b076e4276d9b39b5a33cec7a05e1413 F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37 @@ -1382,7 +1382,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 05a3a2cd140587265b5427d23c93c5be1f39e199 443a5eb8e17fd4f0b83ecc5bba74848e2c8a968c -R 3837e1c9ccb54eb6541c962b4dbcd417 +P 0b7e4ab8abde3ae32459233df115c433dd58d2c1 +R bc4ba10677c1404768969390b0481a23 U dan -Z 9252bca216b9264b05e49a4db9d079d2 +Z d0a91d26d23415f8f905dbc22cf63e6b diff --git a/manifest.uuid b/manifest.uuid index 36dfbd374c..43a83fe0dd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0b7e4ab8abde3ae32459233df115c433dd58d2c1 \ No newline at end of file +d648ddd93de039820f5abe064c7bc1318cd9d6b1 \ No newline at end of file From edb04ed946673e8c8f1d50df81edc94f0be485d6 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 4 Sep 2015 12:54:01 +0000 Subject: [PATCH 038/100] Continue to support the (broken) legacy syntax of allowing strings for column names in CREATE INDEX statements and in UNIQUE and PRIMARY KEY constraints. FossilOrigin-Name: 3d3df79bfaf9dbc7aa711c08a19d2f6dbe744b32 --- manifest | 22 +++++++++++----------- manifest.uuid | 2 +- src/build.c | 25 +++++++++++++++++++++++++ test/index2.test | 3 +-- test/index3.test | 34 +++++++++++++++++++++++++++++----- test/where.test | 4 ++-- test/where4.test | 6 +++--- 7 files changed, 72 insertions(+), 24 deletions(-) diff --git a/manifest b/manifest index b900ddf1c9..ccdfa438a5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sshowfts5.tcl\sso\sthat\sit\scan\soptionally\sdisplay\sthe\snumber\sof\sterms\sin\seach\ssegment. -D 2015-09-04T11:13:00.822 +C Continue\sto\ssupport\sthe\s(broken)\slegacy\ssyntax\sof\sallowing\sstrings\sfor\ncolumn\snames\sin\sCREATE\sINDEX\sstatements\sand\sin\sUNIQUE\sand\sPRIMARY\sKEY\nconstraints. +D 2015-09-04T12:54:01.394 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -284,7 +284,7 @@ F src/btmutex.c 45a968cc85afed9b5e6cf55bf1f42f8d18107f79 F src/btree.c 4084d9eed2817331f6e6a82230ba30e448cad497 F src/btree.h 969adc948e89e449220ff0ff724c94bb2a52e9f1 F src/btreeInt.h 8177c9ab90d772d6d2c6c517e05bed774b7c92c0 -F src/build.c 6c3a8a9b21402a6be98126f7d86b76527e68ca67 +F src/build.c 3f6176b3af04b85715559d435097c556ba473801 F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 F src/complete.c addcd8160b081131005d5bc2d34adf20c1c5c92f F src/ctime.c 5a0b735dc95604766f5dac73973658eef782ee8b @@ -775,8 +775,8 @@ F test/incrvacuum2.test 676c41428765d58f1da7dbe659ef27726d3d30ac F test/incrvacuum3.test 75256fb1377e7c39ef2de62bfc42bbff67be295a F test/incrvacuum_ioerr.test 6ae2f783424e47a0033304808fe27789cf93e635 F test/index.test 4d990005a67a36984e4f1a5f1bdccea8d08da4ee -F test/index2.test ee83c6b5e3173a3d7137140d945d9a5d4fdfb9d6 -F test/index3.test b6ec456cf3b81d9a32123fe7e449bde434db338b +F test/index2.test f835d5e13ca163bd78c4459ca15fd2e4ed487407 +F test/index3.test fa3e49bbaa4f38091c9c742e36a1abe67c4ef1fc F test/index4.test ab92e736d5946840236cd61ac3191f91a7856bf6 F test/index5.test 8621491915800ec274609e42e02a97d67e9b13e7 F test/index6.test 7102ec371414c42dfb1d5ca37eb4519aa9edc23a @@ -1288,10 +1288,10 @@ F test/walro.test 34422d1d95aaff0388f0791ec20edb34e2a3ed57 F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417 F test/walslow.test e7be6d9888f83aa5d3d3c7c08aa9b5c28b93609a F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e -F test/where.test 1ff3d9f8da0a6c0dc5ccfd38d9225b2cdb5b6afb +F test/where.test 66d4c107e82dfe86c01a96277b77e7a8809aff0b F test/where2.test af78c55589cbc82d793449493adba0dc3d659f23 F test/where3.test 1ad55ba900bd7747f98b6082e65bd3e442c5004e -F test/where4.test 68aa5ad796e33816db2078bc0f6de719c7a0e21f +F test/where4.test 44f506bf1737cf0fa4fc795e340208250f1fcd89 F test/where5.test fdf66f96d29a064b63eb543e28da4dfdccd81ad2 F test/where6.test 5da5a98cec820d488e82708301b96cb8c18a258b F test/where7.test 5a4b0abc207d71da4deecd734ad8579e8dd40aa8 @@ -1382,7 +1382,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 0b7e4ab8abde3ae32459233df115c433dd58d2c1 -R bc4ba10677c1404768969390b0481a23 -U dan -Z d0a91d26d23415f8f905dbc22cf63e6b +P d648ddd93de039820f5abe064c7bc1318cd9d6b1 +R d6ad877a8d4dd88a7ffbb10ce1648f9a +U drh +Z fa2f98c7a26d464055290dbf8522fd2e diff --git a/manifest.uuid b/manifest.uuid index 43a83fe0dd..fac9e66268 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d648ddd93de039820f5abe064c7bc1318cd9d6b1 \ No newline at end of file +3d3df79bfaf9dbc7aa711c08a19d2f6dbe744b32 \ No newline at end of file diff --git a/src/build.c b/src/build.c index a5e838ccb9..2ea67447cc 100644 --- a/src/build.c +++ b/src/build.c @@ -2848,6 +2848,30 @@ Index *sqlite3AllocateIndexObject( return p; } +/* +** Backwards Compatibility Hack: +** +** Historical versions of SQLite accepted strings as column names in +** indexes and PRIMARY KEY constraints and in UNIQUE constraints. Example: +** +** CREATE TABLE xyz(a,b,c,d,e,PRIMARY KEY('a'),UNIQUE('b','c' COLLATE trim) +** CREATE INDEX abc ON xyz('c','d' DESC,'e' COLLATE nocase DESC); +** +** This is goofy. But to preserve backwards compatibility we continue to +** accept it. This routine does the necessary conversion. It converts +** the expression given in its argument from a TK_STRING into a TK_ID +** if the expression is just a TK_STRING with an optional COLLATE clause. +** If the epxression is anything other than TK_STRING, the expression is +** unchanged. +*/ +static void sqlite3StringToId(Expr *p){ + if( p->op==TK_STRING ){ + p->op = TK_ID; + }else if( p->op==TK_COLLATE && p->pLeft->op==TK_STRING ){ + p->pLeft->op = TK_ID; + } +} + /* ** Create a new index for an SQL table. pName1.pName2 is the name of the index ** and pTblList is the name of the table that is to be indexed. Both will @@ -3118,6 +3142,7 @@ Index *sqlite3CreateIndex( int requestedSortOrder; char *zColl; /* Collation sequence name */ + sqlite3StringToId(pListItem->pExpr); pCExpr = sqlite3ExprSkipCollate(pListItem->pExpr); if( pCExpr->op!=TK_ID ){ sqlite3ErrorMsg(pParse, "indexes on expressions not yet supported"); diff --git a/test/index2.test b/test/index2.test index 48d0c38e18..b1d7e227fb 100644 --- a/test/index2.test +++ b/test/index2.test @@ -1,4 +1,4 @@ -# 2005 January 11 +# 2005-01-11 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: @@ -11,7 +11,6 @@ # This file implements regression tests for SQLite library. The # focus of this file is testing the CREATE INDEX statement. # -# $Id: index2.test,v 1.3 2006/03/03 19:12:30 drh Exp $ set testdir [file dirname $argv0] source $testdir/tester.tcl diff --git a/test/index3.test b/test/index3.test index 0cdc6e088d..1d90de1b9b 100644 --- a/test/index3.test +++ b/test/index3.test @@ -1,4 +1,4 @@ -# 2005 February 14 +# 2005-02-14 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: @@ -11,7 +11,6 @@ # This file implements regression tests for SQLite library. The # focus of this file is testing the CREATE INDEX statement. # -# $Id: index3.test,v 1.3 2008/03/19 13:03:34 drh Exp $ set testdir [file dirname $argv0] @@ -40,17 +39,42 @@ do_test index3-1.3 { } {0 {}} integrity_check index3-1.4 +# Backwards compatibility test: +# +# Verify that CREATE INDEX statements that use strings instead of +# identifiers for the the column names continue to work correctly. +# This is undocumented behavior retained for backwards compatiblity. +# +do_execsql_test index3-2.1 { + DROP TABLE t1; + CREATE TABLE t1(a, b, c, d, e, + PRIMARY KEY('a'), UNIQUE('b' COLLATE nocase DESC)); + CREATE INDEX t1c ON t1('c'); + CREATE INDEX t1d ON t1('d' COLLATE binary ASC); + WITH RECURSIVE c(x) AS (VALUES(1) UNION SELECT x+1 FROM c WHERE x<30) + INSERT INTO t1(a,b,c,d,e) + SELECT x, printf('ab%03xxy',x), x, x, x FROM c; +} {} +do_execsql_test index3-2.2 { + SELECT a FROM t1 WHERE b='ab005xy' COLLATE nocase; +} {5} +do_execsql_test index3-2.2eqp { + EXPLAIN QUERY PLAN + SELECT a FROM t1 WHERE b='ab005xy' COLLATE nocase; +} {/USING INDEX/} + + # This test corrupts the database file so it must be the last test # in the series. # do_test index3-99.1 { execsql { PRAGMA writable_schema=on; - UPDATE sqlite_master SET sql='nonsense'; + UPDATE sqlite_master SET sql='nonsense' WHERE name='t1d' } db close catch { sqlite3 db test.db } - catchsql { DROP INDEX i1 } -} {1 {malformed database schema (t1)}} + catchsql { DROP INDEX t1c } +} {1 {malformed database schema (t1d)}} finish_test diff --git a/test/where.test b/test/where.test index 72fd696306..e94047564c 100644 --- a/test/where.test +++ b/test/where.test @@ -42,8 +42,8 @@ do_test where-1.0 { } execsql { - CREATE INDEX i1w ON t1(w); - CREATE INDEX i1xy ON t1(x,y); + CREATE INDEX i1w ON t1("w"); -- Verify quoted identifier names + CREATE INDEX i1xy ON t1(`x`,'y' ASC); -- Old MySQL compatibility CREATE INDEX i2p ON t2(p); CREATE INDEX i2r ON t2(r); CREATE INDEX i2qs ON t2(q, s); diff --git a/test/where4.test b/test/where4.test index 20c69771ad..3b24711514 100644 --- a/test/where4.test +++ b/test/where4.test @@ -136,7 +136,7 @@ do_test where4-3.1 { INSERT INTO t2 VALUES(1); INSERT INTO t2 VALUES(2); INSERT INTO t2 VALUES(3); - CREATE TABLE t3(x,y,UNIQUE(x,y)); + CREATE TABLE t3(x,y,UNIQUE("x",'y' ASC)); -- Goofy syntax allowed INSERT INTO t3 VALUES(1,11); INSERT INTO t3 VALUES(2,NULL); @@ -200,7 +200,8 @@ do_test where4-4.4 { ifcapable subquery { do_test where4-5.1 { execsql { - CREATE TABLE t4(x,y,z,PRIMARY KEY(x,y)); + -- Allow the 'x' syntax for backwards compatibility + CREATE TABLE t4(x,y,z,PRIMARY KEY('x' ASC, "y" ASC)); } execsql { SELECT * @@ -304,4 +305,3 @@ do_execsql_test 8.2 { SELECT * FROM u9 WHERE a IS $null } {{} 1 {} 2} finish_test - From 2547336dadb65cb5e67dd13f35ab893ed366a40b Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 4 Sep 2015 18:03:45 +0000 Subject: [PATCH 039/100] Fix over-length source code lines in Lemon. FossilOrigin-Name: 1efece95ff8777b89558be59277732ba2a68d5ab --- manifest | 13 ++++++------- manifest.uuid | 2 +- tool/lemon.c | 32 +++++++++++++++++++++++--------- 3 files changed, 30 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index 2fb2749921..a7ab65d3ef 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\ssupport\sfor\sCREATE\sINDEX\sstatements\sthat\suse\sdeterministic\sexpressions\nrather\sthan\sonly\scolumn\snames. -D 2015-09-04T17:32:19.351 +C Fix\sover-length\ssource\scode\slines\sin\sLemon. +D 2015-09-04T18:03:45.061 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1338,7 +1338,7 @@ F tool/fuzzershell.c f2fc86dd22df654b28851b85019d3bd007361751 F tool/genfkey.README cf68fddd4643bbe3ff8e31b8b6d8b0a1b85e20f4 F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce -F tool/lemon.c b9109f59b57e7b6f101c4fe644c8361ba6dee969 +F tool/lemon.c b12cb605725dabfa3a8607ec6bd9e978973c7af9 F tool/lempar.c 01ca97f87610d1dac6d8cd96ab109ab1130e76dc F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862 F tool/logest.c eef612f8adf4d0993dafed0416064cf50d5d33c6 @@ -1383,8 +1383,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 3d3df79bfaf9dbc7aa711c08a19d2f6dbe744b32 5ff855293865c244ac632c630e8e7e8d7c05a5f6 -R 46fafd1e16150312779d43bb23b81793 -T +closed 5ff855293865c244ac632c630e8e7e8d7c05a5f6 +P 2131a5ca53f0e9b0b98a9dd9a20e495d54d146a7 +R c504fcfc7ca364ef5b7aec55028c181c U drh -Z 4941f6208637da2261b21635bf7704d1 +Z 31da7673766f0dcc42d3887f78dbee2d diff --git a/manifest.uuid b/manifest.uuid index 8d3663c4f3..6ca3648ba2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2131a5ca53f0e9b0b98a9dd9a20e495d54d146a7 \ No newline at end of file +1efece95ff8777b89558be59277732ba2a68d5ab \ No newline at end of file diff --git a/tool/lemon.c b/tool/lemon.c index 89d992c37e..5967885021 100644 --- a/tool/lemon.c +++ b/tool/lemon.c @@ -55,7 +55,7 @@ static char *msort(char*,char**,int(*)(const char*,const char*)); ** saying they are unsafe. So we define our own versions of those routines too. ** ** There are three routines here: lemon_sprintf(), lemon_vsprintf(), and -** lemon_addtext(). The first two are replacements for sprintf() and vsprintf(). +** lemon_addtext(). The first two are replacements for sprintf() and vsprintf(). ** The third is a helper routine for vsnprintf() that adds texts to the end of a ** buffer, making sure the buffer is always zero-terminated. ** @@ -1375,14 +1375,16 @@ void Configlist_closure(struct lemon *lemp) /* Sort the configuration list */ void Configlist_sort(){ - current = (struct config *)msort((char *)current,(char **)&(current->next),Configcmp); + current = (struct config*)msort((char*)current,(char**)&(current->next), + Configcmp); currentend = 0; return; } /* Sort the basis configuration list */ void Configlist_sortbasis(){ - basis = (struct config *)msort((char *)current,(char **)&(current->bp),Configcmp); + basis = (struct config*)msort((char*)current,(char**)&(current->bp), + Configcmp); basisend = 0; return; } @@ -1613,7 +1615,8 @@ int main(int argc, char **argv) if( statistics ){ printf("Parser statistics: %d terminals, %d nonterminals, %d rules\n", lem.nterminal, lem.nsymbol - lem.nterminal, lem.nrule); - printf(" %d states, %d parser table entries, %d conflicts\n", + printf(" %d states, %d parser table entries," + " %d conflicts\n", lem.nstate, lem.tablesize, lem.nconflict); } if( lem.nconflict > 0 ){ @@ -1873,7 +1876,8 @@ static int handleswitch(int i, FILE *err) dv = strtod(cp,&end); if( *end ){ if( err ){ - fprintf(err,"%sillegal character in floating-point argument.\n",emsg); + fprintf(err, + "%sillegal character in floating-point argument.\n",emsg); errline(i,(int)((char*)end-(char*)argv[i]),err); } errcnt++; @@ -3228,7 +3232,8 @@ PRIVATE FILE *tplt_open(struct lemon *lemp) } in = fopen(user_templatename,"rb"); if( in==0 ){ - fprintf(stderr,"Can't open the template file \"%s\".\n",user_templatename); + fprintf(stderr,"Can't open the template file \"%s\".\n", + user_templatename); lemp->errorcnt++; return 0; } @@ -3313,7 +3318,10 @@ void emit_destructor_code( }else if( sp->destructor ){ cp = sp->destructor; fprintf(out,"{\n"); (*lineno)++; - if (!lemp->nolinenosflag) { (*lineno)++; tplt_linedir(out,sp->destLineno,lemp->filename); } + if( !lemp->nolinenosflag ){ + (*lineno)++; + tplt_linedir(out,sp->destLineno,lemp->filename); + } }else if( lemp->vardest ){ cp = lemp->vardest; if( cp==0 ) return; @@ -3510,13 +3518,19 @@ PRIVATE void emit_code( /* Generate code to do the reduce action */ if( rp->code ){ - if (!lemp->nolinenosflag) { (*lineno)++; tplt_linedir(out,rp->line,lemp->filename); } + if( !lemp->nolinenosflag ){ + (*lineno)++; + tplt_linedir(out,rp->line,lemp->filename); + } fprintf(out,"{%s",rp->code); for(cp=rp->code; *cp; cp++){ if( *cp=='\n' ) (*lineno)++; } /* End loop */ fprintf(out,"}\n"); (*lineno)++; - if (!lemp->nolinenosflag) { (*lineno)++; tplt_linedir(out,*lineno,lemp->outname); } + if( !lemp->nolinenosflag ){ + (*lineno)++; + tplt_linedir(out,*lineno,lemp->outname); + } } /* End if( rp->code ) */ return; From e3a7307e2dd806033b5f3959c7f2a3fd6c965f97 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 5 Sep 2015 19:07:08 +0000 Subject: [PATCH 040/100] Get STAT4 range scan estimates work again when the bounds are determined by date/time functions. FossilOrigin-Name: d2761357a0496ec1e590c7c9e397c5b5c904f91a --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbemem.c | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index a7ab65d3ef..434d78ec41 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sover-length\ssource\scode\slines\sin\sLemon. -D 2015-09-04T18:03:45.061 +C Get\sSTAT4\srange\sscan\sestimates\swork\sagain\swhen\sthe\sbounds\sare\sdetermined\nby\sdate/time\sfunctions. +D 2015-09-05T19:07:08.860 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -408,7 +408,7 @@ F src/vdbeInt.h 8b54e01ad0463590e7cffabce0bc36da9ee4f816 F src/vdbeapi.c bda74ef4b5103d7b4a4be36f936d3cf2b56a7d6f F src/vdbeaux.c fd00b489ab3f44f2dca1e4344faf289b7bfcf649 F src/vdbeblob.c 1d7b97115e7bbac4c318db416d2ca83fc779544a -F src/vdbemem.c ae38a0d35ae71cf604381a887c170466ba518090 +F src/vdbemem.c 19b3036aa4d676e7103b0fb5efd6327da455f915 F src/vdbesort.c f5009e7a35e3065635d8918b9a31f498a499976b F src/vdbetrace.c 8befe829faff6d9e6f6e4dee5a7d3f85cc85f1a0 F src/vtab.c 2ecfe020c10e0a0c7b078203fdba2fae844744bc @@ -1383,7 +1383,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 2131a5ca53f0e9b0b98a9dd9a20e495d54d146a7 -R c504fcfc7ca364ef5b7aec55028c181c +P 1efece95ff8777b89558be59277732ba2a68d5ab +R 33b468f201c817bccc2bf5f080039adb U drh -Z 31da7673766f0dcc42d3887f78dbee2d +Z 535e2ce68f49a8491abd9f8ccc62b768 diff --git a/manifest.uuid b/manifest.uuid index 6ca3648ba2..3ddd4c577d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1efece95ff8777b89558be59277732ba2a68d5ab \ No newline at end of file +d2761357a0496ec1e590c7c9e397c5b5c904f91a \ No newline at end of file diff --git a/src/vdbemem.c b/src/vdbemem.c index 648a53d2a3..28dd5d9572 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -1155,7 +1155,7 @@ static sqlite3_value *valueNew(sqlite3 *db, struct ValueNewStat4Ctx *p){ ** to be a scalar SQL function. If ** ** * all function arguments are SQL literals, -** * the SQLITE_FUNC_CONSTANT function flag is set, and +** * one of the SQLITE_FUNC_CONSTANT or _SLOCHNG function flags is set, and ** * the SQLITE_FUNC_NEEDCOLL function flag is not set, ** ** then this routine attempts to invoke the SQL function. Assuming no @@ -1196,7 +1196,7 @@ static int valueFromFunction( nName = sqlite3Strlen30(p->u.zToken); pFunc = sqlite3FindFunction(db, p->u.zToken, nName, nVal, enc, 0); assert( pFunc ); - if( (pFunc->funcFlags & SQLITE_FUNC_CONSTANT)==0 + if( (pFunc->funcFlags & (SQLITE_FUNC_CONSTANT|SQLITE_FUNC_SLOCHNG))==0 || (pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL) ){ return SQLITE_OK; From 28b9e0fc058c5e6c212109b58162fbe7d9f5a651 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 5 Sep 2015 19:21:00 +0000 Subject: [PATCH 041/100] Fix an unreachable conditional in the WHERE clause analysis logic. FossilOrigin-Name: 24924a58197e558a9e8800cc5c91dc8fb32f3557 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 3 ++- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 434d78ec41..f9b770eebc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Get\sSTAT4\srange\sscan\sestimates\swork\sagain\swhen\sthe\sbounds\sare\sdetermined\nby\sdate/time\sfunctions. -D 2015-09-05T19:07:08.860 +C Fix\san\sunreachable\sconditional\sin\sthe\sWHERE\sclause\sanalysis\slogic. +D 2015-09-05T19:21:00.671 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -416,7 +416,7 @@ F src/vxworks.h c18586c8edc1bddbc15c004fa16aeb1e1342b4fb F src/wal.c 8cd07f1f99e1a81346db1c9da879bef6c6f97cf6 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 2e14d17f592d176b6dc879c33fbdec4fbccaa2ba -F src/where.c e6efbb9e5f0c20943cd03909b3894ad5b105dedf +F src/where.c ed1cd1cb0434bca9f4a5379582c637bf393b34ac F src/whereInt.h 292d3ac90da4eab1e03ac8452f1add746bcafaa1 F src/wherecode.c 6ac8599523f4840d9efac335329f627ebf3f79fd F src/whereexpr.c 2473e4350e30f9b55d1c6a8f66ca23c689f23f1d @@ -1383,7 +1383,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 1efece95ff8777b89558be59277732ba2a68d5ab -R 33b468f201c817bccc2bf5f080039adb +P d2761357a0496ec1e590c7c9e397c5b5c904f91a +R 9921ff0c69ded6987711a9a840df906d U drh -Z 535e2ce68f49a8491abd9f8ccc62b768 +Z 612add4179403f74341dc4eb50c9c3d3 diff --git a/manifest.uuid b/manifest.uuid index 3ddd4c577d..4767aec925 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d2761357a0496ec1e590c7c9e397c5b5c904f91a \ No newline at end of file +24924a58197e558a9e8800cc5c91dc8fb32f3557 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 1ee31fef3b..a29988a8e0 100644 --- a/src/where.c +++ b/src/where.c @@ -191,9 +191,10 @@ static WhereTerm *whereScanNext(WhereScan *pScan){ ){ if( (pTerm->eOperator & WO_EQUIV)!=0 && pScan->nEquivaiCur) - && (pX = sqlite3ExprSkipCollate(pTerm->pExpr->pRight))->op==TK_COLUMN ){ int j; + pX = sqlite3ExprSkipCollate(pTerm->pExpr->pRight); + assert( pX->op==TK_COLUMN ); for(j=0; jnEquiv; j++){ if( pScan->aiCur[j]==pX->iTable && pScan->aiColumn[j]==pX->iColumn ){ From e386a1ba25dcb607fd740619c8f14eb4257ecb63 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 5 Sep 2015 19:52:08 +0000 Subject: [PATCH 042/100] Experiment with a different fts5 leaf page format that allows faster seeks. FossilOrigin-Name: a1f4c3b543eed84e808f6b901a38179786fffe16 --- ext/fts5/fts5_index.c | 328 +++++++++++++++++++++++++--------- ext/fts5/test/fts5aa.test | 7 +- ext/fts5/test/fts5ad.test | 3 + ext/fts5/test/fts5simple.test | 74 ++++++++ ext/fts5/tool/loadfts5.tcl | 1 + manifest | 26 +-- manifest.uuid | 2 +- test/permutations.test | 2 +- 8 files changed, 341 insertions(+), 102 deletions(-) create mode 100644 ext/fts5/test/fts5simple.test diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 5bf4feba93..7fa5a0fae4 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -303,7 +303,8 @@ typedef struct Fts5StructureSegment Fts5StructureSegment; struct Fts5Data { u8 *p; /* Pointer to buffer containing record */ - int n; /* Size of record in bytes */ + int nn; /* Size of record in bytes */ + int szLeaf; /* Size of leaf without page-index */ }; /* @@ -377,7 +378,8 @@ struct Fts5Structure { */ struct Fts5PageWriter { int pgno; /* Page number for this page */ - Fts5Buffer buf; /* Buffer containing page data */ + Fts5Buffer buf; /* Buffer containing leaf data */ + Fts5Buffer pgidx; /* Buffer containing page-index */ Fts5Buffer term; /* Buffer containing previous term on page */ }; struct Fts5DlidxWriter { @@ -392,6 +394,7 @@ struct Fts5SegWriter { i64 iPrevRowid; /* Previous rowid written to current leaf */ u8 bFirstRowidInDoclist; /* True if next rowid is first in doclist */ u8 bFirstRowidInPage; /* True if next rowid is first in page */ + /* TODO1: Can use (writer.pgidx.n==0) instead of bFirstTermInPage */ u8 bFirstTermInPage; /* True if next term will be first in leaf */ int nLeafWritten; /* Number of leaf pages written */ int nEmpty; /* Number of contiguous term-less nodes */ @@ -500,10 +503,27 @@ struct Fts5SegIter { int bDel; /* True if the delete flag is set */ }; +/* +** Argument is a pointer to an Fts5Data structure that contains a +** leaf page. +*/ +#define ASSERT_SZLEAF_OK(x) assert( \ + (x)->szLeaf==fts5GetU16(&(x)->p[2]) || (x)->szLeaf==(x)->nn \ +) + #define FTS5_SEGITER_ONETERM 0x01 #define FTS5_SEGITER_REVERSE 0x02 +/* +** Argument is a pointer to an Fts5Data structure that contains a leaf +** page. This macro evaluates to true if the leaf contains no terms, or +** false if it contains at least one term. +*/ +#define fts5LeafIsTermless(x) ((x)->szLeaf >= (x)->nn) + +#define fts5LeafFirstTermOff(x) (fts5GetU16(&x->p[(x)->szLeaf])) + /* ** poslist: ** Used by sqlite3Fts5IterPoslist() when the poslist needs to be buffered. @@ -679,7 +699,7 @@ static Fts5Data *fts5DataRead(Fts5Index *p, i64 iRowid){ int nAlloc = sizeof(Fts5Data) + nByte + FTS5_DATA_PADDING; pRet = (Fts5Data*)sqlite3_malloc(nAlloc); if( pRet ){ - pRet->n = nByte; + pRet->nn = nByte; aOut = pRet->p = (u8*)&pRet[1]; }else{ rc = SQLITE_NOMEM; @@ -691,6 +711,9 @@ static Fts5Data *fts5DataRead(Fts5Index *p, i64 iRowid){ if( rc!=SQLITE_OK ){ sqlite3_free(pRet); pRet = 0; + }else{ + /* TODO1: Fix this */ + pRet->szLeaf = fts5GetU16(&pRet->p[2]); } } p->rc = rc; @@ -974,8 +997,9 @@ static Fts5Structure *fts5StructureRead(Fts5Index *p){ pData = fts5DataRead(p, FTS5_STRUCTURE_ROWID); if( p->rc ) return 0; - memset(&pData->p[pData->n], 0, FTS5_DATA_PADDING); - p->rc = fts5StructureDecode(pData->p, pData->n, &iCookie, &pRet); + /* TODO: Do we need this if the leaf-index is appended? Probably... */ + memset(&pData->p[pData->nn], 0, FTS5_DATA_PADDING); + p->rc = fts5StructureDecode(pData->p, pData->nn, &iCookie, &pRet); if( p->rc==SQLITE_OK && pConfig->iCookie!=iCookie ){ p->rc = sqlite3Fts5ConfigLoad(pConfig, iCookie); } @@ -1178,11 +1202,11 @@ static int fts5DlidxLvlNext(Fts5DlidxLvl *pLvl){ pLvl->iFirstOff = pLvl->iOff; }else{ int iOff; - for(iOff=pLvl->iOff; iOffn; iOff++){ + for(iOff=pLvl->iOff; iOffnn; iOff++){ if( pData->p[iOff] ) break; } - if( iOffn ){ + if( iOffnn ){ i64 iVal; pLvl->iLeafPgno += (iOff - pLvl->iOff) + 1; iOff += fts5GetVarint(&pData->p[iOff], (u64*)&iVal); @@ -1470,7 +1494,8 @@ static int fts5GetPoslistSize(const u8 *p, int *pnSz, int *pbDel){ static void fts5SegIterLoadNPos(Fts5Index *p, Fts5SegIter *pIter){ if( p->rc==SQLITE_OK ){ int iOff = pIter->iLeafOffset; /* Offset to read at */ - if( iOff>=pIter->pLeaf->n ){ + ASSERT_SZLEAF_OK(pIter->pLeaf); + if( iOff>=pIter->pLeaf->szLeaf ){ p->rc = FTS5_CORRUPT; }else{ const u8 *a = &pIter->pLeaf->p[iOff]; @@ -1483,7 +1508,8 @@ static void fts5SegIterLoadRowid(Fts5Index *p, Fts5SegIter *pIter){ u8 *a = pIter->pLeaf->p; /* Buffer to read data from */ int iOff = pIter->iLeafOffset; - if( iOff>=pIter->pLeaf->n ){ + ASSERT_SZLEAF_OK(pIter->pLeaf); + if( iOff>=pIter->pLeaf->szLeaf ){ fts5SegIterNextPage(p, pIter); if( pIter->pLeaf==0 ){ if( p->rc==SQLITE_OK ) p->rc = FTS5_CORRUPT; @@ -1559,7 +1585,8 @@ static void fts5SegIterInit( if( p->rc==SQLITE_OK ){ u8 *a = pIter->pLeaf->p; - pIter->iLeafOffset = fts5GetU16(&a[2]); + pIter->iLeafOffset = fts5GetU16(&a[pIter->pLeaf->szLeaf]); + assert( pIter->iLeafOffset==4 ); fts5SegIterLoadTerm(p, pIter, 0); fts5SegIterLoadNPos(p, pIter); } @@ -1581,11 +1608,12 @@ static void fts5SegIterInit( ** byte of the position list content associated with said rowid. */ static void fts5SegIterReverseInitPage(Fts5Index *p, Fts5SegIter *pIter){ - int n = pIter->pLeaf->n; + int n = pIter->pLeaf->szLeaf; int i = pIter->iLeafOffset; u8 *a = pIter->pLeaf->p; int iRowidOffset = 0; + ASSERT_SZLEAF_OK(pIter->pLeaf); while( 1 ){ i64 iDelta = 0; int nPos; @@ -1633,7 +1661,7 @@ static void fts5SegIterReverseNewPage(Fts5Index *p, Fts5SegIter *pIter){ )); if( pNew ){ if( pIter->iLeafPgno==pIter->iTermLeafPgno ){ - if( pIter->iTermLeafOffsetn ){ + if( pIter->iTermLeafOffsetszLeaf ){ pIter->pLeaf = pNew; pIter->iLeafOffset = pIter->iTermLeafOffset; } @@ -1712,8 +1740,9 @@ static void fts5SegIterNext( /* Search for the end of the position list within the current page. */ u8 *a = pLeaf->p; - int n = pLeaf->n; + int n = pLeaf->szLeaf; + ASSERT_SZLEAF_OK(pLeaf); iOff = pIter->iLeafOffset + pIter->nPos; if( iOff=n ){ fts5SegIterNextPage(p, pIter); pIter->iLeafOffset = 4; - }else if( iOff!=fts5GetU16(&a[2]) ){ + }else if( iOff!=fts5LeafFirstTermOff(pLeaf) ){ pIter->iLeafOffset += fts5GetVarint32(&a[iOff], nKeep); } }else{ @@ -1745,7 +1774,8 @@ static void fts5SegIterNext( pIter->pLeaf = 0; }else{ pIter->pLeaf->p = (u8*)pList; - pIter->pLeaf->n = nList; + pIter->pLeaf->nn = nList; + pIter->pLeaf->szLeaf = nList; sqlite3Fts5BufferSet(&p->rc, &pIter->term, strlen(zTerm), (u8*)zTerm); pIter->iLeafOffset = fts5GetVarint(pList, (u64*)&pIter->iRowid); } @@ -1756,15 +1786,17 @@ static void fts5SegIterNext( fts5SegIterNextPage(p, pIter); pLeaf = pIter->pLeaf; if( pLeaf==0 ) break; - if( (iOff = fts5GetU16(&pLeaf->p[0])) && iOffn ){ + ASSERT_SZLEAF_OK(pLeaf); + if( (iOff = fts5GetU16(&pLeaf->p[0])) && iOffszLeaf ){ iOff += sqlite3Fts5GetVarint(&pLeaf->p[iOff], (u64*)&pIter->iRowid); pIter->iLeafOffset = iOff; } - else if( (iOff = fts5GetU16(&pLeaf->p[2])) ){ + else if( pLeaf->nn>pLeaf->szLeaf ){ + iOff = fts5GetU16(&pLeaf->p[pLeaf->szLeaf]); pIter->iLeafOffset = iOff; bNewTerm = 1; } - if( iOff>=pLeaf->n ){ + if( iOff>=pLeaf->szLeaf ){ p->rc = FTS5_CORRUPT; return; } @@ -1819,7 +1851,7 @@ static void fts5SegIterReverse(Fts5Index *p, Fts5SegIter *pIter){ /* Search for a new term within the current leaf. If one can be found, ** then this page contains the largest rowid for the current term. */ - while( iOffn ){ + while( iOffszLeaf ){ int nPos; i64 iDelta; int bDummy; @@ -1827,7 +1859,7 @@ static void fts5SegIterReverse(Fts5Index *p, Fts5SegIter *pIter){ /* Read the position-list size field */ iOff += fts5GetPoslistSize(&pLeaf->p[iOff], &nPos, &bDummy); iOff += nPos; - if( iOff>=pLeaf->n ) break; + if( iOff>=pLeaf->szLeaf ) break; /* Rowid delta. Or, if 0x00, the end of doclist marker. */ nPos = fts5GetVarint(&pLeaf->p[iOff], (u64*)&iDelta); @@ -1838,7 +1870,7 @@ static void fts5SegIterReverse(Fts5Index *p, Fts5SegIter *pIter){ /* If this condition is true then the largest rowid for the current ** term may not be stored on the current page. So search forward to ** see where said rowid really is. */ - if( iOff>=pLeaf->n ){ + if( iOff>=pLeaf->szLeaf ){ int pgno; Fts5StructureSegment *pSeg = pIter->pSeg; @@ -1848,14 +1880,15 @@ static void fts5SegIterReverse(Fts5Index *p, Fts5SegIter *pIter){ i64 iAbs = FTS5_SEGMENT_ROWID(pSeg->iSegid, 0, pgno); Fts5Data *pNew = fts5DataRead(p, iAbs); if( pNew ){ - int iRowid, iTerm; - fts5LeafHeader(pNew, &iRowid, &iTerm); + int iRowid, bTermless; + iRowid = fts5GetU16(pNew->p); + bTermless = fts5LeafIsTermless(pNew); if( iRowid ){ SWAPVAL(Fts5Data*, pNew, pLast); pgnoLast = pgno; } fts5DataRelease(pNew); - if( iTerm ) break; + if( bTermless==0 ) break; } } } @@ -1903,7 +1936,7 @@ static void fts5SegIterLoadDlidx(Fts5Index *p, Fts5SegIter *pIter){ ** term. */ if( pIter->iTermLeafPgno==pIter->iLeafPgno ){ int iOff = pIter->iLeafOffset + pIter->nPos; - while( iOffn ){ + while( iOffszLeaf ){ int bDummy; int nPos; i64 iDelta; @@ -1911,7 +1944,7 @@ static void fts5SegIterLoadDlidx(Fts5Index *p, Fts5SegIter *pIter){ /* iOff is currently the offset of the start of position list data */ iOff += fts5GetVarint(&pLeaf->p[iOff], (u64*)&iDelta); if( iDelta==0 ) return; - assert_nc( iOffn ); + assert_nc( iOffszLeaf ); iOff += fts5GetPoslistSize(&pLeaf->p[iOff], &nPos, &bDummy); iOff += nPos; } @@ -1955,16 +1988,18 @@ static void fts5LeafSeek( ){ int iOff; const u8 *a = pIter->pLeaf->p; - int n = pIter->pLeaf->n; + int n = pIter->pLeaf->szLeaf; int nMatch = 0; int nKeep = 0; int nNew = 0; + int iTerm = 0; + int nPgTerm = (pIter->pLeaf->nn - pIter->pLeaf->szLeaf) >> 1; assert( p->rc==SQLITE_OK ); assert( pIter->pLeaf ); - iOff = fts5GetU16(&a[2]); + iOff = fts5GetU16(&a[n]); if( iOff<4 || iOff>=n ){ p->rc = FTS5_CORRUPT; return; @@ -2001,6 +2036,7 @@ static void fts5LeafSeek( } iOff += nNew; +#if 0 /* Skip past the doclist. If the end of the page is reached, bail out. */ while( 1 ){ int nPos; @@ -2023,6 +2059,19 @@ static void fts5LeafSeek( } }; + iTerm++; + assert( iTerm=nPgTerm ){ + iOff = n; + break; + } + iOff = fts5GetU16(&a[n + iTerm*2]); +#endif + /* Read the nKeep field of the next term. */ fts5IndexGetVarint32(a, iOff, nKeep); } @@ -2037,9 +2086,9 @@ static void fts5LeafSeek( fts5SegIterNextPage(p, pIter); if( pIter->pLeaf==0 ) return; a = pIter->pLeaf->p; - iOff = fts5GetU16(&a[2]); - if( iOff ){ - if( iOff<4 || iOff>=n ){ + if( fts5LeafIsTermless(pIter->pLeaf)==0 ){ + iOff = fts5LeafFirstTermOff(pIter->pLeaf); + if( iOff<4 || iOff>=pIter->pLeaf->szLeaf ){ p->rc = FTS5_CORRUPT; }else{ nKeep = 0; @@ -2190,7 +2239,7 @@ static void fts5SegIterHashInit( pLeaf = fts5IdxMalloc(p, sizeof(Fts5Data)); if( pLeaf==0 ) return; pLeaf->p = (u8*)pList; - pLeaf->n = nList; + pLeaf->nn = pLeaf->szLeaf = nList; pIter->pLeaf = pLeaf; pIter->iLeafOffset = fts5GetVarint(pLeaf->p, (u64*)&pIter->iRowid); @@ -2383,7 +2432,7 @@ static void fts5SegIterGotoPage( if( p->rc==SQLITE_OK ){ int iOff; u8 *a = pIter->pLeaf->p; - int n = pIter->pLeaf->n; + int n = pIter->pLeaf->szLeaf; iOff = fts5GetU16(&a[0]); if( iOff<4 || iOff>=n ){ @@ -2717,7 +2766,7 @@ static void fts5MultiIterNew2( Fts5SegIter *pIter = &pNew->aSeg[1]; pIter->flags = FTS5_SEGITER_ONETERM; - if( pData->n>0 ){ + if( pData->szLeaf>0 ){ pIter->pLeaf = pData; pIter->iLeafOffset = fts5GetVarint(pData->p, (u64*)&pIter->iRowid); pNew->aFirst[1].iFirst = 1; @@ -2797,7 +2846,7 @@ static void fts5ChunkIterate( int nRem = pSeg->nPos; /* Number of bytes still to come */ Fts5Data *pData = 0; u8 *pChunk = &pSeg->pLeaf->p[pSeg->iLeafOffset]; - int nChunk = MIN(nRem, pSeg->pLeaf->n - pSeg->iLeafOffset); + int nChunk = MIN(nRem, pSeg->pLeaf->szLeaf - pSeg->iLeafOffset); int pgno = pSeg->iLeafPgno; int pgnoSave = 0; @@ -2816,7 +2865,7 @@ static void fts5ChunkIterate( pData = fts5DataRead(p, FTS5_SEGMENT_ROWID(pSeg->pSeg->iSegid, 0, pgno)); if( pData==0 ) break; pChunk = &pData->p[4]; - nChunk = MIN(nRem, pData->n - 4); + nChunk = MIN(nRem, pData->szLeaf - 4); if( pgno==pgnoSave ){ assert( pSeg->pNextLeaf==0 ); pSeg->pNextLeaf = pData; @@ -3102,18 +3151,28 @@ static void fts5WriteFlushLeaf(Fts5Index *p, Fts5SegWriter *pWriter){ Fts5PageWriter *pPage = &pWriter->writer; i64 iRowid; + assert( (pPage->pgidx.n==0)==(pWriter->bFirstTermInPage) ); + + /* Set the szLeaf header field. */ + assert( 0==fts5GetU16(&pPage->buf.p[2]) ); + fts5PutU16(&pPage->buf.p[2], pPage->buf.n); + if( pWriter->bFirstTermInPage ){ /* No term was written to this page. */ - assert( 0==fts5GetU16(&pPage->buf.p[2]) ); + assert( pPage->pgidx.n==0 ); fts5WriteBtreeNoTerm(p, pWriter); + }else{ + /* Append the pgidx to the page buffer. Set the szLeaf header field. */ + fts5BufferAppendBlob(&p->rc, &pPage->buf, pPage->pgidx.n, pPage->pgidx.p); } - /* Write the current page to the db. */ + /* Write the page out to disk */ iRowid = FTS5_SEGMENT_ROWID(pWriter->iSegid, 0, pPage->pgno); fts5DataWrite(p, iRowid, pPage->buf.p, pPage->buf.n); /* Initialize the next page. */ fts5BufferZero(&pPage->buf); + fts5BufferZero(&pPage->pgidx); fts5BufferAppendBlob(&p->rc, &pPage->buf, 4, zero); pPage->pgno++; @@ -3139,6 +3198,7 @@ static void fts5WriteAppendTerm( ){ int nPrefix; /* Bytes of prefix compression for term */ Fts5PageWriter *pPage = &pWriter->writer; + Fts5Buffer *pPgidx = &pWriter->writer.pgidx; assert( pPage->buf.n==0 || pPage->buf.n>4 ); if( pPage->buf.n==0 ){ @@ -3149,10 +3209,16 @@ static void fts5WriteAppendTerm( } if( p->rc ) return; + /* TODO1: Can this be consolidated with FlushOneHash version? */ + fts5PutU16(&pPgidx->p[pPgidx->n], pPage->buf.n); + pPgidx->n += 2; + if( pWriter->bFirstTermInPage ){ /* Update the "first term" field of the page header. */ +#if 0 assert( pPage->buf.p[2]==0 && pPage->buf.p[3]==0 ); fts5PutU16(&pPage->buf.p[2], pPage->buf.n); +#endif nPrefix = 0; if( pPage->pgno!=1 ){ /* This is the first term on a leaf that is not the leftmost leaf in @@ -3196,7 +3262,7 @@ static void fts5WriteAppendTerm( pWriter->aDlidx[0].pgno = pPage->pgno; /* If the current leaf page is full, flush it to disk. */ - if( pPage->buf.n>=p->pConfig->pgsz ){ + if( (pPage->buf.n + pPage->pgidx.n)>=p->pConfig->pgsz ){ fts5WriteFlushLeaf(p, pWriter); } } @@ -3234,7 +3300,7 @@ static void fts5WriteAppendRowid( fts5BufferAppendVarint(&p->rc, &pPage->buf, nPos); - if( pPage->buf.n>=p->pConfig->pgsz ){ + if( (pPage->buf.n + pPage->pgidx.n)>=p->pConfig->pgsz ){ fts5WriteFlushLeaf(p, pWriter); } } @@ -3251,8 +3317,10 @@ static void fts5WriteAppendPoslistData( int n = nData; assert( p->pConfig->pgsz>0 ); - while( p->rc==SQLITE_OK && (pPage->buf.n + n)>=p->pConfig->pgsz ){ - int nReq = p->pConfig->pgsz - pPage->buf.n; + while( p->rc==SQLITE_OK + && (pPage->buf.n + pPage->pgidx.n + n)>=p->pConfig->pgsz + ){ + int nReq = p->pConfig->pgsz - pPage->buf.n - pPage->pgidx.n; int nCopy = 0; while( nCopyterm); fts5BufferFree(&pLeaf->buf); + fts5BufferFree(&pLeaf->pgidx); fts5BufferFree(&pWriter->btterm); for(i=0; inDlidx; i++){ @@ -3321,6 +3390,7 @@ static void fts5WriteInit( pWriter->bFirstTermInPage = 1; pWriter->iBtPage = 1; + fts5BufferGrow(&p->rc, &pWriter->writer.pgidx, p->pConfig->pgsz + 20); if( p->pIdxWriter==0 ){ Fts5Config *pConfig = p->pConfig; fts5IndexPrepareStmt(p, &p->pIdxWriter, sqlite3_mprintf( @@ -3334,6 +3404,51 @@ static void fts5WriteInit( } } +/* +** The buffer passed as the second argument contains a leaf page that is +** missing its page-idx array. The first term is guaranteed to start at +** byte offset 4 of the buffer. The szLeaf field of the leaf page header +** is already populated. +** +** This function appends a page-index to the buffer. The buffer is +** guaranteed to be large enough to fit the page-index. +*/ +static void fts5MakePageidx(Fts5Index *p, Fts5Buffer *pBuf){ + if( p->rc==SQLITE_OK ){ + u8 *a = pBuf->p; + int szLeaf = pBuf->n; + int iOff = 4; + int nTerm; + + fts5PutU16(&pBuf->p[pBuf->n], iOff); + pBuf->n += 2; + fts5IndexGetVarint32(a, iOff, nTerm); + iOff += nTerm; + + while( iOff=szLeaf ) break; + + /* Skip past position list */ + fts5IndexGetVarint32(a, iOff, nTerm); + iOff += (nTerm >> 1); + + if( iOff>=(szLeaf-2) ) break; + + /* If this is the end of the doclist, break out of the loop */ + if( a[iOff]==0x00 ){ + iOff++; + fts5PutU16(&pBuf->p[pBuf->n], iOff); + pBuf->n += 2; + fts5IndexGetVarint32(a, iOff, nTerm); + fts5IndexGetVarint32(a, iOff, nTerm); + iOff += nTerm; + } + } + } +} + /* ** Iterator pIter was used to iterate through the input segments of on an ** incremental merge operation. This function is called if the incremental @@ -3358,16 +3473,25 @@ static void fts5TrimSegments(Fts5Index *p, Fts5IndexIter *pIter){ i64 iLeafRowid; Fts5Data *pData; int iId = pSeg->pSeg->iSegid; - u8 aHdr[4] = {0x00, 0x00, 0x00, 0x04}; + u8 aHdr[4] = {0x00, 0x00, 0x00, 0x00}; iLeafRowid = FTS5_SEGMENT_ROWID(iId, 0, pSeg->iTermLeafPgno); pData = fts5DataRead(p, iLeafRowid); if( pData ){ fts5BufferZero(&buf); + fts5BufferGrow(&p->rc, &buf, pData->nn); fts5BufferAppendBlob(&p->rc, &buf, sizeof(aHdr), aHdr); fts5BufferAppendVarint(&p->rc, &buf, pSeg->term.n); fts5BufferAppendBlob(&p->rc, &buf, pSeg->term.n, pSeg->term.p); - fts5BufferAppendBlob(&p->rc, &buf, pData->n - iOff, &pData->p[iOff]); + fts5BufferAppendBlob(&p->rc, &buf, pData->szLeaf-iOff, &pData->p[iOff]); + if( p->rc==SQLITE_OK ){ + /* Set the szLeaf field */ + fts5PutU16(&buf.p[2], buf.n); + } + + /* Set up the new page-index array */ + fts5MakePageidx(p, &buf); + fts5DataRelease(pData); pSeg->pSeg->pgnoFirst = pSeg->iTermLeafPgno; fts5DataDelete(p, FTS5_SEGMENT_ROWID(iId, 0, 1), iLeafRowid); @@ -3679,6 +3803,7 @@ static void fts5FlushOneHash(Fts5Index *p){ Fts5StructureSegment *pSeg; /* New segment within pStruct */ int nHeight; /* Height of new segment b-tree */ Fts5Buffer *pBuf; /* Buffer in which to assemble leaf page */ + Fts5Buffer *pPgidx; /* Buffer in which to assemble pgidx */ const u8 *zPrev = 0; Fts5SegWriter writer; @@ -3688,6 +3813,7 @@ static void fts5FlushOneHash(Fts5Index *p){ ** page size. */ assert( pgsz>0 ); pBuf = &writer.writer.buf; + pPgidx = &writer.writer.pgidx; fts5BufferGrow(&p->rc, pBuf, pgsz + 20); /* Begin scanning through hash table entries. This loop runs once for each @@ -3707,11 +3833,11 @@ static void fts5FlushOneHash(Fts5Index *p){ sqlite3Fts5HashScanEntry(pHash, &zTerm, &pDoclist, &nDoclist); nTerm = strlen(zTerm); - /* Decide if the term will fit on the current leaf. If it will not, - ** flush the leaf to disk here. */ - if( pBuf->n>4 && (pBuf->n + nTerm + 2) > pgsz ){ + /* Decide if the term will fit on the current leaf. If it will not, + ** flush the leaf to disk here. + ** TODO1: Is this calculation still correct? */ + if( pBuf->n>4 && (pBuf->n + nTerm + 2 + pPgidx->n + 2) > pgsz ){ fts5WriteFlushLeaf(p, &writer); - pBuf = &writer.writer.buf; if( (nTerm + 32) > pBuf->nSpace ){ fts5BufferGrow(&p->rc, pBuf, nTerm + 32 - pBuf->n); if( p->rc ) break; @@ -3721,12 +3847,15 @@ static void fts5FlushOneHash(Fts5Index *p){ /* Write the term to the leaf. And if it is the first on the leaf, and ** the leaf is not page number 1, push it up into the b-tree hierarchy ** as well. */ + + /* TODO1: Writing pgidx here! */ + fts5PutU16(&pPgidx->p[pPgidx->n], pBuf->n); + pPgidx->n += 2; if( writer.bFirstTermInPage==0 ){ int nPre = fts5PrefixCompress(nTerm, zPrev, nTerm, (const u8*)zTerm); pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], nPre); nSuffix = nTerm - nPre; }else{ - fts5PutU16(&pBuf->p[2], pBuf->n); writer.bFirstTermInPage = 0; if( writer.writer.pgno!=1 ){ int nPre = fts5PrefixCompress(nTerm, zPrev, nTerm, (const u8*)zTerm); @@ -3745,7 +3874,7 @@ static void fts5FlushOneHash(Fts5Index *p){ assert( writer.nDlidx>0 && writer.aDlidx[0].buf.n==0 ); writer.aDlidx[0].pgno = writer.writer.pgno; - if( pgsz>=(pBuf->n + nDoclist + 1) ){ + if( pgsz>=(pBuf->n + pPgidx->n + nDoclist + 1) ){ /* The entire doclist will fit on the current leaf. */ fts5BufferSafeAppendBlob(pBuf, pDoclist, nDoclist); }else{ @@ -3777,7 +3906,7 @@ static void fts5FlushOneHash(Fts5Index *p){ } assert( pBuf->n<=pBuf->nSpace ); - if( (pBuf->n + nCopy) <= pgsz ){ + if( (pBuf->n + pPgidx->n + nCopy) <= pgsz ){ /* The entire poslist will fit on the current leaf. So copy ** it in one go. */ fts5BufferSafeAppendBlob(pBuf, &pDoclist[iOff], nCopy); @@ -3788,7 +3917,7 @@ static void fts5FlushOneHash(Fts5Index *p){ const u8 *pPoslist = &pDoclist[iOff]; int iPos = 0; while( p->rc==SQLITE_OK ){ - int nSpace = pgsz - pBuf->n; + int nSpace = pgsz - pBuf->n - pPgidx->n; int n = 0; if( (nCopy - iPos)<=nSpace ){ n = nCopy - iPos; @@ -3798,9 +3927,8 @@ static void fts5FlushOneHash(Fts5Index *p){ assert( n>0 ); fts5BufferSafeAppendBlob(pBuf, &pPoslist[iPos], n); iPos += n; - if( pBuf->n>=pgsz ){ + if( (pBuf->n + pPgidx->n)>=pgsz ){ fts5WriteFlushLeaf(p, &writer); - pBuf = &writer.writer.buf; } if( iPos>=nCopy ) break; } @@ -4163,7 +4291,7 @@ static void fts5SetupPrefixIter( pData = fts5IdxMalloc(p, sizeof(Fts5Data) + doclist.n); if( pData ){ pData->p = (u8*)&pData[1]; - pData->n = doclist.n; + pData->nn = pData->szLeaf = doclist.n; memcpy(pData->p, doclist.p, doclist.n); fts5MultiIterNew2(p, pData, bDesc, ppIter); } @@ -4393,6 +4521,11 @@ int sqlite3Fts5IndexQuery( memcpy(&buf.p[1], pToken, nToken); #ifdef SQLITE_DEBUG + /* If the QUERY_TEST_NOIDX flag was specified, then this must be a + ** prefix-query. Instead of using a prefix-index (if one exists), + ** evaluate the prefix query using the main FTS index. This is used + ** for internal sanity checking by the integrity-check in debug + ** mode only. */ if( flags & FTS5INDEX_QUERY_TEST_NOIDX ){ assert( flags & FTS5INDEX_QUERY_PREFIX ); iIdx = 1+pConfig->nPrefix; @@ -4513,7 +4646,7 @@ int sqlite3Fts5IterPoslist( assert( pIter->pIndex->rc==SQLITE_OK ); *piRowid = pSeg->iRowid; *pn = pSeg->nPos; - if( pSeg->iLeafOffset+pSeg->nPos <= pSeg->pLeaf->n ){ + if( pSeg->iLeafOffset+pSeg->nPos <= pSeg->pLeaf->szLeaf ){ *pp = &pSeg->pLeaf->p[pSeg->iLeafOffset]; }else{ fts5BufferZero(&pIter->poslist); @@ -4561,11 +4694,11 @@ int sqlite3Fts5IndexGetAverages(Fts5Index *p, i64 *pnRow, i64 *anSize){ *pnRow = 0; memset(anSize, 0, sizeof(i64) * nCol); pData = fts5DataRead(p, FTS5_AVERAGES_ROWID); - if( p->rc==SQLITE_OK && pData->n ){ + if( p->rc==SQLITE_OK && pData->nn ){ int i = 0; int iCol; i += fts5GetVarint(&pData->p[i], (u64*)pnRow); - for(iCol=0; in && iColnn && iColp[i], (u64*)&anSize[iCol]); } } @@ -4770,18 +4903,25 @@ static void fts5TestTerm( if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT; /* If this is a prefix query, check that the results returned if the - ** the index is disabled are the same. In both ASC and DESC order. */ - if( iIdx>0 && rc==SQLITE_OK ){ - int f = flags|FTS5INDEX_QUERY_TEST_NOIDX; - ck2 = 0; - rc = fts5QueryCksum(p, iIdx, zTerm, nTerm, f, &ck2); - if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT; - } - if( iIdx>0 && rc==SQLITE_OK ){ - int f = flags|FTS5INDEX_QUERY_TEST_NOIDX|FTS5INDEX_QUERY_DESC; - ck2 = 0; - rc = fts5QueryCksum(p, iIdx, zTerm, nTerm, f, &ck2); - if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT; + ** the index is disabled are the same. In both ASC and DESC order. + ** + ** This check may only be performed if the hash table is empty. This + ** is because the hash table only supports a single scan query at + ** a time, and the multi-iter loop from which this function is called + ** is already performing such a scan. */ + if( p->nPendingData==0 ){ + if( iIdx>0 && rc==SQLITE_OK ){ + int f = flags|FTS5INDEX_QUERY_TEST_NOIDX; + ck2 = 0; + rc = fts5QueryCksum(p, iIdx, zTerm, nTerm, f, &ck2); + if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT; + } + if( iIdx>0 && rc==SQLITE_OK ){ + int f = flags|FTS5INDEX_QUERY_TEST_NOIDX|FTS5INDEX_QUERY_DESC; + ck2 = 0; + rc = fts5QueryCksum(p, iIdx, zTerm, nTerm, f, &ck2); + if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT; + } } cksum3 ^= ck1; @@ -4822,7 +4962,7 @@ static void fts5IndexIntegrityCheckEmpty( for(i=iFirst; p->rc==SQLITE_OK && i<=iLast; i++){ Fts5Data *pLeaf = fts5DataRead(p, FTS5_SEGMENT_ROWID(pSeg->iSegid, 0, i)); if( pLeaf ){ - if( 0!=fts5GetU16(&pLeaf->p[2]) ) p->rc = FTS5_CORRUPT; + if( !fts5LeafIsTermless(pLeaf) ) p->rc = FTS5_CORRUPT; if( i>=iNoRowid && 0!=fts5GetU16(&pLeaf->p[0]) ) p->rc = FTS5_CORRUPT; } fts5DataRelease(pLeaf); @@ -4851,7 +4991,6 @@ static void fts5IndexIntegrityCheckSegment( while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ i64 iRow; /* Rowid for this leaf */ Fts5Data *pLeaf; /* Data for this leaf */ - int iOff; /* Offset of first term on leaf */ int nIdxTerm = sqlite3_column_bytes(pStmt, 1); const char *zIdxTerm = (const char*)sqlite3_column_text(pStmt, 1); @@ -4869,14 +5008,15 @@ static void fts5IndexIntegrityCheckSegment( ** to or larger than the split-key in zIdxTerm. Also check that if there ** is also a rowid pointer within the leaf page header, it points to a ** location before the term. */ - iOff = fts5GetU16(&pLeaf->p[2]); - if( iOff==0 ){ + if( pLeaf->nn<=pLeaf->szLeaf ){ p->rc = FTS5_CORRUPT; }else{ - int iRowidOff; + int iOff; /* Offset of first term on leaf */ + int iRowidOff; /* Offset of first rowid on leaf */ int nTerm; /* Size of term on leaf in bytes */ int res; /* Comparison of term and split-key */ + iOff = fts5LeafFirstTermOff(pLeaf); iRowidOff = fts5GetU16(&pLeaf->p[0]); if( iRowidOff>=iOff ){ p->rc = FTS5_CORRUPT; @@ -4929,7 +5069,8 @@ static void fts5IndexIntegrityCheckSegment( if( pLeaf ){ i64 iRowid; int iRowidOff = fts5GetU16(&pLeaf->p[0]); - if( iRowidOff>=pLeaf->n ){ + ASSERT_SZLEAF_OK(pLeaf); + if( iRowidOff>=pLeaf->szLeaf ){ p->rc = FTS5_CORRUPT; }else{ fts5GetVarint(&pLeaf->p[iRowidOff], (u64*)&iRowid); @@ -5193,8 +5334,10 @@ static int fts5DecodeDoclist(int *pRc, Fts5Buffer *pBuf, const u8 *a, int n){ i64 iDocid; int iOff = 0; - iOff = sqlite3Fts5GetVarint(&a[iOff], (u64*)&iDocid); - sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " rowid=%lld", iDocid); + if( n>0 ){ + iOff = sqlite3Fts5GetVarint(a, (u64*)&iDocid); + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " id=%lld", iDocid); + } while( iOff=4 ){ iRowidOff = fts5GetU16(&a[0]); - iTermOff = fts5GetU16(&a[2]); + szLeaf = fts5GetU16(&a[2]); + if( szLeaf Date: Sat, 5 Sep 2015 22:36:07 +0000 Subject: [PATCH 043/100] Omit all use of Expr nodes for TK_AS, as those nodes no longer served a useful purpose and in fact interferred with the query planner. FossilOrigin-Name: 7ab0b258eabfcfb7f1b0bd1b12e166d2f267823d --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/expr.c | 8 ++------ src/resolve.c | 38 +++----------------------------------- src/sqliteInt.h | 1 + src/treeview.c | 5 ----- test/indexexpr1.test | 21 +++++++++++++++++++++ 7 files changed, 38 insertions(+), 57 deletions(-) diff --git a/manifest b/manifest index f9b770eebc..bcf650cd13 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sunreachable\sconditional\sin\sthe\sWHERE\sclause\sanalysis\slogic. -D 2015-09-05T19:21:00.671 +C Omit\sall\suse\sof\sExpr\snodes\sfor\sTK_AS,\sas\sthose\snodes\sno\slonger\sserved\sa\suseful\npurpose\sand\sin\sfact\sinterferred\swith\sthe\squery\splanner. +D 2015-09-05T22:36:07.095 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/ctime.c 5a0b735dc95604766f5dac73973658eef782ee8b F src/date.c fb1c99172017dcc8e237339132c91a21a0788584 F src/dbstat.c f402e77e25089c6003d0c60b3233b9b3947d599a F src/delete.c 6792c80d7fb54c4df9f7680413952600e7439492 -F src/expr.c f3c4d165dea72b268a7b922d00035af1d510dd78 +F src/expr.c 3a76afcdac925294c39903b7002ddb9e5fd29863 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 83e1baba999bed3144ea5a2143fc922edf51135f F src/func.c ecdd69ec6a1e406f04cc73324be2ebbf6354197f @@ -337,14 +337,14 @@ F src/pragma.h 631a91c8b0e6ca8f051a1d8a4a0da4150e04620a F src/prepare.c 82e5db1013846a819f198336fed72c44c974e7b1 F src/printf.c 0c4bcdd1c2e2521024f0a69cb5eb334f86b3652a F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 -F src/resolve.c 47c00ca0ab24d4b8113c6c05aa07bc6cf6eac9af +F src/resolve.c 3126f7694b8ce0f97282d7dd3a5198b8fa18dce9 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c c17613385bc6b095c421b1f30548814f5fd8a9b2 F src/shell.c 6332ef06db1390ef812cfdff1fc97b4fd76cdd42 F src/sqlite.h.in 378bebc8fe6a88bade25e5f23b7e6123fdc64b00 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h f700e6a9dd1fdcccc9951ab022b366fb66b9e413 -F src/sqliteInt.h e7e92112a71aeb214c66ebdea1366560535614d1 +F src/sqliteInt.h ad5504745ef37aca44365a2b91173bc399d71e7e F src/sqliteLimit.h 216557999cb45f2e3578ed53ebefe228d779cb46 F src/status.c f266ad8a2892d659b74f0f50cb6a88b6e7c12179 F src/table.c 51b46b2a62d1b3a959633d593b89bab5e2c9155e @@ -396,7 +396,7 @@ F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 6bbcc9fe50c917864d48287b4792d46d6e873481 F src/tokenize.c 57cb3720f53f84d811def2069c2b169b6be539a5 -F src/treeview.c 46036cbbceada0836833531b2d963edbca3d9cfa +F src/treeview.c 154f0acc622fa3514de8777dcedf4c8a8802b4ce F src/trigger.c 322f23aad694e8f31d384dcfa386d52a48d3c52f F src/update.c 3c5bc9570df3bfafa0db36828406a8a14e4c426e F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c @@ -782,7 +782,7 @@ F test/index5.test 8621491915800ec274609e42e02a97d67e9b13e7 F test/index6.test 7102ec371414c42dfb1d5ca37eb4519aa9edc23a F test/index7.test 9c6765a74fc3fcde7aebc5b3bd40d98df14a527c F test/indexedby.test 5f527a78bae74c61b8046ae3037f9dfb0bf0c353 -F test/indexexpr1.test 3c5033412f851f225e3a37d6795709df71bea638 +F test/indexexpr1.test 4feec154aadacb033b41acc1760a18edc4c60470 F test/indexfault.test 31d4ab9a7d2f6e9616933eb079722362a883eb1d F test/init.test 15c823093fdabbf7b531fe22cf037134d09587a7 F test/insert.test 38742b5e9601c8f8d76e9b7555f7270288c2d371 @@ -1383,7 +1383,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P d2761357a0496ec1e590c7c9e397c5b5c904f91a -R 9921ff0c69ded6987711a9a840df906d +P 24924a58197e558a9e8800cc5c91dc8fb32f3557 +R 7e67010ff7c10ab44fd6cfd6cc975796 U drh -Z 612add4179403f74341dc4eb50c9c3d3 +Z 8a5f85b9c3f71859f18b9284198b5fad diff --git a/manifest.uuid b/manifest.uuid index 4767aec925..86f4bfaeac 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -24924a58197e558a9e8800cc5c91dc8fb32f3557 \ No newline at end of file +7ab0b258eabfcfb7f1b0bd1b12e166d2f267823d \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 00228d9a3f..3141cd9dbd 100644 --- a/src/expr.c +++ b/src/expr.c @@ -91,7 +91,7 @@ Expr *sqlite3ExprAddCollateString(Parse *pParse, Expr *pExpr, const char *zC){ } /* -** Skip over any TK_COLLATE or TK_AS operators and any unlikely() +** Skip over any TK_COLLATE operators and any unlikely() ** or likelihood() function at the root of an expression. */ Expr *sqlite3ExprSkipCollate(Expr *pExpr){ @@ -102,7 +102,7 @@ Expr *sqlite3ExprSkipCollate(Expr *pExpr){ assert( pExpr->op==TK_FUNCTION ); pExpr = pExpr->x.pList->a[0].pExpr; }else{ - assert( pExpr->op==TK_COLLATE || pExpr->op==TK_AS ); + assert( pExpr->op==TK_COLLATE ); pExpr = pExpr->pLeft; } } @@ -2701,10 +2701,6 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ inReg = pExpr->iTable; break; } - case TK_AS: { - inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); - break; - } #ifndef SQLITE_OMIT_CAST case TK_CAST: { /* Expressions of the form: CAST(pLeft AS token) */ diff --git a/src/resolve.c b/src/resolve.c index 22df223948..2c4212ba7d 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -45,30 +45,6 @@ static void incrAggFunctionDepth(Expr *pExpr, int N){ ** Turn the pExpr expression into an alias for the iCol-th column of the ** result set in pEList. ** -** If the result set column is a simple column reference, then this routine -** makes an exact copy. But for any other kind of expression, this -** routine make a copy of the result set column as the argument to the -** TK_AS operator. The TK_AS operator causes the expression to be -** evaluated just once and then reused for each alias. -** -** The reason for suppressing the TK_AS term when the expression is a simple -** column reference is so that the column reference will be recognized as -** usable by indices within the WHERE clause processing logic. -** -** The TK_AS operator is inhibited if zType[0]=='G'. This means -** that in a GROUP BY clause, the expression is evaluated twice. Hence: -** -** SELECT random()%5 AS x, count(*) FROM tab GROUP BY x -** -** Is equivalent to: -** -** SELECT random()%5 AS x, count(*) FROM tab GROUP BY random()%5 -** -** The result of random()%5 in the GROUP BY clause is probably different -** from the result in the result-set. On the other hand Standard SQL does -** not allow the GROUP BY clause to contain references to result-set columns. -** So this should never come up in well-formed queries. -** ** If the reference is followed by a COLLATE operator, then make sure ** the COLLATE operator is preserved. For example: ** @@ -102,19 +78,11 @@ static void resolveAlias( db = pParse->db; pDup = sqlite3ExprDup(db, pOrig, 0); if( pDup==0 ) return; - if( pOrig->op!=TK_COLUMN && zType[0]!='G' ){ - incrAggFunctionDepth(pDup, nSubquery); - pDup = sqlite3PExpr(pParse, TK_AS, pDup, 0, 0); - if( pDup==0 ) return; - ExprSetProperty(pDup, EP_Skip); - if( pEList->a[iCol].u.x.iAlias==0 ){ - pEList->a[iCol].u.x.iAlias = (u16)(++pParse->nAlias); - } - pDup->iTable = pEList->a[iCol].u.x.iAlias; - } + if( zType[0]!='G' ) incrAggFunctionDepth(pDup, nSubquery); if( pExpr->op==TK_COLLATE ){ pDup = sqlite3ExprAddCollateString(pParse, pDup, pExpr->u.zToken); } + ExprSetProperty(pDup, EP_Alias); /* Before calling sqlite3ExprDelete(), set the EP_Static flag. This ** prevents ExprDelete() from deleting the Expr structure itself, @@ -506,7 +474,7 @@ static int lookupName( lookupname_end: if( cnt==1 ){ assert( pNC!=0 ); - if( pExpr->op!=TK_AS ){ + if( !ExprHasProperty(pExpr, EP_Alias) ){ sqlite3AuthRead(pParse, pExpr, pSchema, pNC->pSrcList); } /* Increment the nRef value on all name contexts from TopNC up to diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 69315ed466..89b3d798f1 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2135,6 +2135,7 @@ struct Expr { #define EP_ConstFunc 0x080000 /* A SQLITE_FUNC_CONSTANT or _SLOCHNG function */ #define EP_CanBeNull 0x100000 /* Can be null despite NOT NULL constraint */ #define EP_Subquery 0x200000 /* Tree contains a TK_SELECT operator */ +#define EP_Alias 0x400000 /* Is an alias for a result set column */ /* ** Combinations of two or more EP_* flags diff --git a/src/treeview.c b/src/treeview.c index 59c07e4742..971de4e8bc 100644 --- a/src/treeview.c +++ b/src/treeview.c @@ -253,11 +253,6 @@ void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){ sqlite3TreeViewLine(pView,"REGISTER(%d)", pExpr->iTable); break; } - case TK_AS: { - sqlite3TreeViewLine(pView,"AS %Q", pExpr->u.zToken); - sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); - break; - } case TK_ID: { sqlite3TreeViewLine(pView,"ID \"%w\"", pExpr->u.zToken); break; diff --git a/test/indexexpr1.test b/test/indexexpr1.test index faaf292de5..0c925c9f92 100644 --- a/test/indexexpr1.test +++ b/test/indexexpr1.test @@ -198,4 +198,25 @@ do_catchsql_test indexexpr1-410 { INSERT INTO t3 SELECT * FROM t3 WHERE rowid=10; } {1 {UNIQUE constraint failed: index 't3abc'}} +do_execsql_test indexexpr1-500 { + CREATE TABLE t5(a); + CREATE TABLE cnt(x); + WITH RECURSIVE + c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<5) + INSERT INTO cnt(x) SELECT x FROM c; + INSERT INTO t5(a) SELECT printf('abc%03dxyz',x) FROM cnt; + CREATE INDEX t5ax ON t5( substr(a,4,3) ); +} {} +do_execsql_test indexexpr1-510 { + -- The use of the "k" alias in the WHERE clause is technically + -- illegal, but SQLite allows it for historical reasons. In this + -- test and the next, verify that "k" can be used by the t5ax index + SELECT substr(a,4,3) AS k FROM cnt, t5 WHERE k=printf('%03d',x); +} {001 002 003 004 005} +do_execsql_test indexexpr1-510eqp { + EXPLAIN QUERY PLAN + SELECT substr(a,4,3) AS k FROM cnt, t5 WHERE k=printf('%03d',x); +} {/USING INDEX t5ax/} + + finish_test From 6081c1dbdf7730752bbde89ebb17d01bb30bf8f0 Mon Sep 17 00:00:00 2001 From: drh Date: Sun, 6 Sep 2015 02:51:04 +0000 Subject: [PATCH 044/100] Add a memory barrier to the mutex initialization logic, try to work around an issue reported by WebKit. FossilOrigin-Name: 11a9a786ec06403addb47f5c6fb142b382fae522 --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/mutex.c | 1 + src/mutex.h | 1 + src/mutex_unix.c | 12 ++++++++++++ src/mutex_w32.c | 7 +++++++ src/sqliteInt.h | 1 + 7 files changed, 33 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index bcf650cd13..6ceaeb6515 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Omit\sall\suse\sof\sExpr\snodes\sfor\sTK_AS,\sas\sthose\snodes\sno\slonger\sserved\sa\suseful\npurpose\sand\sin\sfact\sinterferred\swith\sthe\squery\splanner. -D 2015-09-05T22:36:07.095 +C Add\sa\smemory\sbarrier\sto\sthe\smutex\sinitialization\slogic,\stry\sto\swork\saround\nan\sissue\sreported\sby\sWebKit. +D 2015-09-06T02:51:04.140 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -313,11 +313,11 @@ F src/mem3.c 61c9d47b792908c532ca3a62b999cf21795c6534 F src/mem5.c 61eeb90134f9a5be6c2e68d8daae7628b25953fb F src/memjournal.c 3eb2c0b51adbd869cb6a44780323f05fa904dc85 F src/msvc.h d9ba56c6851227ab44b3f228a35f3f5772296495 -F src/mutex.c 529e95739f815300a33c73fd8a7d6bdf0c24bd18 -F src/mutex.h 779d588e3b7756ec3ecf7d78cde1d84aba414f85 +F src/mutex.c a39809c6c33f1ebc9cc781186c338ad90433e1e7 +F src/mutex.h 012503b51ccfcf85b8b3846709a4c60a5839f16c F src/mutex_noop.c 9d4309c075ba9cc7249e19412d3d62f7f94839c4 -F src/mutex_unix.c b0d280089df0f49545f1318f45d61d07d2f674a8 -F src/mutex_w32.c b601f9e3073f7bd2c1f42a8c0ce59e42d6a08f85 +F src/mutex_unix.c c3f415ebd3c1a7952d24f46f748aef796beabdcc +F src/mutex_w32.c 0f323eb53f092393be9d1fa91c9730baf56c4718 F src/notify.c 9711a7575036f0d3040ba61bc6e217f13a9888e7 F src/os.c 8fd25588eeba74068d41102d26810e216999b6c8 F src/os.h 3e57a24e2794a94d3cf2342c6d9a884888cd96bf @@ -344,7 +344,7 @@ F src/shell.c 6332ef06db1390ef812cfdff1fc97b4fd76cdd42 F src/sqlite.h.in 378bebc8fe6a88bade25e5f23b7e6123fdc64b00 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h f700e6a9dd1fdcccc9951ab022b366fb66b9e413 -F src/sqliteInt.h ad5504745ef37aca44365a2b91173bc399d71e7e +F src/sqliteInt.h dba8add0b95fcea1047af7a6875a1e305fcb96d7 F src/sqliteLimit.h 216557999cb45f2e3578ed53ebefe228d779cb46 F src/status.c f266ad8a2892d659b74f0f50cb6a88b6e7c12179 F src/table.c 51b46b2a62d1b3a959633d593b89bab5e2c9155e @@ -1383,7 +1383,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 24924a58197e558a9e8800cc5c91dc8fb32f3557 -R 7e67010ff7c10ab44fd6cfd6cc975796 +P 7ab0b258eabfcfb7f1b0bd1b12e166d2f267823d +R b2a67150d05e13e20516640b31b0c7ad U drh -Z 8a5f85b9c3f71859f18b9284198b5fad +Z b9d369a816ef742a733a926cd28b24f3 diff --git a/manifest.uuid b/manifest.uuid index 86f4bfaeac..7aaeaf8ce5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7ab0b258eabfcfb7f1b0bd1b12e166d2f267823d \ No newline at end of file +11a9a786ec06403addb47f5c6fb142b382fae522 \ No newline at end of file diff --git a/src/mutex.c b/src/mutex.c index 64efd3b05e..a2e4e6387a 100644 --- a/src/mutex.c +++ b/src/mutex.c @@ -53,6 +53,7 @@ int sqlite3MutexInit(void){ pTo->xMutexLeave = pFrom->xMutexLeave; pTo->xMutexHeld = pFrom->xMutexHeld; pTo->xMutexNotheld = pFrom->xMutexNotheld; + sqlite3MemoryBarrier(); pTo->xMutexAlloc = pFrom->xMutexAlloc; } rc = sqlite3GlobalConfig.mutex.xMutexInit(); diff --git a/src/mutex.h b/src/mutex.h index 03eb1faadb..8bcf2353f7 100644 --- a/src/mutex.h +++ b/src/mutex.h @@ -64,6 +64,7 @@ #define sqlite3MutexAlloc(X) ((sqlite3_mutex*)8) #define sqlite3MutexInit() SQLITE_OK #define sqlite3MutexEnd() +#define sqlite3MemoryBarrier() #define MUTEX_LOGIC(X) #else #define MUTEX_LOGIC(X) X diff --git a/src/mutex_unix.c b/src/mutex_unix.c index 0a493fa6a7..e181ba5bd1 100644 --- a/src/mutex_unix.c +++ b/src/mutex_unix.c @@ -80,6 +80,18 @@ static int pthreadMutexNotheld(sqlite3_mutex *p){ } #endif +/* +** Try to provide a memory barrier operation, needed for initialization only. +*/ +void sqlite3MemoryBarrier(void){ +#if defined(__GNUC__) + __sync_synchronize(); +#endif +#ifdef SQLITE_MEMORY_BARRIER + SQLITE_MEMORY_BARRIER; +#endif +} + /* ** Initialize and deinitialize the mutex subsystem. */ diff --git a/src/mutex_w32.c b/src/mutex_w32.c index fc943acaa0..9f2fb048fd 100644 --- a/src/mutex_w32.c +++ b/src/mutex_w32.c @@ -77,6 +77,13 @@ static int winMutexNotheld(sqlite3_mutex *p){ } #endif +/* +** Try to provide a memory barrier operation, needed for initialization only. +*/ +void sqlite3MemoryBarrier(void){ + MemoryBarrier(); +} + /* ** Initialize and deinitialize the mutex subsystem. */ diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 89b3d798f1..eeb0b9950c 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3191,6 +3191,7 @@ const sqlite3_mem_methods *sqlite3MemGetMemsys5(void); sqlite3_mutex *sqlite3MutexAlloc(int); int sqlite3MutexInit(void); int sqlite3MutexEnd(void); + void sqlite3MemoryBarrier(void); #endif sqlite3_int64 sqlite3StatusValue(int); From 2d64034bc2b1d61eea4c75e11af5c5ef15576352 Mon Sep 17 00:00:00 2001 From: drh Date: Sun, 6 Sep 2015 10:31:37 +0000 Subject: [PATCH 045/100] Improved memory barrier that should work with MinGW on older versions of Windows. FossilOrigin-Name: 47dc24bd1e8f76eb17ba53a883b8984b3e1b2934 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/mutex_unix.c | 7 +++---- src/mutex_w32.c | 6 ++++++ 4 files changed, 17 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 6ceaeb6515..54933846aa 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\smemory\sbarrier\sto\sthe\smutex\sinitialization\slogic,\stry\sto\swork\saround\nan\sissue\sreported\sby\sWebKit. -D 2015-09-06T02:51:04.140 +C Improved\smemory\sbarrier\sthat\sshould\swork\swith\sMinGW\son\solder\sversions\sof\nWindows. +D 2015-09-06T10:31:37.987 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -316,8 +316,8 @@ F src/msvc.h d9ba56c6851227ab44b3f228a35f3f5772296495 F src/mutex.c a39809c6c33f1ebc9cc781186c338ad90433e1e7 F src/mutex.h 012503b51ccfcf85b8b3846709a4c60a5839f16c F src/mutex_noop.c 9d4309c075ba9cc7249e19412d3d62f7f94839c4 -F src/mutex_unix.c c3f415ebd3c1a7952d24f46f748aef796beabdcc -F src/mutex_w32.c 0f323eb53f092393be9d1fa91c9730baf56c4718 +F src/mutex_unix.c 7762c8ec907379204f2ed751a0e33663ab1c14d7 +F src/mutex_w32.c 2e025e6642eaf27597403690980f560d1a91f62c F src/notify.c 9711a7575036f0d3040ba61bc6e217f13a9888e7 F src/os.c 8fd25588eeba74068d41102d26810e216999b6c8 F src/os.h 3e57a24e2794a94d3cf2342c6d9a884888cd96bf @@ -1383,7 +1383,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 7ab0b258eabfcfb7f1b0bd1b12e166d2f267823d -R b2a67150d05e13e20516640b31b0c7ad +P 11a9a786ec06403addb47f5c6fb142b382fae522 +R ddcc46edb79b17bfa61348716fbd1950 U drh -Z b9d369a816ef742a733a926cd28b24f3 +Z 19a2121ad81300b8c56d01dcfbdbe0e2 diff --git a/manifest.uuid b/manifest.uuid index 7aaeaf8ce5..5d01533397 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -11a9a786ec06403addb47f5c6fb142b382fae522 \ No newline at end of file +47dc24bd1e8f76eb17ba53a883b8984b3e1b2934 \ No newline at end of file diff --git a/src/mutex_unix.c b/src/mutex_unix.c index e181ba5bd1..78fba1d81f 100644 --- a/src/mutex_unix.c +++ b/src/mutex_unix.c @@ -84,11 +84,10 @@ static int pthreadMutexNotheld(sqlite3_mutex *p){ ** Try to provide a memory barrier operation, needed for initialization only. */ void sqlite3MemoryBarrier(void){ -#if defined(__GNUC__) - __sync_synchronize(); -#endif -#ifdef SQLITE_MEMORY_BARRIER +#if defined(SQLITE_MEMORY_BARRIER) SQLITE_MEMORY_BARRIER; +#elif defined(__GNUC__) + __sync_synchronize(); #endif } diff --git a/src/mutex_w32.c b/src/mutex_w32.c index 9f2fb048fd..90be07db2d 100644 --- a/src/mutex_w32.c +++ b/src/mutex_w32.c @@ -81,7 +81,13 @@ static int winMutexNotheld(sqlite3_mutex *p){ ** Try to provide a memory barrier operation, needed for initialization only. */ void sqlite3MemoryBarrier(void){ +#if defined(SQLITE_MEMORY_BARRIER) + SQLITE_MEMORY_BARRIER; +#elif defined(__GNUC__) + __sync_synchronize(); +#else MemoryBarrier(); +#endif } /* From c75e0166f08b72417ca617c85f8c8d627c880d46 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 7 Sep 2015 02:23:02 +0000 Subject: [PATCH 046/100] Improved "Parser Statistics" output (the -s option) for the Lemon parser generator. FossilOrigin-Name: 809503e4efcdb498d176e8c0794a5ba0882adef2 --- manifest | 12 ++++----- manifest.uuid | 2 +- tool/lemon.c | 75 ++++++++++++++++++++++++++++++++++++--------------- 3 files changed, 61 insertions(+), 28 deletions(-) diff --git a/manifest b/manifest index 54933846aa..f8137c54e5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\smemory\sbarrier\sthat\sshould\swork\swith\sMinGW\son\solder\sversions\sof\nWindows. -D 2015-09-06T10:31:37.987 +C Improved\s"Parser\sStatistics"\soutput\s(the\s-s\soption)\sfor\sthe\sLemon\sparser\ngenerator. +D 2015-09-07T02:23:02.396 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1338,7 +1338,7 @@ F tool/fuzzershell.c f2fc86dd22df654b28851b85019d3bd007361751 F tool/genfkey.README cf68fddd4643bbe3ff8e31b8b6d8b0a1b85e20f4 F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce -F tool/lemon.c b12cb605725dabfa3a8607ec6bd9e978973c7af9 +F tool/lemon.c cc515b02a6610ed76c93d49bd01fdb219356d7ac F tool/lempar.c 01ca97f87610d1dac6d8cd96ab109ab1130e76dc F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862 F tool/logest.c eef612f8adf4d0993dafed0416064cf50d5d33c6 @@ -1383,7 +1383,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 11a9a786ec06403addb47f5c6fb142b382fae522 -R ddcc46edb79b17bfa61348716fbd1950 +P 47dc24bd1e8f76eb17ba53a883b8984b3e1b2934 +R d6df305b5709e894bdfc542577063f51 U drh -Z 19a2121ad81300b8c56d01dcfbdbe0e2 +Z 643bd9ba035c2e0820f3e9522cf3c1e3 diff --git a/manifest.uuid b/manifest.uuid index 5d01533397..f1edfd5794 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -47dc24bd1e8f76eb17ba53a883b8984b3e1b2934 \ No newline at end of file +809503e4efcdb498d176e8c0794a5ba0882adef2 \ No newline at end of file diff --git a/tool/lemon.c b/tool/lemon.c index 5967885021..f27fe4d24a 100644 --- a/tool/lemon.c +++ b/tool/lemon.c @@ -385,7 +385,8 @@ struct lemon { char *outname; /* Name of the current output file */ char *tokenprefix; /* A prefix added to token names in the .h file */ int nconflict; /* Number of parsing conflicts */ - int tablesize; /* Size of the parse tables */ + int nactiontab; /* Number of entries in the yy_action[] table */ + int tablesize; /* Total table size of all tables in bytes */ int basisflag; /* Print only basis configurations */ int has_fallback; /* True if any %fallback is seen in the grammar */ int nolinenosflag; /* True if #line statements should not be printed */ @@ -1482,6 +1483,18 @@ static void handle_T_option(char *z){ lemon_strcpy(user_templatename, z); } +/* forward reference */ +static const char *minimum_size_type(int lwr, int upr, int *pnByte); + +/* Print a single line of the "Parser Stats" output +*/ +static void stats_line(const char *zLabel, int iValue){ + int nLabel = lemonStrlen(zLabel); + printf(" %s%.*s %5d\n", zLabel, + 35-nLabel, "................................", + iValue); +} + /* The main program. Parse the command line and do it... */ int main(int argc, char **argv) { @@ -1613,11 +1626,15 @@ int main(int argc, char **argv) if( !mhflag ) ReportHeader(&lem); } if( statistics ){ - printf("Parser statistics: %d terminals, %d nonterminals, %d rules\n", - lem.nterminal, lem.nsymbol - lem.nterminal, lem.nrule); - printf(" %d states, %d parser table entries," - " %d conflicts\n", - lem.nstate, lem.tablesize, lem.nconflict); + printf("Parser statistics:\n"); + stats_line("terminal symbols", lem.nterminal); + stats_line("non-terminal symbols", lem.nsymbol - lem.nterminal); + stats_line("total symbols", lem.nsymbol); + stats_line("rules", lem.nrule); + stats_line("states", lem.nstate); + stats_line("conflicts", lem.nconflict); + stats_line("action table entries", lem.nactiontab); + stats_line("total table size (bytes)", lem.tablesize); } if( lem.nconflict > 0 ){ fprintf(stderr,"%d parsing conflicts.\n",lem.nconflict); @@ -3661,24 +3678,32 @@ void print_stack_union( /* ** Return the name of a C datatype able to represent values between -** lwr and upr, inclusive. +** lwr and upr, inclusive. If pnByte!=NULL then also write the sizeof +** for that type (1, 2, or 4) into *pnByte. */ -static const char *minimum_size_type(int lwr, int upr){ +static const char *minimum_size_type(int lwr, int upr, int *pnByte){ + const char *zType = "int"; + int nByte = 4; if( lwr>=0 ){ if( upr<=255 ){ - return "unsigned char"; + zType = "unsigned char"; + nByte = 1; }else if( upr<65535 ){ - return "unsigned short int"; + zType = "unsigned short int"; + nByte = 2; }else{ - return "unsigned int"; + zType = "unsigned int"; + nByte = 4; } }else if( lwr>=-127 && upr<=127 ){ - return "signed char"; + zType = "signed char"; + nByte = 1; }else if( lwr>=-32767 && upr<32767 ){ - return "short"; - }else{ - return "int"; + zType = "short"; + nByte = 2; } + if( pnByte ) *pnByte = nByte; + return zType; } /* @@ -3742,7 +3767,9 @@ void ReportTable( struct action *ap; struct rule *rp; struct acttab *pActtab; - int i, j, n; + int i, j, n, sz; + int szActionType; /* sizeof(YYACTIONTYPE) */ + int szCodeType; /* sizeof(YYCODETYPE) */ const char *name; int mnTknOfst, mxTknOfst; int mnNtOfst, mxNtOfst; @@ -3783,10 +3810,10 @@ void ReportTable( /* Generate the defines */ fprintf(out,"#define YYCODETYPE %s\n", - minimum_size_type(0, lemp->nsymbol+1)); lineno++; + minimum_size_type(0, lemp->nsymbol+1, &szCodeType)); lineno++; fprintf(out,"#define YYNOCODE %d\n",lemp->nsymbol+1); lineno++; fprintf(out,"#define YYACTIONTYPE %s\n", - minimum_size_type(0, lemp->nstate+lemp->nrule+5)); lineno++; + minimum_size_type(0, lemp->nstate+lemp->nrule+5, &szActionType)); lineno++; if( lemp->wildcard ){ fprintf(out,"#define YYWILDCARD %d\n", lemp->wildcard->index); lineno++; @@ -3900,7 +3927,8 @@ void ReportTable( free(ax); /* Output the yy_action table */ - n = acttab_size(pActtab); + lemp->nactiontab = n = acttab_size(pActtab); + lemp->tablesize += n*szActionType; fprintf(out,"#define YY_ACTTAB_COUNT (%d)\n", n); lineno++; fprintf(out,"static const YYACTIONTYPE yy_action[] = {\n"); lineno++; for(i=j=0; itablesize += n*szCodeType; fprintf(out,"static const YYCODETYPE yy_lookahead[] = {\n"); lineno++; for(i=j=0; itablesize += n*sz; for(i=j=0; isorted[i]; @@ -3966,7 +3996,8 @@ void ReportTable( fprintf(out, "#define YY_REDUCE_MIN (%d)\n", mnNtOfst); lineno++; fprintf(out, "#define YY_REDUCE_MAX (%d)\n", mxNtOfst); lineno++; fprintf(out, "static const %s yy_reduce_ofst[] = {\n", - minimum_size_type(mnNtOfst-1, mxNtOfst)); lineno++; + minimum_size_type(mnNtOfst-1, mxNtOfst, &sz)); lineno++; + lemp->tablesize += n*sz; for(i=j=0; isorted[i]; @@ -3986,6 +4017,7 @@ void ReportTable( /* Output the default action table */ fprintf(out, "static const YYACTIONTYPE yy_default[] = {\n"); lineno++; n = lemp->nstate; + lemp->tablesize += n*szActionType; for(i=j=0; isorted[i]; if( j==0 ) fprintf(out," /* %5d */ ", i); @@ -4005,6 +4037,7 @@ void ReportTable( if( lemp->has_fallback ){ int mx = lemp->nterminal - 1; while( mx>0 && lemp->symbols[mx]->fallback==0 ){ mx--; } + lemp->tablesize += (mx+1)*szCodeType; for(i=0; i<=mx; i++){ struct symbol *p = lemp->symbols[i]; if( p->fallback==0 ){ From 60f8139a963ac96406d95e4222a64c94af5346a6 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 7 Sep 2015 08:14:30 +0000 Subject: [PATCH 047/100] Use macros to make the code in fts5_index.c easier to read. FossilOrigin-Name: 67ff5ae81357eb7fa28049bb724a22cb6f52e076 --- ext/fts5/fts5_index.c | 25 +++++++++++++------------ manifest | 15 ++++++--------- manifest.uuid | 2 +- 3 files changed, 20 insertions(+), 22 deletions(-) diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 7fa5a0fae4..87eb54b125 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -522,7 +522,8 @@ struct Fts5SegIter { */ #define fts5LeafIsTermless(x) ((x)->szLeaf >= (x)->nn) -#define fts5LeafFirstTermOff(x) (fts5GetU16(&x->p[(x)->szLeaf])) +#define fts5LeafFirstTermOff(x) (fts5GetU16(&(x)->p[(x)->szLeaf])) +#define fts5LeafFirstRowidOff(x) (fts5GetU16((x)->p)) /* ** poslist: @@ -1585,8 +1586,8 @@ static void fts5SegIterInit( if( p->rc==SQLITE_OK ){ u8 *a = pIter->pLeaf->p; - pIter->iLeafOffset = fts5GetU16(&a[pIter->pLeaf->szLeaf]); - assert( pIter->iLeafOffset==4 ); + pIter->iLeafOffset = 4; + assert( fts5LeafFirstTermOff(pIter->pLeaf)==4 ); fts5SegIterLoadTerm(p, pIter, 0); fts5SegIterLoadNPos(p, pIter); } @@ -1787,12 +1788,12 @@ static void fts5SegIterNext( pLeaf = pIter->pLeaf; if( pLeaf==0 ) break; ASSERT_SZLEAF_OK(pLeaf); - if( (iOff = fts5GetU16(&pLeaf->p[0])) && iOffszLeaf ){ + if( (iOff = fts5LeafFirstRowidOff(pLeaf)) && iOffszLeaf ){ iOff += sqlite3Fts5GetVarint(&pLeaf->p[iOff], (u64*)&pIter->iRowid); pIter->iLeafOffset = iOff; } else if( pLeaf->nn>pLeaf->szLeaf ){ - iOff = fts5GetU16(&pLeaf->p[pLeaf->szLeaf]); + iOff = fts5LeafFirstTermOff(pLeaf); pIter->iLeafOffset = iOff; bNewTerm = 1; } @@ -1881,7 +1882,7 @@ static void fts5SegIterReverse(Fts5Index *p, Fts5SegIter *pIter){ Fts5Data *pNew = fts5DataRead(p, iAbs); if( pNew ){ int iRowid, bTermless; - iRowid = fts5GetU16(pNew->p); + iRowid = fts5LeafFirstRowidOff(pNew); bTermless = fts5LeafIsTermless(pNew); if( iRowid ){ SWAPVAL(Fts5Data*, pNew, pLast); @@ -1999,7 +2000,7 @@ static void fts5LeafSeek( assert( p->rc==SQLITE_OK ); assert( pIter->pLeaf ); - iOff = fts5GetU16(&a[n]); + iOff = fts5LeafFirstTermOff(pIter->pLeaf); if( iOff<4 || iOff>=n ){ p->rc = FTS5_CORRUPT; return; @@ -2434,7 +2435,7 @@ static void fts5SegIterGotoPage( u8 *a = pIter->pLeaf->p; int n = pIter->pLeaf->szLeaf; - iOff = fts5GetU16(&a[0]); + iOff = fts5LeafFirstRowidOff(pIter->pLeaf); if( iOff<4 || iOff>=n ){ p->rc = FTS5_CORRUPT; }else{ @@ -4963,7 +4964,7 @@ static void fts5IndexIntegrityCheckEmpty( Fts5Data *pLeaf = fts5DataRead(p, FTS5_SEGMENT_ROWID(pSeg->iSegid, 0, i)); if( pLeaf ){ if( !fts5LeafIsTermless(pLeaf) ) p->rc = FTS5_CORRUPT; - if( i>=iNoRowid && 0!=fts5GetU16(&pLeaf->p[0]) ) p->rc = FTS5_CORRUPT; + if( i>=iNoRowid && 0!=fts5LeafFirstRowidOff(pLeaf) ) p->rc = FTS5_CORRUPT; } fts5DataRelease(pLeaf); if( p->rc ) break; @@ -5017,7 +5018,7 @@ static void fts5IndexIntegrityCheckSegment( int res; /* Comparison of term and split-key */ iOff = fts5LeafFirstTermOff(pLeaf); - iRowidOff = fts5GetU16(&pLeaf->p[0]); + iRowidOff = fts5LeafFirstRowidOff(pLeaf); if( iRowidOff>=iOff ){ p->rc = FTS5_CORRUPT; }else{ @@ -5056,7 +5057,7 @@ static void fts5IndexIntegrityCheckSegment( iKey = FTS5_SEGMENT_ROWID(iSegid, 0, iPg); pLeaf = fts5DataRead(p, iKey); if( pLeaf ){ - if( fts5GetU16(&pLeaf->p[0])!=0 ) p->rc = FTS5_CORRUPT; + if( fts5LeafFirstRowidOff(pLeaf)!=0 ) p->rc = FTS5_CORRUPT; fts5DataRelease(pLeaf); } } @@ -5068,7 +5069,7 @@ static void fts5IndexIntegrityCheckSegment( pLeaf = fts5DataRead(p, iKey); if( pLeaf ){ i64 iRowid; - int iRowidOff = fts5GetU16(&pLeaf->p[0]); + int iRowidOff = fts5LeafFirstRowidOff(pLeaf); ASSERT_SZLEAF_OK(pLeaf); if( iRowidOff>=pLeaf->szLeaf ){ p->rc = FTS5_CORRUPT; diff --git a/manifest b/manifest index 6a1f86e904..f47bf05938 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Experiment\swith\sa\sdifferent\sfts5\sleaf\spage\sformat\sthat\sallows\sfaster\sseeks. -D 2015-09-05T19:52:08.105 +C Use\smacros\sto\smake\sthe\scode\sin\sfts5_index.c\seasier\sto\sread. +D 2015-09-07T08:14:30.857 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -112,7 +112,7 @@ F ext/fts5/fts5_buffer.c 80f9ba4431848cb857e3d2158f5280093dcd8015 F ext/fts5/fts5_config.c 80b61fd2c6844b64a3e72a64572d50a812da9384 F ext/fts5/fts5_expr.c 1c24e1a2ffb286bfe37e537a43b7fadabfe993d4 F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246 -F ext/fts5/fts5_index.c c34a64666c3b573aaed0fe103ce739ca2c0b88e5 +F ext/fts5/fts5_index.c 213e5aea27100a2ebb7a576d6574bcc28c594520 F ext/fts5/fts5_main.c e9d0892424bb7f0a8b58613d4ff75cb650cf286e F ext/fts5/fts5_storage.c 120f7b143688b5b7710dacbd48cff211609b8059 F ext/fts5/fts5_tcl.c 6da58d6e8f42a93c4486b5ba9b187a7f995dee37 @@ -1384,10 +1384,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 24924a58197e558a9e8800cc5c91dc8fb32f3557 -R 00e6b769eaa54af1e95eef69109c890a -T *branch * fts5-incompatible -T *sym-fts5-incompatible * -T -sym-trunk * +P a1f4c3b543eed84e808f6b901a38179786fffe16 +R d9f9c87817152716ded47406f74ba235 U dan -Z 9f2973699597cf834c49e709dc04f160 +Z 4c8df884c6ae97a50d0e8662a6ca4b0e diff --git a/manifest.uuid b/manifest.uuid index 6229a84693..a4473a78af 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a1f4c3b543eed84e808f6b901a38179786fffe16 \ No newline at end of file +67ff5ae81357eb7fa28049bb724a22cb6f52e076 \ No newline at end of file From 7e698e9d71606c5f3d4364db26a0f72b2e7ceb4e Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 7 Sep 2015 14:22:24 +0000 Subject: [PATCH 048/100] In the "parse.out" output file from Lemon, show addition the complete text of rules on reduce actions. FossilOrigin-Name: b6ffb7e471e51ff69668154ad2c8790e466c9d37 --- main.mk | 2 +- manifest | 14 +++++------ manifest.uuid | 2 +- tool/lemon.c | 68 +++++++++++++++++++++++++++++++++++++-------------- 4 files changed, 58 insertions(+), 28 deletions(-) diff --git a/main.mk b/main.mk index d07e473be6..3668c84321 100644 --- a/main.mk +++ b/main.mk @@ -582,7 +582,7 @@ parse.h: parse.c parse.c: $(TOP)/src/parse.y lemon $(TOP)/addopcodes.awk cp $(TOP)/src/parse.y . rm -f parse.h - ./lemon $(OPTS) parse.y + ./lemon -s $(OPTS) parse.y mv parse.h parse.h.temp $(NAWK) -f $(TOP)/addopcodes.awk parse.h.temp >parse.h diff --git a/manifest b/manifest index f8137c54e5..28359665c7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\s"Parser\sStatistics"\soutput\s(the\s-s\soption)\sfor\sthe\sLemon\sparser\ngenerator. -D 2015-09-07T02:23:02.396 +C In\sthe\s"parse.out"\soutput\sfile\sfrom\sLemon,\sshow\saddition\sthe\scomplete\stext\nof\srules\son\sreduce\sactions. +D 2015-09-07T14:22:24.531 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -260,7 +260,7 @@ F ext/userauth/userauth.c 5fa3bdb492f481bbc1709fc83c91ebd13460c69e F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk 61821e43596648bfacce2d6283377bee35986131 +F main.mk 58eb74de702467c3b71cdf06f213cefe7f5ce544 F mkopcodec.awk c2ff431854d702cdd2d779c9c0d1f58fa16fa4ea F mkopcodeh.awk 0e7f04a8eb90f92259e47d80110e4e98d7ce337a F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 @@ -1338,7 +1338,7 @@ F tool/fuzzershell.c f2fc86dd22df654b28851b85019d3bd007361751 F tool/genfkey.README cf68fddd4643bbe3ff8e31b8b6d8b0a1b85e20f4 F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce -F tool/lemon.c cc515b02a6610ed76c93d49bd01fdb219356d7ac +F tool/lemon.c a110c3850af6b91a553e153141186c32dee58f0d F tool/lempar.c 01ca97f87610d1dac6d8cd96ab109ab1130e76dc F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862 F tool/logest.c eef612f8adf4d0993dafed0416064cf50d5d33c6 @@ -1383,7 +1383,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 47dc24bd1e8f76eb17ba53a883b8984b3e1b2934 -R d6df305b5709e894bdfc542577063f51 +P 809503e4efcdb498d176e8c0794a5ba0882adef2 +R fba651a02e6066be297178ded140cace U drh -Z 643bd9ba035c2e0820f3e9522cf3c1e3 +Z 5f03c60d31dd0f570ea326a36f822715 diff --git a/manifest.uuid b/manifest.uuid index f1edfd5794..31a7538355 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -809503e4efcdb498d176e8c0794a5ba0882adef2 \ No newline at end of file +b6ffb7e471e51ff69668154ad2c8790e466c9d37 \ No newline at end of file diff --git a/tool/lemon.c b/tool/lemon.c index f27fe4d24a..1f9ef7b84d 100644 --- a/tool/lemon.c +++ b/tool/lemon.c @@ -340,7 +340,7 @@ struct state { struct action *ap; /* Array of actions for this state */ int nTknAct, nNtAct; /* Number of actions on terminals and nonterminals */ int iTknOfst, iNtOfst; /* yy_action[] offset for terminals and nonterms */ - int iDflt; /* Default action */ + int iDflt; /* Default action is reduce by this rule */ }; #define NO_OFFSET (-2147483647) @@ -2960,15 +2960,14 @@ void Reprint(struct lemon *lemp) } } -void ConfigPrint(FILE *fp, struct config *cfp) -{ - struct rule *rp; +/* Print a single rule. +*/ +void RulePrint(FILE *fp, struct rule *rp, int iCursor){ struct symbol *sp; int i, j; - rp = cfp->rp; fprintf(fp,"%s ::=",rp->lhs->name); for(i=0; i<=rp->nrhs; i++){ - if( i==cfp->dot ) fprintf(fp," *"); + if( i==iCursor ) fprintf(fp," *"); if( i==rp->nrhs ) break; sp = rp->rhs[i]; if( sp->type==MULTITERMINAL ){ @@ -2982,6 +2981,12 @@ void ConfigPrint(FILE *fp, struct config *cfp) } } +/* Print the rule for a configuration. +*/ +void ConfigPrint(FILE *fp, struct config *cfp){ + RulePrint(fp, cfp->rp, cfp->dot); +} + /* #define TEST */ #if 0 /* Print a set */ @@ -3021,15 +3026,29 @@ char *tag; /* Print an action to the given file descriptor. Return FALSE if ** nothing was actually printed. */ -int PrintAction(struct action *ap, FILE *fp, int indent){ +int PrintAction( + struct action *ap, /* The action to print */ + FILE *fp, /* Print the action here */ + int indent, /* Indent by this amount */ + struct rule **apRule /* All rules by index */ +){ int result = 1; switch( ap->type ){ - case SHIFT: - fprintf(fp,"%*s shift %d",indent,ap->sp->name,ap->x.stp->statenum); + case SHIFT: { + struct state *stp = ap->x.stp; + fprintf(fp,"%*s shift %-7d",indent,ap->sp->name,stp->statenum); + if( stp->nTknAct==0 && stp->nNtAct==0 && apRule ){ + fprintf(fp,"then reduce %d: ", stp->iDflt); + RulePrint(fp, apRule[stp->iDflt], -1); + } break; - case REDUCE: - fprintf(fp,"%*s reduce %d",indent,ap->sp->name,ap->x.rp->index); + } + case REDUCE: { + struct rule *rp = ap->x.rp; + fprintf(fp,"%*s reduce %-7d",indent,ap->sp->name,rp->index); + if( apRule ) RulePrint(fp, apRule[rp->index], -1); break; + } case ACCEPT: fprintf(fp,"%*s accept",indent,ap->sp->name); break; @@ -3038,16 +3057,16 @@ int PrintAction(struct action *ap, FILE *fp, int indent){ break; case SRCONFLICT: case RRCONFLICT: - fprintf(fp,"%*s reduce %-3d ** Parsing conflict **", + fprintf(fp,"%*s reduce %-7d ** Parsing conflict **", indent,ap->sp->name,ap->x.rp->index); break; case SSCONFLICT: - fprintf(fp,"%*s shift %-3d ** Parsing conflict **", + fprintf(fp,"%*s shift %-7d ** Parsing conflict **", indent,ap->sp->name,ap->x.stp->statenum); break; case SH_RESOLVED: if( showPrecedenceConflict ){ - fprintf(fp,"%*s shift %-3d -- dropped by precedence", + fprintf(fp,"%*s shift %-7d -- dropped by precedence", indent,ap->sp->name,ap->x.stp->statenum); }else{ result = 0; @@ -3055,7 +3074,7 @@ int PrintAction(struct action *ap, FILE *fp, int indent){ break; case RD_RESOLVED: if( showPrecedenceConflict ){ - fprintf(fp,"%*s reduce %-3d -- dropped by precedence", + fprintf(fp,"%*s reduce %-7d -- dropped by precedence", indent,ap->sp->name,ap->x.rp->index); }else{ result = 0; @@ -3076,7 +3095,17 @@ void ReportOutput(struct lemon *lemp) struct config *cfp; struct action *ap; FILE *fp; + struct rule **apRule; + apRule = malloc( sizeof(apRule[0])*(lemp->nrule+1) ); + if( apRule ){ + struct rule *x; + memset(apRule, 0, sizeof(apRule[0])*(lemp->nrule+1) ); + for(x=lemp->rule; x; x=x->next){ + assert( x->index>=0 && x->index<(lemp->nrule+1) ); + apRule[x->index] = x; + } + } fp = file_open(lemp,".out","wb"); if( fp==0 ) return; for(i=0; instate; i++){ @@ -3104,7 +3133,7 @@ void ReportOutput(struct lemon *lemp) } fprintf(fp,"\n"); for(ap=stp->ap; ap; ap=ap->next){ - if( PrintAction(ap,fp,30) ) fprintf(fp,"\n"); + if( PrintAction(ap,fp,30,apRule) ) fprintf(fp,"\n"); } fprintf(fp,"\n"); } @@ -3130,6 +3159,7 @@ void ReportOutput(struct lemon *lemp) fprintf(fp, "\n"); } fclose(fp); + free(apRule); return; } @@ -4021,7 +4051,7 @@ void ReportTable( for(i=j=0; isorted[i]; if( j==0 ) fprintf(out," /* %5d */ ", i); - fprintf(out, " %4d,", stp->iDflt); + fprintf(out, " %4d,", stp->iDflt+n); if( j==9 || i==n-1 ){ fprintf(out, "\n"); lineno++; j = 0; @@ -4342,7 +4372,7 @@ void ResortStates(struct lemon *lemp) for(i=0; instate; i++){ stp = lemp->sorted[i]; stp->nTknAct = stp->nNtAct = 0; - stp->iDflt = lemp->nstate + lemp->nrule; + stp->iDflt = lemp->nrule; stp->iTknOfst = NO_OFFSET; stp->iNtOfst = NO_OFFSET; for(ap=stp->ap; ap; ap=ap->next){ @@ -4352,7 +4382,7 @@ void ResortStates(struct lemon *lemp) }else if( ap->sp->indexnsymbol ){ stp->nNtAct++; }else{ - stp->iDflt = compute_action(lemp, ap); + stp->iDflt = compute_action(lemp, ap) - lemp->nstate; } } } From 3bd48ab213bdfd776d04b4e9be4641bc0d0583e4 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 7 Sep 2015 18:23:37 +0000 Subject: [PATCH 049/100] For the Lemon-generated parser, add a new action type SHIFTREDUCE and use it to further compress the parser tables and improve parser performance. FossilOrigin-Name: 531c3974b3d586c1989cde905b2fb4681239a570 --- manifest | 22 +++--- manifest.uuid | 2 +- src/lempar.c | 98 ++++++++++++++++---------- src/parse.y | 2 +- src/tokenize.c | 3 +- test/misc1.test | 2 +- tool/lemon.c | 180 +++++++++++++++++++++++++++++------------------- tool/lempar.c | 99 ++++++++++++++++---------- 8 files changed, 252 insertions(+), 156 deletions(-) diff --git a/manifest b/manifest index 28359665c7..a6e735d796 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\s"parse.out"\soutput\sfile\sfrom\sLemon,\sshow\saddition\sthe\scomplete\stext\nof\srules\son\sreduce\sactions. -D 2015-09-07T14:22:24.531 +C For\sthe\sLemon-generated\sparser,\sadd\sa\snew\saction\stype\sSHIFTREDUCE\sand\suse\sit\nto\sfurther\scompress\sthe\sparser\stables\sand\simprove\sparser\sperformance. +D 2015-09-07T18:23:37.162 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -302,7 +302,7 @@ F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 F src/insert.c 076dc5876e261a9908603d54cfc5344cd680166c F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e -F src/lempar.c 92bafa308607dd985ca389a788cd9e0a2b608712 +F src/lempar.c 406f2e85d2552d88bd4b994cc9c567c0fbc9f483 F src/loadext.c dfcee8c7c032cd0fd55af3e0fc1fcfb01e426df2 F src/main.c e17fcffae4306a9b8334faf3bac80d7396850b54 F src/malloc.c 021012e28a81ffdabf4c30ec3df6ce1f6cc93f1d @@ -328,7 +328,7 @@ F src/os_win.c 40b3af7a47eb1107d0d69e592bec345a3b7b798a F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca F src/pager.c 4784012f80b2197c61ff6eaf4f5c7026d93253fd F src/pager.h 6d435f563b3f7fcae4b84433b76a6ac2730036e2 -F src/parse.y f599aa5e871a493330d567ced93de696f61f48f7 +F src/parse.y 9e1777c68d5bb483d7217327e524dcfe5263b01c F src/pcache.c 24be750c79272e0ca7b6e007bc94999700f3e5ef F src/pcache.h 9968603796240cdf83da7e7bef76edf90619cea9 F src/pcache1.c bf2afe64a3dedb8643c8dcbd94a145cc80ab2a67 @@ -395,7 +395,7 @@ F src/test_vfs.c 3b65d42e18b262805716bd96178c81da8f2d9283 F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 6bbcc9fe50c917864d48287b4792d46d6e873481 -F src/tokenize.c 57cb3720f53f84d811def2069c2b169b6be539a5 +F src/tokenize.c 9655e20ade774e5c8e580634f1359288eba3fada F src/treeview.c 154f0acc622fa3514de8777dcedf4c8a8802b4ce F src/trigger.c 322f23aad694e8f31d384dcfa386d52a48d3c52f F src/update.c 3c5bc9570df3bfafa0db36828406a8a14e4c426e @@ -866,7 +866,7 @@ F test/minmax.test 42fbad0e81afaa6e0de41c960329f2b2c3526efd F test/minmax2.test b44bae787fc7b227597b01b0ca5575c7cb54d3bc F test/minmax3.test cc1e8b010136db0d01a6f2a29ba5a9f321034354 F test/minmax4.test 936941484ebdceb8adec7c86b6cd9b6e5e897c1f -F test/misc1.test 3f1c479c5a093a6280f378c0fbff1c2701486660 +F test/misc1.test 0be38a2944dd3e63dfb2899f24446339487d17c7 F test/misc2.test 00d7de54eda90e237fc9a38b9e5ccc769ebf6d4d F test/misc3.test cf3dda47d5dda3e53fc5804a100d3c82be736c9d F test/misc4.test 0d8be3466adf123a7791a66ba2bc8e8d229e87f3 @@ -1338,8 +1338,8 @@ F tool/fuzzershell.c f2fc86dd22df654b28851b85019d3bd007361751 F tool/genfkey.README cf68fddd4643bbe3ff8e31b8b6d8b0a1b85e20f4 F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce -F tool/lemon.c a110c3850af6b91a553e153141186c32dee58f0d -F tool/lempar.c 01ca97f87610d1dac6d8cd96ab109ab1130e76dc +F tool/lemon.c 0c455691cc1e59a8f782d51a83dd6bbd7c5c44e7 +F tool/lempar.c 1522366692ef87584d4eacdbb1e95bdd0bc7f3b6 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862 F tool/logest.c eef612f8adf4d0993dafed0416064cf50d5d33c6 F tool/mkautoconfamal.sh d1a2da0e15b2ed33d60af35c7e9d483f13a8eb9f @@ -1383,7 +1383,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 809503e4efcdb498d176e8c0794a5ba0882adef2 -R fba651a02e6066be297178ded140cace +P b6ffb7e471e51ff69668154ad2c8790e466c9d37 +R 90541ca0731ebb2fb0e53dfdeffae5a4 U drh -Z 5f03c60d31dd0f570ea326a36f822715 +Z 08f2caec8f48c0982df46628b104ff5c diff --git a/manifest.uuid b/manifest.uuid index 31a7538355..1eb95138eb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b6ffb7e471e51ff69668154ad2c8790e466c9d37 \ No newline at end of file +531c3974b3d586c1989cde905b2fb4681239a570 \ No newline at end of file diff --git a/src/lempar.c b/src/lempar.c index b6c60a25b6..d014ebcc75 100644 --- a/src/lempar.c +++ b/src/lempar.c @@ -56,15 +56,19 @@ ** ParseARG_PDECL A parameter declaration for the %extra_argument ** ParseARG_STORE Code to store %extra_argument into yypParser ** ParseARG_FETCH Code to extract %extra_argument from yypParser -** YYNSTATE the combined number of states. -** YYNRULE the number of rules in the grammar ** YYERRORSYMBOL is the code number of the error symbol. If not ** defined, then do no error processing. +** YYNSTATE the combined number of states. +** YYNRULE the number of rules in the grammar +** YY_MAX_SHIFT Maximum value for shift actions +** YY_MIN_SHIFTREDUCE Minimum value for shift-reduce actions +** YY_MAX_SHIFTREDUCE Maximum value for shift-reduce actions +** YY_MIN_REDUCE Maximum value for reduce actions +** YY_ERROR_ACTION The yy_action[] code for syntax error +** YY_ACCEPT_ACTION The yy_action[] code for accept +** YY_NO_ACTION The yy_action[] code for no-op */ %% -#define YY_NO_ACTION (YYNSTATE+YYNRULE+2) -#define YY_ACCEPT_ACTION (YYNSTATE+YYNRULE+1) -#define YY_ERROR_ACTION (YYNSTATE+YYNRULE) /* The yyzerominor constant is used to initialize instances of ** YYMINORTYPE objects to zero. */ @@ -91,16 +95,20 @@ static const YYMINORTYPE yyzerominor = { 0 }; ** Suppose the action integer is N. Then the action is determined as ** follows ** -** 0 <= N < YYNSTATE Shift N. That is, push the lookahead +** 0 <= N <= YY_MAX_SHIFT Shift N. That is, push the lookahead ** token onto the stack and goto state N. ** -** YYNSTATE <= N < YYNSTATE+YYNRULE Reduce by rule N-YYNSTATE. +** N between YY_MIN_SHIFTREDUCE Shift to an arbitrary state then +** and YY_MAX_SHIFTREDUCE reduce by rule N-YY_MIN_SHIFTREDUCE. ** -** N == YYNSTATE+YYNRULE A syntax error has occurred. +** N between YY_MIN_REDUCE Reduce by rule N-YY_MIN_REDUCE +** and YY_MAX_REDUCE + +** N == YY_ERROR_ACTION A syntax error has occurred. ** -** N == YYNSTATE+YYNRULE+1 The parser accepts its input. +** N == YY_ACCEPT_ACTION The parser accepts its input. ** -** N == YYNSTATE+YYNRULE+2 No such action. Denotes unused +** N == YY_NO_ACTION No such action. Denotes unused ** slots in the yy_action[] table. ** ** The action table is constructed as a single large table named yy_action[]. @@ -499,9 +507,9 @@ static void yyStackOverflow(yyParser *yypParser, YYMINORTYPE *yypMinor){ } /* -** Perform a shift action. +** Perform a shift action. Return the number of errors. */ -static void yy_shift( +static int yy_shift( yyParser *yypParser, /* The parser to be shifted */ int yyNewState, /* The new state to shift in */ int yyMajor, /* The major token to shift in */ @@ -517,14 +525,14 @@ static void yy_shift( #if YYSTACKDEPTH>0 if( yypParser->yyidx>=YYSTACKDEPTH ){ yyStackOverflow(yypParser, yypMinor); - return; + return 1; } #else if( yypParser->yyidx>=yypParser->yystksz ){ yyGrowStack(yypParser); if( yypParser->yyidx>=yypParser->yystksz ){ yyStackOverflow(yypParser, yypMinor); - return; + return 1; } } #endif @@ -535,13 +543,18 @@ static void yy_shift( #ifndef NDEBUG if( yyTraceFILE && yypParser->yyidx>0 ){ int i; - fprintf(yyTraceFILE,"%sShift %d\n",yyTracePrompt,yyNewState); - fprintf(yyTraceFILE,"%sStack:",yyTracePrompt); - for(i=1; i<=yypParser->yyidx; i++) - fprintf(yyTraceFILE," %s",yyTokenName[yypParser->yystack[i].major]); - fprintf(yyTraceFILE,"\n"); + if( yyNewStateyyidx; i++) + fprintf(yyTraceFILE," %s",yyTokenName[yypParser->yystack[i].major]); + fprintf(yyTraceFILE,"\n"); + }else{ + fprintf(yyTraceFILE,"%sShift *\n",yyTracePrompt); + } } #endif + return 0; } /* The following table contains information about every rule that @@ -574,8 +587,9 @@ static void yy_reduce( #ifndef NDEBUG if( yyTraceFILE && yyruleno>=0 && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){ - fprintf(yyTraceFILE, "%sReduce [%s].\n", yyTracePrompt, - yyRuleName[yyruleno]); + yysize = yyRuleInfo[yyruleno].nrhs; + fprintf(yyTraceFILE, "%sReduce [%s] -> state %d.\n", yyTracePrompt, + yyRuleName[yyruleno], yymsp[-yysize].stateno); } #endif /* NDEBUG */ @@ -613,9 +627,8 @@ static void yy_reduce( yysize = yyRuleInfo[yyruleno].nrhs; yypParser->yyidx -= yysize; yyact = yy_find_reduce_action(yymsp[-yysize].stateno,(YYCODETYPE)yygoto); - if( yyact < YYNSTATE ){ -#ifdef NDEBUG - /* If we are not debugging and the reduce action popped at least + if( yyact <= YY_MAX_SHIFTREDUCE ){ + /* If the reduce action popped at least ** one element off the stack, then we can push the new element back ** onto the stack here, and skip the stack overflow test in yy_shift(). ** That gives a significant speed improvement. */ @@ -625,13 +638,19 @@ static void yy_reduce( yymsp->stateno = (YYACTIONTYPE)yyact; yymsp->major = (YYCODETYPE)yygoto; yymsp->minor = yygotominor; - }else +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sShift %d\n",yyTracePrompt,yyact); + } #endif - { - yy_shift(yypParser,yyact,yygoto,&yygotominor); + }else{ + if( yy_shift(yypParser,yyact,yygoto,&yygotominor) ) yyact = 0; + } + if( yyact>=YY_MIN_SHIFTREDUCE ){ + yy_reduce(yypParser, yyact - YY_MIN_SHIFTREDUCE); } }else{ - assert( yyact == YYNSTATE + YYNRULE + 1 ); + assert( yyact == YY_ACCEPT_ACTION ); yy_accept(yypParser); } } @@ -755,12 +774,16 @@ void Parse( do{ yyact = yy_find_shift_action(yypParser,(YYCODETYPE)yymajor); - if( yyactyyerrcnt--; - yymajor = YYNOCODE; - }else if( yyact < YYNSTATE + YYNRULE ){ - yy_reduce(yypParser,yyact-YYNSTATE); + if( yyact <= YY_MAX_SHIFTREDUCE ){ + if( yy_shift(yypParser,yyact,yymajor,&yyminorunion)==0 ){ + yypParser->yyerrcnt--; + yymajor = YYNOCODE; + if( yyact > YY_MAX_SHIFT ){ + yy_reduce(yypParser, yyact-YY_MIN_SHIFTREDUCE); + } + } + }else if( yyact <= YY_MAX_REDUCE ){ + yy_reduce(yypParser,yyact-YY_MIN_REDUCE); }else{ assert( yyact == YY_ERROR_ACTION ); #ifdef YYERRORSYMBOL @@ -810,7 +833,7 @@ void Parse( yymx != YYERRORSYMBOL && (yyact = yy_find_reduce_action( yypParser->yystack[yypParser->yyidx].stateno, - YYERRORSYMBOL)) >= YYNSTATE + YYERRORSYMBOL)) >= YY_MIN_REDUCE ){ yy_pop_parser_stack(yypParser); } @@ -860,5 +883,10 @@ void Parse( #endif } }while( yymajor!=YYNOCODE && yypParser->yyidx>=0 ); +#ifdef SQLITE_DEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sReturn\n",yyTracePrompt); + } +#endif return; } diff --git a/src/parse.y b/src/parse.y index e99feeefc1..cdfef7bd3b 100644 --- a/src/parse.y +++ b/src/parse.y @@ -587,7 +587,7 @@ from(A) ::= FROM seltablist(X). { // stl_prefix(A) ::= seltablist(X) joinop(Y). { A = X; - if( ALWAYS(A && A->nSrc>0) ) A->a[A->nSrc-1].fg.jointype = (u8)Y; + if( A && A->nSrc>0 ) A->a[A->nSrc-1].fg.jointype = (u8)Y; } stl_prefix(A) ::= . {A = 0;} seltablist(A) ::= stl_prefix(X) nm(Y) dbnm(D) as(Z) indexed_opt(I) diff --git a/src/tokenize.c b/src/tokenize.c index 3d08f75a2a..30a8ad06d0 100644 --- a/src/tokenize.c +++ b/src/tokenize.c @@ -403,6 +403,7 @@ int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){ pParse->zTail = zSql; i = 0; assert( pzErrMsg!=0 ); + /* sqlite3ParserTrace(stdout, "parser: "); */ pEngine = sqlite3ParserAlloc(sqlite3Malloc); if( pEngine==0 ){ db->mallocFailed = 1; @@ -458,12 +459,12 @@ abort_parse: assert( zSql[i]==0 ); if( lastTokenParsed!=TK_SEMI ){ sqlite3Parser(pEngine, TK_SEMI, pParse->sLastToken, pParse); - pParse->zTail = &zSql[i]; } if( pParse->rc==SQLITE_OK && db->mallocFailed==0 ){ sqlite3Parser(pEngine, 0, pParse->sLastToken, pParse); } } + pParse->zTail = &zSql[i]; #ifdef YYTRACKMAXSTACKDEPTH sqlite3_mutex_enter(sqlite3MallocMutex()); sqlite3StatusSet(SQLITE_STATUS_PARSER_STACK, diff --git a/test/misc1.test b/test/misc1.test index 25e9bd813e..be4ab8646d 100644 --- a/test/misc1.test +++ b/test/misc1.test @@ -646,7 +646,7 @@ do_catchsql_test misc1-21.1 { } {1 {near "#0": syntax error}} do_catchsql_test misc1-21.2 { VALUES(0,0x0MATCH#0; -} {1 {near ";": syntax error}} +} {1 {near "#0": syntax error}} # 2015-04-15 do_execsql_test misc1-22.1 { diff --git a/tool/lemon.c b/tool/lemon.c index 1f9ef7b84d..2bb3d5ef58 100644 --- a/tool/lemon.c +++ b/tool/lemon.c @@ -316,7 +316,8 @@ enum e_action { RRCONFLICT, /* Was a reduce, but part of a conflict */ SH_RESOLVED, /* Was a shift. Precedence resolved conflict */ RD_RESOLVED, /* Was reduce. Precedence resolved conflict */ - NOT_USED /* Deleted by compression */ + NOT_USED, /* Deleted by compression */ + SHIFTREDUCE /* Shift first, then reduce */ }; /* Every shift or reduce operation is stored as one of the following */ @@ -340,7 +341,9 @@ struct state { struct action *ap; /* Array of actions for this state */ int nTknAct, nNtAct; /* Number of actions on terminals and nonterminals */ int iTknOfst, iNtOfst; /* yy_action[] offset for terminals and nonterms */ - int iDflt; /* Default action is reduce by this rule */ + int iDfltReduce; /* Default action is to REDUCE by this rule */ + struct rule *pDfltReduce;/* The default REDUCE rule. */ + int autoReduce; /* True if this is an auto-reduce state */ }; #define NO_OFFSET (-2147483647) @@ -360,6 +363,7 @@ struct lemon { struct state **sorted; /* Table of states sorted by state number */ struct rule *rule; /* List of all rules */ int nstate; /* Number of states */ + int nxstate; /* nstate with tail degenerate states removed */ int nrule; /* Number of rules */ int nsymbol; /* Number of terminal and nonterminal symbols */ int nterminal; /* Number of terminal symbols */ @@ -484,7 +488,7 @@ static int actioncmp( if( rc==0 ){ rc = (int)ap1->type - (int)ap2->type; } - if( rc==0 && ap1->type==REDUCE ){ + if( rc==0 && (ap1->type==REDUCE || ap1->type==SHIFTREDUCE) ){ rc = ap1->x.rp->index - ap2->x.rp->index; } if( rc==0 ){ @@ -1631,7 +1635,7 @@ int main(int argc, char **argv) stats_line("non-terminal symbols", lem.nsymbol - lem.nterminal); stats_line("total symbols", lem.nsymbol); stats_line("rules", lem.nrule); - stats_line("states", lem.nstate); + stats_line("states", lem.nxstate); stats_line("conflicts", lem.nconflict); stats_line("action table entries", lem.nactiontab); stats_line("total table size (bytes)", lem.tablesize); @@ -3029,24 +3033,25 @@ char *tag; int PrintAction( struct action *ap, /* The action to print */ FILE *fp, /* Print the action here */ - int indent, /* Indent by this amount */ - struct rule **apRule /* All rules by index */ + int indent /* Indent by this amount */ ){ int result = 1; switch( ap->type ){ case SHIFT: { struct state *stp = ap->x.stp; - fprintf(fp,"%*s shift %-7d",indent,ap->sp->name,stp->statenum); - if( stp->nTknAct==0 && stp->nNtAct==0 && apRule ){ - fprintf(fp,"then reduce %d: ", stp->iDflt); - RulePrint(fp, apRule[stp->iDflt], -1); - } + fprintf(fp,"%*s shift %-7d",indent,ap->sp->name,stp->statenum); break; } case REDUCE: { struct rule *rp = ap->x.rp; - fprintf(fp,"%*s reduce %-7d",indent,ap->sp->name,rp->index); - if( apRule ) RulePrint(fp, apRule[rp->index], -1); + fprintf(fp,"%*s reduce %-7d",indent,ap->sp->name,rp->index); + RulePrint(fp, rp, -1); + break; + } + case SHIFTREDUCE: { + struct rule *rp = ap->x.rp; + fprintf(fp,"%*s shift-reduce %-7d",indent,ap->sp->name,rp->index); + RulePrint(fp, rp, -1); break; } case ACCEPT: @@ -3057,16 +3062,16 @@ int PrintAction( break; case SRCONFLICT: case RRCONFLICT: - fprintf(fp,"%*s reduce %-7d ** Parsing conflict **", + fprintf(fp,"%*s reduce %-7d ** Parsing conflict **", indent,ap->sp->name,ap->x.rp->index); break; case SSCONFLICT: - fprintf(fp,"%*s shift %-7d ** Parsing conflict **", + fprintf(fp,"%*s shift %-7d ** Parsing conflict **", indent,ap->sp->name,ap->x.stp->statenum); break; case SH_RESOLVED: if( showPrecedenceConflict ){ - fprintf(fp,"%*s shift %-7d -- dropped by precedence", + fprintf(fp,"%*s shift %-7d -- dropped by precedence", indent,ap->sp->name,ap->x.stp->statenum); }else{ result = 0; @@ -3087,7 +3092,7 @@ int PrintAction( return result; } -/* Generate the "y.output" log file */ +/* Generate the "*.out" log file */ void ReportOutput(struct lemon *lemp) { int i; @@ -3095,20 +3100,10 @@ void ReportOutput(struct lemon *lemp) struct config *cfp; struct action *ap; FILE *fp; - struct rule **apRule; - apRule = malloc( sizeof(apRule[0])*(lemp->nrule+1) ); - if( apRule ){ - struct rule *x; - memset(apRule, 0, sizeof(apRule[0])*(lemp->nrule+1) ); - for(x=lemp->rule; x; x=x->next){ - assert( x->index>=0 && x->index<(lemp->nrule+1) ); - apRule[x->index] = x; - } - } fp = file_open(lemp,".out","wb"); if( fp==0 ) return; - for(i=0; instate; i++){ + for(i=0; inxstate; i++){ stp = lemp->sorted[i]; fprintf(fp,"State %d:\n",stp->statenum); if( lemp->basisflag ) cfp=stp->bp; @@ -3133,7 +3128,7 @@ void ReportOutput(struct lemon *lemp) } fprintf(fp,"\n"); for(ap=stp->ap; ap; ap=ap->next){ - if( PrintAction(ap,fp,30,apRule) ) fprintf(fp,"\n"); + if( PrintAction(ap,fp,30) ) fprintf(fp,"\n"); } fprintf(fp,"\n"); } @@ -3159,7 +3154,6 @@ void ReportOutput(struct lemon *lemp) fprintf(fp, "\n"); } fclose(fp); - free(apRule); return; } @@ -3217,10 +3211,11 @@ PRIVATE int compute_action(struct lemon *lemp, struct action *ap) { int act; switch( ap->type ){ - case SHIFT: act = ap->x.stp->statenum; break; - case REDUCE: act = ap->x.rp->index + lemp->nstate; break; - case ERROR: act = lemp->nstate + lemp->nrule; break; - case ACCEPT: act = lemp->nstate + lemp->nrule + 1; break; + case SHIFT: act = ap->x.stp->statenum; break; + case SHIFTREDUCE: act = ap->x.rp->index + lemp->nstate; break; + case REDUCE: act = ap->x.rp->index + lemp->nstate+lemp->nrule; break; + case ERROR: act = lemp->nstate + lemp->nrule*2; break; + case ACCEPT: act = lemp->nstate + lemp->nrule*2 + 1; break; default: act = -1; break; } return act; @@ -3843,7 +3838,7 @@ void ReportTable( minimum_size_type(0, lemp->nsymbol+1, &szCodeType)); lineno++; fprintf(out,"#define YYNOCODE %d\n",lemp->nsymbol+1); lineno++; fprintf(out,"#define YYACTIONTYPE %s\n", - minimum_size_type(0, lemp->nstate+lemp->nrule+5, &szActionType)); lineno++; + minimum_size_type(0,lemp->nstate+lemp->nrule*2+5,&szActionType)); lineno++; if( lemp->wildcard ){ fprintf(out,"#define YYWILDCARD %d\n", lemp->wildcard->index); lineno++; @@ -3879,36 +3874,24 @@ void ReportTable( if( mhflag ){ fprintf(out,"#endif\n"); lineno++; } - fprintf(out,"#define YYNSTATE %d\n",lemp->nstate); lineno++; - fprintf(out,"#define YYNRULE %d\n",lemp->nrule); lineno++; if( lemp->errsym->useCnt ){ - fprintf(out,"#define YYERRORSYMBOL %d\n",lemp->errsym->index); lineno++; - fprintf(out,"#define YYERRSYMDT yy%d\n",lemp->errsym->dtnum); lineno++; + fprintf(out,"#define YYERRORSYMBOL %d\n",lemp->errsym->index); lineno++; + fprintf(out,"#define YYERRSYMDT yy%d\n",lemp->errsym->dtnum); lineno++; } if( lemp->has_fallback ){ fprintf(out,"#define YYFALLBACK 1\n"); lineno++; } - tplt_xfer(lemp->name,in,out,&lineno); - /* Generate the action table and its associates: - ** - ** yy_action[] A single table containing all actions. - ** yy_lookahead[] A table containing the lookahead for each entry in - ** yy_action. Used to detect hash collisions. - ** yy_shift_ofst[] For each state, the offset into yy_action for - ** shifting terminals. - ** yy_reduce_ofst[] For each state, the offset into yy_action for - ** shifting non-terminals after a reduce. - ** yy_default[] Default action for each state. + /* Compute the action table, but do not output it yet. The action + ** table must be computed before generating the YYNSTATE macro because + ** we need to know how many states can be eliminated. */ - - /* Compute the actions on all states and count them up */ - ax = (struct axset *) calloc(lemp->nstate*2, sizeof(ax[0])); + ax = (struct axset *) calloc(lemp->nxstate*2, sizeof(ax[0])); if( ax==0 ){ fprintf(stderr,"malloc failed\n"); exit(1); } - for(i=0; instate; i++){ + for(i=0; inxstate; i++){ stp = lemp->sorted[i]; ax[i*2].stp = stp; ax[i*2].isTkn = 1; @@ -3919,15 +3902,12 @@ void ReportTable( } mxTknOfst = mnTknOfst = 0; mxNtOfst = mnNtOfst = 0; - - /* Compute the action table. In order to try to keep the size of the - ** action table to a minimum, the heuristic of placing the largest action - ** sets first is used. - */ - for(i=0; instate*2; i++) ax[i].iOrder = i; - qsort(ax, lemp->nstate*2, sizeof(ax[0]), axset_compare); + /* In an effort to minimize the action table size, use the heuristic + ** of placing the largest action sets first */ + for(i=0; inxstate*2; i++) ax[i].iOrder = i; + qsort(ax, lemp->nxstate*2, sizeof(ax[0]), axset_compare); pActtab = acttab_alloc(); - for(i=0; instate*2 && ax[i].nAction>0; i++){ + for(i=0; inxstate*2 && ax[i].nAction>0; i++){ stp = ax[i].stp; if( ax[i].isTkn ){ for(ap=stp->ap; ap; ap=ap->next){ @@ -3956,6 +3936,34 @@ void ReportTable( } free(ax); + /* Finish rendering the constants now that the action table has + ** been computed */ + fprintf(out,"#define YYNSTATE %d\n",lemp->nxstate); lineno++; + fprintf(out,"#define YYNRULE %d\n",lemp->nrule); lineno++; + fprintf(out,"#define YY_MAX_SHIFT %d\n",lemp->nstate-1); lineno++; + fprintf(out,"#define YY_MIN_SHIFTREDUCE %d\n",lemp->nstate); lineno++; + i = lemp->nstate + lemp->nrule; + fprintf(out,"#define YY_MAX_SHIFTREDUCE %d\n", i-1); lineno++; + fprintf(out,"#define YY_MIN_REDUCE %d\n", i); lineno++; + i = lemp->nstate + lemp->nrule*2; + fprintf(out,"#define YY_MAX_REDUCE %d\n", i-1); lineno++; + fprintf(out,"#define YY_ERROR_ACTION %d\n", i); lineno++; + fprintf(out,"#define YY_ACCEPT_ACTION %d\n", i+1); lineno++; + fprintf(out,"#define YY_NO_ACTION %d\n", i+2); lineno++; + tplt_xfer(lemp->name,in,out,&lineno); + + /* Now output the action table and its associates: + ** + ** yy_action[] A single table containing all actions. + ** yy_lookahead[] A table containing the lookahead for each entry in + ** yy_action. Used to detect hash collisions. + ** yy_shift_ofst[] For each state, the offset into yy_action for + ** shifting terminals. + ** yy_reduce_ofst[] For each state, the offset into yy_action for + ** shifting non-terminals after a reduce. + ** yy_default[] Default action for each state. + */ + /* Output the yy_action table */ lemp->nactiontab = n = acttab_size(pActtab); lemp->tablesize += n*szActionType; @@ -3994,7 +4002,7 @@ void ReportTable( /* Output the yy_shift_ofst[] table */ fprintf(out, "#define YY_SHIFT_USE_DFLT (%d)\n", mnTknOfst-1); lineno++; - n = lemp->nstate; + n = lemp->nxstate; while( n>0 && lemp->sorted[n-1]->iTknOfst==NO_OFFSET ) n--; fprintf(out, "#define YY_SHIFT_COUNT (%d)\n", n-1); lineno++; fprintf(out, "#define YY_SHIFT_MIN (%d)\n", mnTknOfst); lineno++; @@ -4020,7 +4028,7 @@ void ReportTable( /* Output the yy_reduce_ofst[] table */ fprintf(out, "#define YY_REDUCE_USE_DFLT (%d)\n", mnNtOfst-1); lineno++; - n = lemp->nstate; + n = lemp->nxstate; while( n>0 && lemp->sorted[n-1]->iNtOfst==NO_OFFSET ) n--; fprintf(out, "#define YY_REDUCE_COUNT (%d)\n", n-1); lineno++; fprintf(out, "#define YY_REDUCE_MIN (%d)\n", mnNtOfst); lineno++; @@ -4046,12 +4054,12 @@ void ReportTable( /* Output the default action table */ fprintf(out, "static const YYACTIONTYPE yy_default[] = {\n"); lineno++; - n = lemp->nstate; + n = lemp->nxstate; lemp->tablesize += n*szActionType; for(i=j=0; isorted[i]; if( j==0 ) fprintf(out," /* %5d */ ", i); - fprintf(out, " %4d,", stp->iDflt+n); + fprintf(out, " %4d,", stp->iDfltReduce+lemp->nstate+lemp->nrule); if( j==9 || i==n-1 ){ fprintf(out, "\n"); lineno++; j = 0; @@ -4284,7 +4292,7 @@ void CompressTables(struct lemon *lemp) struct state *stp; struct action *ap, *ap2; struct rule *rp, *rp2, *rbest; - int nbest, n; + int nbest, n, nshift; int i; int usesWildcard; @@ -4332,6 +4340,32 @@ void CompressTables(struct lemon *lemp) if( ap->type==REDUCE && ap->x.rp==rbest ) ap->type = NOT_USED; } stp->ap = Action_sort(stp->ap); + + for(ap=stp->ap; ap; ap=ap->next){ + if( ap->type==SHIFT ) break; + if( ap->type==REDUCE && ap->x.rp!=rbest ) break; + } + if( ap==0 ){ + stp->autoReduce = 1; + stp->pDfltReduce = rbest; + } + } + + /* Make a second pass over all states and actions. Convert + ** every action that is a SHIFT to an autoReduce state into + ** a SHIFTREDUCE action. + */ + for(i=0; instate; i++){ + stp = lemp->sorted[i]; + for(ap=stp->ap; ap; ap=ap->next){ + struct state *pNextState; + if( ap->type!=SHIFT ) continue; + pNextState = ap->x.stp; + if( pNextState->autoReduce && pNextState->pDfltReduce!=0 ){ + ap->type = SHIFTREDUCE; + ap->x.rp = pNextState->pDfltReduce; + } + } } } @@ -4372,17 +4406,19 @@ void ResortStates(struct lemon *lemp) for(i=0; instate; i++){ stp = lemp->sorted[i]; stp->nTknAct = stp->nNtAct = 0; - stp->iDflt = lemp->nrule; + stp->iDfltReduce = lemp->nrule; /* Init dflt action to "syntax error" */ stp->iTknOfst = NO_OFFSET; stp->iNtOfst = NO_OFFSET; for(ap=stp->ap; ap; ap=ap->next){ - if( compute_action(lemp,ap)>=0 ){ + int iAction = compute_action(lemp,ap); + if( iAction>=0 ){ if( ap->sp->indexnterminal ){ stp->nTknAct++; }else if( ap->sp->indexnsymbol ){ stp->nNtAct++; }else{ - stp->iDflt = compute_action(lemp, ap) - lemp->nstate; + assert( stp->autoReduce==0 || stp->pDfltReduce==ap->x.rp ); + stp->iDfltReduce = iAction - lemp->nstate - lemp->nrule; } } } @@ -4392,6 +4428,10 @@ void ResortStates(struct lemon *lemp) for(i=0; instate; i++){ lemp->sorted[i]->statenum = i; } + lemp->nxstate = lemp->nstate; + while( lemp->nxstate>1 && lemp->sorted[lemp->nxstate-1]->autoReduce ){ + lemp->nxstate--; + } } diff --git a/tool/lempar.c b/tool/lempar.c index fe56d2dc16..ddace228f6 100644 --- a/tool/lempar.c +++ b/tool/lempar.c @@ -50,15 +50,19 @@ ** ParseARG_PDECL A parameter declaration for the %extra_argument ** ParseARG_STORE Code to store %extra_argument into yypParser ** ParseARG_FETCH Code to extract %extra_argument from yypParser -** YYNSTATE the combined number of states. -** YYNRULE the number of rules in the grammar ** YYERRORSYMBOL is the code number of the error symbol. If not ** defined, then do no error processing. +** YYNSTATE the combined number of states. +** YYNRULE the number of rules in the grammar +** YY_MAX_SHIFT Maximum value for shift actions +** YY_MIN_SHIFTREDUCE Minimum value for shift-reduce actions +** YY_MAX_SHIFTREDUCE Maximum value for shift-reduce actions +** YY_MIN_REDUCE Maximum value for reduce actions +** YY_ERROR_ACTION The yy_action[] code for syntax error +** YY_ACCEPT_ACTION The yy_action[] code for accept +** YY_NO_ACTION The yy_action[] code for no-op */ %% -#define YY_NO_ACTION (YYNSTATE+YYNRULE+2) -#define YY_ACCEPT_ACTION (YYNSTATE+YYNRULE+1) -#define YY_ERROR_ACTION (YYNSTATE+YYNRULE) /* The yyzerominor constant is used to initialize instances of ** YYMINORTYPE objects to zero. */ @@ -85,16 +89,20 @@ static const YYMINORTYPE yyzerominor = { 0 }; ** Suppose the action integer is N. Then the action is determined as ** follows ** -** 0 <= N < YYNSTATE Shift N. That is, push the lookahead +** 0 <= N <= YY_MAX_SHIFT Shift N. That is, push the lookahead ** token onto the stack and goto state N. ** -** YYNSTATE <= N < YYNSTATE+YYNRULE Reduce by rule N-YYNSTATE. +** N between YY_MIN_SHIFTREDUCE Shift to an arbitrary state then +** and YY_MAX_SHIFTREDUCE reduce by rule N-YY_MIN_SHIFTREDUCE. ** -** N == YYNSTATE+YYNRULE A syntax error has occurred. +** N between YY_MIN_REDUCE Reduce by rule N-YY_MIN_REDUCE +** and YY_MAX_REDUCE + +** N == YY_ERROR_ACTION A syntax error has occurred. ** -** N == YYNSTATE+YYNRULE+1 The parser accepts its input. +** N == YY_ACCEPT_ACTION The parser accepts its input. ** -** N == YYNSTATE+YYNRULE+2 No such action. Denotes unused +** N == YY_NO_ACTION No such action. Denotes unused ** slots in the yy_action[] table. ** ** The action table is constructed as a single large table named yy_action[]. @@ -489,9 +497,9 @@ static void yyStackOverflow(yyParser *yypParser, YYMINORTYPE *yypMinor){ } /* -** Perform a shift action. +** Perform a shift action. Return the number of errors. */ -static void yy_shift( +static int yy_shift( yyParser *yypParser, /* The parser to be shifted */ int yyNewState, /* The new state to shift in */ int yyMajor, /* The major token to shift in */ @@ -507,14 +515,14 @@ static void yy_shift( #if YYSTACKDEPTH>0 if( yypParser->yyidx>=YYSTACKDEPTH ){ yyStackOverflow(yypParser, yypMinor); - return; + return 1; } #else if( yypParser->yyidx>=yypParser->yystksz ){ yyGrowStack(yypParser); if( yypParser->yyidx>=yypParser->yystksz ){ yyStackOverflow(yypParser, yypMinor); - return; + return 1; } } #endif @@ -525,13 +533,18 @@ static void yy_shift( #ifndef NDEBUG if( yyTraceFILE && yypParser->yyidx>0 ){ int i; - fprintf(yyTraceFILE,"%sShift %d\n",yyTracePrompt,yyNewState); - fprintf(yyTraceFILE,"%sStack:",yyTracePrompt); - for(i=1; i<=yypParser->yyidx; i++) - fprintf(yyTraceFILE," %s",yyTokenName[yypParser->yystack[i].major]); - fprintf(yyTraceFILE,"\n"); + if( yyNewStateyyidx; i++) + fprintf(yyTraceFILE," %s",yyTokenName[yypParser->yystack[i].major]); + fprintf(yyTraceFILE,"\n"); + }else{ + fprintf(yyTraceFILE,"%sShift *\n",yyTracePrompt); + } } #endif + return 0; } /* The following table contains information about every rule that @@ -564,8 +577,9 @@ static void yy_reduce( #ifndef NDEBUG if( yyTraceFILE && yyruleno>=0 && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){ - fprintf(yyTraceFILE, "%sReduce [%s].\n", yyTracePrompt, - yyRuleName[yyruleno]); + yysize = yyRuleInfo[yyruleno].nrhs; + fprintf(yyTraceFILE, "%sReduce [%s] -> state %d.\n", yyTracePrompt, + yyRuleName[yyruleno], yymsp[-yysize].stateno); } #endif /* NDEBUG */ @@ -602,9 +616,8 @@ static void yy_reduce( yysize = yyRuleInfo[yyruleno].nrhs; yypParser->yyidx -= yysize; yyact = yy_find_reduce_action(yymsp[-yysize].stateno,(YYCODETYPE)yygoto); - if( yyact < YYNSTATE ){ -#ifdef NDEBUG - /* If we are not debugging and the reduce action popped at least + if( yyact < YY_MAX_SHIFTREDUCE ){ + /* If the reduce action popped at least ** one element off the stack, then we can push the new element back ** onto the stack here, and skip the stack overflow test in yy_shift(). ** That gives a significant speed improvement. */ @@ -614,13 +627,19 @@ static void yy_reduce( yymsp->stateno = (YYACTIONTYPE)yyact; yymsp->major = (YYCODETYPE)yygoto; yymsp->minor = yygotominor; - }else +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sShift %d\n",yyTracePrompt,yyact); + } #endif - { - yy_shift(yypParser,yyact,yygoto,&yygotominor); + }else{ + if( yy_shift(yypParser,yyact,yygoto,&yygotominor) ) yyact = 0; + } + if( yyact>=YY_MIN_SHIFTREDUCE ){ + yy_reduce(yypParser, yyact - YY_MIN_SHIFTREDUCE); } }else{ - assert( yyact == YYNSTATE + YYNRULE + 1 ); + assert( yyact == YY_ACCEPT_ACTION ); yy_accept(yypParser); } } @@ -740,13 +759,16 @@ void Parse( do{ yyact = yy_find_shift_action(yypParser,(YYCODETYPE)yymajor); - if( yyactyyerrcnt--; - yymajor = YYNOCODE; - }else if( yyact < YYNSTATE + YYNRULE ){ - yy_reduce(yypParser,yyact-YYNSTATE); + if( yyact <= YY_MAX_SHIFTREDUCE ){ + if( yy_shift(yypParser,yyact,yymajor,&yyminorunion)==0 ){ + yypParser->yyerrcnt--; + yymajor = YYNOCODE; + if( yyact > YY_MAX_SHIFT ){ + yy_reduce(yypParser, yyact-YY_MIN_SHIFTREDUCE); + } + } + }else if( yyact <= YY_MAX_REDUCE ){ + yy_reduce(yypParser,yyact-YY_MIN_REDUCE); }else{ assert( yyact == YY_ERROR_ACTION ); #ifdef YYERRORSYMBOL @@ -796,7 +818,7 @@ void Parse( yymx != YYERRORSYMBOL && (yyact = yy_find_reduce_action( yypParser->yystack[yypParser->yyidx].stateno, - YYERRORSYMBOL)) >= YYNSTATE + YYERRORSYMBOL)) >= YY_MIN_REDUCE ){ yy_pop_parser_stack(yypParser); } @@ -846,5 +868,10 @@ void Parse( #endif } }while( yymajor!=YYNOCODE && yypParser->yyidx>=0 ); +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sReturn\n",yyTracePrompt); + } +#endif return; } From a248a722cf721cf294bd591ccb2e7398aeb6ecdf Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 7 Sep 2015 19:52:55 +0000 Subject: [PATCH 050/100] Change the parser engine so that it (once again) waits for a lookahead token before reducing, even in a SHIFTREDUCE action. FossilOrigin-Name: 2c17a1358353a0845b039283be79353f033e2491 --- manifest | 20 ++++++------- manifest.uuid | 2 +- src/lempar.c | 76 ++++++++++++++++++++++++----------------------- src/parse.y | 2 +- src/tokenize.c | 2 +- test/misc1.test | 2 +- tool/lempar.c | 78 ++++++++++++++++++++++++++----------------------- 7 files changed, 95 insertions(+), 87 deletions(-) diff --git a/manifest b/manifest index a6e735d796..c4c835e219 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C For\sthe\sLemon-generated\sparser,\sadd\sa\snew\saction\stype\sSHIFTREDUCE\sand\suse\sit\nto\sfurther\scompress\sthe\sparser\stables\sand\simprove\sparser\sperformance. -D 2015-09-07T18:23:37.162 +C Change\sthe\sparser\sengine\sso\sthat\sit\s(once\sagain)\swaits\sfor\sa\slookahead\stoken\nbefore\sreducing,\seven\sin\sa\sSHIFTREDUCE\saction. +D 2015-09-07T19:52:55.484 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -302,7 +302,7 @@ F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 F src/insert.c 076dc5876e261a9908603d54cfc5344cd680166c F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e -F src/lempar.c 406f2e85d2552d88bd4b994cc9c567c0fbc9f483 +F src/lempar.c d98d6c2229d5ed5e628a63819fc53ed5b292d4bd F src/loadext.c dfcee8c7c032cd0fd55af3e0fc1fcfb01e426df2 F src/main.c e17fcffae4306a9b8334faf3bac80d7396850b54 F src/malloc.c 021012e28a81ffdabf4c30ec3df6ce1f6cc93f1d @@ -328,7 +328,7 @@ F src/os_win.c 40b3af7a47eb1107d0d69e592bec345a3b7b798a F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca F src/pager.c 4784012f80b2197c61ff6eaf4f5c7026d93253fd F src/pager.h 6d435f563b3f7fcae4b84433b76a6ac2730036e2 -F src/parse.y 9e1777c68d5bb483d7217327e524dcfe5263b01c +F src/parse.y f599aa5e871a493330d567ced93de696f61f48f7 F src/pcache.c 24be750c79272e0ca7b6e007bc94999700f3e5ef F src/pcache.h 9968603796240cdf83da7e7bef76edf90619cea9 F src/pcache1.c bf2afe64a3dedb8643c8dcbd94a145cc80ab2a67 @@ -395,7 +395,7 @@ F src/test_vfs.c 3b65d42e18b262805716bd96178c81da8f2d9283 F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 6bbcc9fe50c917864d48287b4792d46d6e873481 -F src/tokenize.c 9655e20ade774e5c8e580634f1359288eba3fada +F src/tokenize.c 83c6ed569423a3af83a83973b444cf7123be33a6 F src/treeview.c 154f0acc622fa3514de8777dcedf4c8a8802b4ce F src/trigger.c 322f23aad694e8f31d384dcfa386d52a48d3c52f F src/update.c 3c5bc9570df3bfafa0db36828406a8a14e4c426e @@ -866,7 +866,7 @@ F test/minmax.test 42fbad0e81afaa6e0de41c960329f2b2c3526efd F test/minmax2.test b44bae787fc7b227597b01b0ca5575c7cb54d3bc F test/minmax3.test cc1e8b010136db0d01a6f2a29ba5a9f321034354 F test/minmax4.test 936941484ebdceb8adec7c86b6cd9b6e5e897c1f -F test/misc1.test 0be38a2944dd3e63dfb2899f24446339487d17c7 +F test/misc1.test 3f1c479c5a093a6280f378c0fbff1c2701486660 F test/misc2.test 00d7de54eda90e237fc9a38b9e5ccc769ebf6d4d F test/misc3.test cf3dda47d5dda3e53fc5804a100d3c82be736c9d F test/misc4.test 0d8be3466adf123a7791a66ba2bc8e8d229e87f3 @@ -1339,7 +1339,7 @@ F tool/genfkey.README cf68fddd4643bbe3ff8e31b8b6d8b0a1b85e20f4 F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce F tool/lemon.c 0c455691cc1e59a8f782d51a83dd6bbd7c5c44e7 -F tool/lempar.c 1522366692ef87584d4eacdbb1e95bdd0bc7f3b6 +F tool/lempar.c 9bec5f85673746e5ed92084e79931afae443a6be F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862 F tool/logest.c eef612f8adf4d0993dafed0416064cf50d5d33c6 F tool/mkautoconfamal.sh d1a2da0e15b2ed33d60af35c7e9d483f13a8eb9f @@ -1383,7 +1383,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P b6ffb7e471e51ff69668154ad2c8790e466c9d37 -R 90541ca0731ebb2fb0e53dfdeffae5a4 +P 531c3974b3d586c1989cde905b2fb4681239a570 +R 9764b1afa291a8128703e02e85f419ca U drh -Z 08f2caec8f48c0982df46628b104ff5c +Z 7cd6903444e24935012e1ca631512fff diff --git a/manifest.uuid b/manifest.uuid index 1eb95138eb..5557c822f6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -531c3974b3d586c1989cde905b2fb4681239a570 \ No newline at end of file +2c17a1358353a0845b039283be79353f033e2491 \ No newline at end of file diff --git a/src/lempar.c b/src/lempar.c index d014ebcc75..f983ff752f 100644 --- a/src/lempar.c +++ b/src/lempar.c @@ -167,9 +167,13 @@ static const YYCODETYPE yyFallback[] = { ** + The semantic value stored at this level of the stack. This is ** the information used by the action routines in the grammar. ** It is sometimes called the "minor" token. +** +** After the "shift" half of a SHIFTREDUCE action, the stateno field +** actually contains the reduce action for the second half of the +** SHIFTREDUCE. */ struct yyStackEntry { - YYACTIONTYPE stateno; /* The state-number */ + YYACTIONTYPE stateno; /* The state-number, or reduce action in SHIFTREDUCE */ YYCODETYPE major; /* The major token value. This is the code ** number for the token at this stack level */ YYMINORTYPE minor; /* The user-supplied minor token value. This @@ -403,6 +407,7 @@ static int yy_find_shift_action( int i; int stateno = pParser->yystack[pParser->yyidx].stateno; + if( stateno>=YY_MIN_REDUCE ) return stateno; if( stateno>YY_SHIFT_COUNT || (i = yy_shift_ofst[stateno])==YY_SHIFT_USE_DFLT ){ return yy_default[stateno]; @@ -506,10 +511,32 @@ static void yyStackOverflow(yyParser *yypParser, YYMINORTYPE *yypMinor){ ParseARG_STORE; /* Suppress warning about unused %extra_argument var */ } +/* +** Print tracing information for a SHIFT action +*/ +#ifndef NDEBUG +static void yyTraceShift(yyParser *yypParser, int yyNewState){ + if( yyTraceFILE ){ + int i; + if( yyNewStateyyidx; i++) + fprintf(yyTraceFILE," %s",yyTokenName[yypParser->yystack[i].major]); + fprintf(yyTraceFILE,"\n"); + }else{ + fprintf(yyTraceFILE,"%sShift *\n",yyTracePrompt); + } + } +} +#else +# define yyTraceShift(X,Y) +#endif + /* ** Perform a shift action. Return the number of errors. */ -static int yy_shift( +static void yy_shift( yyParser *yypParser, /* The parser to be shifted */ int yyNewState, /* The new state to shift in */ int yyMajor, /* The major token to shift in */ @@ -525,14 +552,14 @@ static int yy_shift( #if YYSTACKDEPTH>0 if( yypParser->yyidx>=YYSTACKDEPTH ){ yyStackOverflow(yypParser, yypMinor); - return 1; + return; } #else if( yypParser->yyidx>=yypParser->yystksz ){ yyGrowStack(yypParser); if( yypParser->yyidx>=yypParser->yystksz ){ yyStackOverflow(yypParser, yypMinor); - return 1; + return; } } #endif @@ -540,21 +567,7 @@ static int yy_shift( yytos->stateno = (YYACTIONTYPE)yyNewState; yytos->major = (YYCODETYPE)yyMajor; yytos->minor = *yypMinor; -#ifndef NDEBUG - if( yyTraceFILE && yypParser->yyidx>0 ){ - int i; - if( yyNewStateyyidx; i++) - fprintf(yyTraceFILE," %s",yyTokenName[yypParser->yystack[i].major]); - fprintf(yyTraceFILE,"\n"); - }else{ - fprintf(yyTraceFILE,"%sShift *\n",yyTracePrompt); - } - } -#endif - return 0; + yyTraceShift(yypParser, yyNewState); } /* The following table contains information about every rule that @@ -628,6 +641,7 @@ static void yy_reduce( yypParser->yyidx -= yysize; yyact = yy_find_reduce_action(yymsp[-yysize].stateno,(YYCODETYPE)yygoto); if( yyact <= YY_MAX_SHIFTREDUCE ){ + if( yyact>YY_MAX_SHIFT ) yyact += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE; /* If the reduce action popped at least ** one element off the stack, then we can push the new element back ** onto the stack here, and skip the stack overflow test in yy_shift(). @@ -638,16 +652,9 @@ static void yy_reduce( yymsp->stateno = (YYACTIONTYPE)yyact; yymsp->major = (YYCODETYPE)yygoto; yymsp->minor = yygotominor; -#ifndef NDEBUG - if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sShift %d\n",yyTracePrompt,yyact); - } -#endif + yyTraceShift(yypParser, yyact); }else{ - if( yy_shift(yypParser,yyact,yygoto,&yygotominor) ) yyact = 0; - } - if( yyact>=YY_MIN_SHIFTREDUCE ){ - yy_reduce(yypParser, yyact - YY_MIN_SHIFTREDUCE); + yy_shift(yypParser,yyact,yygoto,&yygotominor); } }else{ assert( yyact == YY_ACCEPT_ACTION ); @@ -775,13 +782,10 @@ void Parse( do{ yyact = yy_find_shift_action(yypParser,(YYCODETYPE)yymajor); if( yyact <= YY_MAX_SHIFTREDUCE ){ - if( yy_shift(yypParser,yyact,yymajor,&yyminorunion)==0 ){ - yypParser->yyerrcnt--; - yymajor = YYNOCODE; - if( yyact > YY_MAX_SHIFT ){ - yy_reduce(yypParser, yyact-YY_MIN_SHIFTREDUCE); - } - } + if( yyact > YY_MAX_SHIFT ) yyact += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE; + yy_shift(yypParser,yyact,yymajor,&yyminorunion); + yypParser->yyerrcnt--; + yymajor = YYNOCODE; }else if( yyact <= YY_MAX_REDUCE ){ yy_reduce(yypParser,yyact-YY_MIN_REDUCE); }else{ @@ -883,7 +887,7 @@ void Parse( #endif } }while( yymajor!=YYNOCODE && yypParser->yyidx>=0 ); -#ifdef SQLITE_DEBUG +#ifndef NDEBUG if( yyTraceFILE ){ fprintf(yyTraceFILE,"%sReturn\n",yyTracePrompt); } diff --git a/src/parse.y b/src/parse.y index cdfef7bd3b..e99feeefc1 100644 --- a/src/parse.y +++ b/src/parse.y @@ -587,7 +587,7 @@ from(A) ::= FROM seltablist(X). { // stl_prefix(A) ::= seltablist(X) joinop(Y). { A = X; - if( A && A->nSrc>0 ) A->a[A->nSrc-1].fg.jointype = (u8)Y; + if( ALWAYS(A && A->nSrc>0) ) A->a[A->nSrc-1].fg.jointype = (u8)Y; } stl_prefix(A) ::= . {A = 0;} seltablist(A) ::= stl_prefix(X) nm(Y) dbnm(D) as(Z) indexed_opt(I) diff --git a/src/tokenize.c b/src/tokenize.c index 30a8ad06d0..6b5ad27901 100644 --- a/src/tokenize.c +++ b/src/tokenize.c @@ -459,12 +459,12 @@ abort_parse: assert( zSql[i]==0 ); if( lastTokenParsed!=TK_SEMI ){ sqlite3Parser(pEngine, TK_SEMI, pParse->sLastToken, pParse); + pParse->zTail = &zSql[i]; } if( pParse->rc==SQLITE_OK && db->mallocFailed==0 ){ sqlite3Parser(pEngine, 0, pParse->sLastToken, pParse); } } - pParse->zTail = &zSql[i]; #ifdef YYTRACKMAXSTACKDEPTH sqlite3_mutex_enter(sqlite3MallocMutex()); sqlite3StatusSet(SQLITE_STATUS_PARSER_STACK, diff --git a/test/misc1.test b/test/misc1.test index be4ab8646d..25e9bd813e 100644 --- a/test/misc1.test +++ b/test/misc1.test @@ -646,7 +646,7 @@ do_catchsql_test misc1-21.1 { } {1 {near "#0": syntax error}} do_catchsql_test misc1-21.2 { VALUES(0,0x0MATCH#0; -} {1 {near "#0": syntax error}} +} {1 {near ";": syntax error}} # 2015-04-15 do_execsql_test misc1-22.1 { diff --git a/tool/lempar.c b/tool/lempar.c index ddace228f6..161632a6c7 100644 --- a/tool/lempar.c +++ b/tool/lempar.c @@ -161,9 +161,13 @@ static const YYCODETYPE yyFallback[] = { ** + The semantic value stored at this level of the stack. This is ** the information used by the action routines in the grammar. ** It is sometimes called the "minor" token. +** +** After the "shift" half of a SHIFTREDUCE action, the stateno field +** actually contains the reduce action for the second half of the +** SHIFTREDUCE. */ struct yyStackEntry { - YYACTIONTYPE stateno; /* The state-number */ + YYACTIONTYPE stateno; /* The state-number, or reduce action in SHIFTREDUCE */ YYCODETYPE major; /* The major token value. This is the code ** number for the token at this stack level */ YYMINORTYPE minor; /* The user-supplied minor token value. This @@ -392,7 +396,8 @@ static int yy_find_shift_action( ){ int i; int stateno = pParser->yystack[pParser->yyidx].stateno; - + + if( stateno>=YY_MIN_REDUCE ) return stateno; if( stateno>YY_SHIFT_COUNT || (i = yy_shift_ofst[stateno])==YY_SHIFT_USE_DFLT ){ return yy_default[stateno]; @@ -496,10 +501,32 @@ static void yyStackOverflow(yyParser *yypParser, YYMINORTYPE *yypMinor){ ParseARG_STORE; /* Suppress warning about unused %extra_argument var */ } +/* +** Print tracing information for a SHIFT action +*/ +#ifndef NDEBUG +static void yyTraceShift(yyParser *yypParser, int yyNewState){ + if( yyTraceFILE ){ + int i; + if( yyNewStateyyidx; i++) + fprintf(yyTraceFILE," %s",yyTokenName[yypParser->yystack[i].major]); + fprintf(yyTraceFILE,"\n"); + }else{ + fprintf(yyTraceFILE,"%sShift *\n",yyTracePrompt); + } + } +} +#else +# define yyTraceShift(X,Y) +#endif + /* ** Perform a shift action. Return the number of errors. */ -static int yy_shift( +static void yy_shift( yyParser *yypParser, /* The parser to be shifted */ int yyNewState, /* The new state to shift in */ int yyMajor, /* The major token to shift in */ @@ -515,14 +542,14 @@ static int yy_shift( #if YYSTACKDEPTH>0 if( yypParser->yyidx>=YYSTACKDEPTH ){ yyStackOverflow(yypParser, yypMinor); - return 1; + return; } #else if( yypParser->yyidx>=yypParser->yystksz ){ yyGrowStack(yypParser); if( yypParser->yyidx>=yypParser->yystksz ){ yyStackOverflow(yypParser, yypMinor); - return 1; + return; } } #endif @@ -530,21 +557,7 @@ static int yy_shift( yytos->stateno = (YYACTIONTYPE)yyNewState; yytos->major = (YYCODETYPE)yyMajor; yytos->minor = *yypMinor; -#ifndef NDEBUG - if( yyTraceFILE && yypParser->yyidx>0 ){ - int i; - if( yyNewStateyyidx; i++) - fprintf(yyTraceFILE," %s",yyTokenName[yypParser->yystack[i].major]); - fprintf(yyTraceFILE,"\n"); - }else{ - fprintf(yyTraceFILE,"%sShift *\n",yyTracePrompt); - } - } -#endif - return 0; + yyTraceShift(yypParser, yyNewState); } /* The following table contains information about every rule that @@ -616,7 +629,8 @@ static void yy_reduce( yysize = yyRuleInfo[yyruleno].nrhs; yypParser->yyidx -= yysize; yyact = yy_find_reduce_action(yymsp[-yysize].stateno,(YYCODETYPE)yygoto); - if( yyact < YY_MAX_SHIFTREDUCE ){ + if( yyact <= YY_MAX_SHIFTREDUCE ){ + if( yyact>YY_MAX_SHIFT ) yyact += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE; /* If the reduce action popped at least ** one element off the stack, then we can push the new element back ** onto the stack here, and skip the stack overflow test in yy_shift(). @@ -627,16 +641,9 @@ static void yy_reduce( yymsp->stateno = (YYACTIONTYPE)yyact; yymsp->major = (YYCODETYPE)yygoto; yymsp->minor = yygotominor; -#ifndef NDEBUG - if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sShift %d\n",yyTracePrompt,yyact); - } -#endif + yyTraceShift(yypParser, yyact); }else{ - if( yy_shift(yypParser,yyact,yygoto,&yygotominor) ) yyact = 0; - } - if( yyact>=YY_MIN_SHIFTREDUCE ){ - yy_reduce(yypParser, yyact - YY_MIN_SHIFTREDUCE); + yy_shift(yypParser,yyact,yygoto,&yygotominor); } }else{ assert( yyact == YY_ACCEPT_ACTION ); @@ -760,13 +767,10 @@ void Parse( do{ yyact = yy_find_shift_action(yypParser,(YYCODETYPE)yymajor); if( yyact <= YY_MAX_SHIFTREDUCE ){ - if( yy_shift(yypParser,yyact,yymajor,&yyminorunion)==0 ){ - yypParser->yyerrcnt--; - yymajor = YYNOCODE; - if( yyact > YY_MAX_SHIFT ){ - yy_reduce(yypParser, yyact-YY_MIN_SHIFTREDUCE); - } - } + if( yyact > YY_MAX_SHIFT ) yyact += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE; + yy_shift(yypParser,yyact,yymajor,&yyminorunion); + yypParser->yyerrcnt--; + yymajor = YYNOCODE; }else if( yyact <= YY_MAX_REDUCE ){ yy_reduce(yypParser,yyact-YY_MIN_REDUCE); }else{ From ae2a4084a238ade6c977f04906983c74b5e9fa60 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 7 Sep 2015 20:02:39 +0000 Subject: [PATCH 051/100] Fix an unreachable branch in the new parse automaton. FossilOrigin-Name: e9d604b4306a86faae315ac3cba59bf07d1b665c --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/lempar.c | 7 +++---- tool/lempar.c | 7 +++---- 4 files changed, 14 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index c4c835e219..41384f854b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\sparser\sengine\sso\sthat\sit\s(once\sagain)\swaits\sfor\sa\slookahead\stoken\nbefore\sreducing,\seven\sin\sa\sSHIFTREDUCE\saction. -D 2015-09-07T19:52:55.484 +C Fix\san\sunreachable\sbranch\sin\sthe\snew\sparse\sautomaton. +D 2015-09-07T20:02:39.726 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -302,7 +302,7 @@ F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 F src/insert.c 076dc5876e261a9908603d54cfc5344cd680166c F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e -F src/lempar.c d98d6c2229d5ed5e628a63819fc53ed5b292d4bd +F src/lempar.c d344a95d60c24e2f490ee59db9784b1b17439012 F src/loadext.c dfcee8c7c032cd0fd55af3e0fc1fcfb01e426df2 F src/main.c e17fcffae4306a9b8334faf3bac80d7396850b54 F src/malloc.c 021012e28a81ffdabf4c30ec3df6ce1f6cc93f1d @@ -1339,7 +1339,7 @@ F tool/genfkey.README cf68fddd4643bbe3ff8e31b8b6d8b0a1b85e20f4 F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce F tool/lemon.c 0c455691cc1e59a8f782d51a83dd6bbd7c5c44e7 -F tool/lempar.c 9bec5f85673746e5ed92084e79931afae443a6be +F tool/lempar.c 3617143ddb9b176c3605defe6a9c798793280120 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862 F tool/logest.c eef612f8adf4d0993dafed0416064cf50d5d33c6 F tool/mkautoconfamal.sh d1a2da0e15b2ed33d60af35c7e9d483f13a8eb9f @@ -1383,7 +1383,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 531c3974b3d586c1989cde905b2fb4681239a570 -R 9764b1afa291a8128703e02e85f419ca +P 2c17a1358353a0845b039283be79353f033e2491 +R c7dbe034ff98374fe795187c44f7caee U drh -Z 7cd6903444e24935012e1ca631512fff +Z f008d623d9e778806c4a18c7c6cec2d4 diff --git a/manifest.uuid b/manifest.uuid index 5557c822f6..0b0630a9a0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2c17a1358353a0845b039283be79353f033e2491 \ No newline at end of file +e9d604b4306a86faae315ac3cba59bf07d1b665c \ No newline at end of file diff --git a/src/lempar.c b/src/lempar.c index f983ff752f..5e5a11aeaa 100644 --- a/src/lempar.c +++ b/src/lempar.c @@ -408,10 +408,9 @@ static int yy_find_shift_action( int stateno = pParser->yystack[pParser->yyidx].stateno; if( stateno>=YY_MIN_REDUCE ) return stateno; - if( stateno>YY_SHIFT_COUNT - || (i = yy_shift_ofst[stateno])==YY_SHIFT_USE_DFLT ){ - return yy_default[stateno]; - } + assert( stateno <= YY_SHIFT_COUNT ); + i = yy_shift_ofst[stateno]; + if( i==YY_SHIFT_USE_DFLT ) return yy_default[stateno]; assert( iLookAhead!=YYNOCODE ); i += iLookAhead; if( i<0 || i>=YY_ACTTAB_COUNT || yy_lookahead[i]!=iLookAhead ){ diff --git a/tool/lempar.c b/tool/lempar.c index 161632a6c7..cdf4ca5a1a 100644 --- a/tool/lempar.c +++ b/tool/lempar.c @@ -398,10 +398,9 @@ static int yy_find_shift_action( int stateno = pParser->yystack[pParser->yyidx].stateno; if( stateno>=YY_MIN_REDUCE ) return stateno; - if( stateno>YY_SHIFT_COUNT - || (i = yy_shift_ofst[stateno])==YY_SHIFT_USE_DFLT ){ - return yy_default[stateno]; - } + assert( stateno <= YY_SHIFT_COUNT ); + i = yy_shift_ofst[stateno]; + if( i==YY_SHIFT_USE_DFLT ) return yy_default[stateno]; assert( iLookAhead!=YYNOCODE ); i += iLookAhead; if( i<0 || i>=YY_ACTTAB_COUNT || yy_lookahead[i]!=iLookAhead ){ From 337cd0def4adc137f26bd47dd10d6e0503f06465 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 7 Sep 2015 23:40:42 +0000 Subject: [PATCH 052/100] Minor tweaks to Lemon. FossilOrigin-Name: 986677224a8da5e79fbbd90673f1b595da89c5d6 --- manifest | 13 ++++++------- manifest.uuid | 2 +- tool/lemon.c | 14 ++++++++++++-- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 8495e41bfe..9ab7e944c5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sthe\sLemon\sparser\sgenerator\sto\sadd\sSHIFTREDUCE\sstates\sthat\sreduce\sthe\nsizes\sof\ssome\sof\sthe\sparser\stables. -D 2015-09-07T20:11:30.937 +C Minor\stweaks\sto\sLemon. +D 2015-09-07T23:40:42.492 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1338,7 +1338,7 @@ F tool/fuzzershell.c f2fc86dd22df654b28851b85019d3bd007361751 F tool/genfkey.README cf68fddd4643bbe3ff8e31b8b6d8b0a1b85e20f4 F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce -F tool/lemon.c 0c455691cc1e59a8f782d51a83dd6bbd7c5c44e7 +F tool/lemon.c d7c82de603f5cd5a8b84a522f8170a73d00dce68 F tool/lempar.c 3617143ddb9b176c3605defe6a9c798793280120 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862 F tool/logest.c eef612f8adf4d0993dafed0416064cf50d5d33c6 @@ -1383,8 +1383,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P b6ffb7e471e51ff69668154ad2c8790e466c9d37 e9d604b4306a86faae315ac3cba59bf07d1b665c -R c7dbe034ff98374fe795187c44f7caee -T +closed e9d604b4306a86faae315ac3cba59bf07d1b665c +P 99b992fa840707711d99f8d05b62412f7008cd93 +R a2ce2475394d66c086e10e908ff421dc U drh -Z 0877e196b03066e842bcbf4d8e5b634a +Z ef0c58cd9e9be4bb6c273d1f71ee47c5 diff --git a/manifest.uuid b/manifest.uuid index 299179057e..f5ab1cac86 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -99b992fa840707711d99f8d05b62412f7008cd93 \ No newline at end of file +986677224a8da5e79fbbd90673f1b595da89c5d6 \ No newline at end of file diff --git a/tool/lemon.c b/tool/lemon.c index 2bb3d5ef58..e24fe4212c 100644 --- a/tool/lemon.c +++ b/tool/lemon.c @@ -3753,7 +3753,7 @@ static int axset_compare(const void *a, const void *b){ int c; c = p2->nAction - p1->nAction; if( c==0 ){ - c = p2->iOrder - p1->iOrder; + c = p1->iOrder - p2->iOrder; } assert( c!=0 || p1==p2 ); return c; @@ -3933,6 +3933,16 @@ void ReportTable( if( stp->iNtOfstiNtOfst; if( stp->iNtOfst>mxNtOfst ) mxNtOfst = stp->iNtOfst; } +#if 0 /* Uncomment for a trace of how the yy_action[] table fills out */ + { int jj, nn; + for(jj=nn=0; jjnAction; jj++){ + if( pActtab->aAction[jj].action<0 ) nn++; + } + printf("%4d: State %3d %s n: %2d size: %5d freespace: %d\n", + i, stp->statenum, ax[i].isTkn ? "Token" : "Var ", + ax[i].nAction, pActtab->nAction, nn); + } +#endif } free(ax); @@ -3940,7 +3950,7 @@ void ReportTable( ** been computed */ fprintf(out,"#define YYNSTATE %d\n",lemp->nxstate); lineno++; fprintf(out,"#define YYNRULE %d\n",lemp->nrule); lineno++; - fprintf(out,"#define YY_MAX_SHIFT %d\n",lemp->nstate-1); lineno++; + fprintf(out,"#define YY_MAX_SHIFT %d\n",lemp->nxstate-1); lineno++; fprintf(out,"#define YY_MIN_SHIFTREDUCE %d\n",lemp->nstate); lineno++; i = lemp->nstate + lemp->nrule; fprintf(out,"#define YY_MAX_SHIFTREDUCE %d\n", i-1); lineno++; From 310a8d666872b205ccf8cdd5ecbf3ebbcec92b71 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 8 Sep 2015 17:31:30 +0000 Subject: [PATCH 053/100] Fix the help message that sqlite3_analyzer.exe generates for invalid arguments. FossilOrigin-Name: 33a14e7be1004abca7a30f675459138d7f8d72b1 --- manifest | 12 ++++++------ manifest.uuid | 2 +- tool/spaceanal.tcl | 14 +++++++++++++- 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 9ab7e944c5..72d1042163 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\stweaks\sto\sLemon. -D 2015-09-07T23:40:42.492 +C Fix\sthe\shelp\smessage\sthat\ssqlite3_analyzer.exe\sgenerates\sfor\sinvalid\narguments. +D 2015-09-08T17:31:30.826 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1365,7 +1365,7 @@ F tool/showstat4.c 9515faa8ec176599d4a8288293ba8ec61f7b728a F tool/showwal.c 85cb36d4fe3e93e2fbd63e786e0d1ce42d0c4fad F tool/soak1.tcl 8d407956e1a45b485a8e072470a3e629a27037fe F tool/space_used.tcl f714c41a59e326b8b9042f415b628b561bafa06b -F tool/spaceanal.tcl 63a415385a66fdbf736bfd204a31c6d851ed8da6 +F tool/spaceanal.tcl 93c1fdc9733c525b17a2024c7df193daa002e037 F tool/speedtest.tcl 06c76698485ccf597b9e7dbb1ac70706eb873355 F tool/speedtest16.c c8a9c793df96db7e4933f0852abb7a03d48f2e81 F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff @@ -1383,7 +1383,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 99b992fa840707711d99f8d05b62412f7008cd93 -R a2ce2475394d66c086e10e908ff421dc +P 986677224a8da5e79fbbd90673f1b595da89c5d6 +R 53efe0392669648bee0c2878b1555bb5 U drh -Z ef0c58cd9e9be4bb6c273d1f71ee47c5 +Z cf7dc4ba83b64f541e9e5f6eb6b8039d diff --git a/manifest.uuid b/manifest.uuid index f5ab1cac86..04b63a44c2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -986677224a8da5e79fbbd90673f1b595da89c5d6 \ No newline at end of file +33a14e7be1004abca7a30f675459138d7f8d72b1 \ No newline at end of file diff --git a/tool/spaceanal.tcl b/tool/spaceanal.tcl index dea2723397..38d954162e 100644 --- a/tool/spaceanal.tcl +++ b/tool/spaceanal.tcl @@ -26,7 +26,19 @@ proc is_without_rowid {tname} { # proc usage {} { set argv0 [file rootname [file tail [info nameofexecutable]]] - puts stderr "Usage: $argv0 \[--pageinfo] \[--stats] database-name" + puts stderr "Usage: $argv0 ?--pageinfo? ?--stats? database-filename" + puts stderr { +Analyze the SQLite3 database file specified by the "database-filename" +argument and output a report detailing size and storage efficiency +information for the database and its constituent tables and indexes. + +Options: + + --stats Output SQL text that creates a new database containing + statistics about the database that was analyzed + + --pageinfo Show how each page of the database-file is used +} exit 1 } set file_to_analyze {} From fe8e2eba0a64aa1234fecc4efa5a9c3f7efed202 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 8 Sep 2015 19:55:26 +0000 Subject: [PATCH 054/100] Remove the 0x00 terminators from the end of doclists stored on disk. FossilOrigin-Name: 00d990061dec3661b0376bd167082942d5563bfe --- ext/fts5/fts5Int.h | 10 + ext/fts5/fts5_buffer.c | 6 +- ext/fts5/fts5_config.c | 3 + ext/fts5/fts5_index.c | 454 ++++++++++++++++++---------------- ext/fts5/fts5_main.c | 4 + ext/fts5/test/fts5aa.test | 5 - ext/fts5/test/fts5simple.test | 53 +++- main.mk | 4 +- manifest | 26 +- manifest.uuid | 2 +- 10 files changed, 323 insertions(+), 244 deletions(-) diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index 837ecb1ccd..c52ce67d70 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -117,6 +117,12 @@ typedef struct Fts5Config Fts5Config; ** bColumnsize: ** True if the %_docsize table is created. ** +** bPrefixIndex: +** This is only used for debugging. If set to false, any prefix indexes +** are ignored. This value is configured using: +** +** INSERT INTO tbl(tbl, rank) VALUES('prefix-index', $bPrefixIndex); +** */ struct Fts5Config { sqlite3 *db; /* Database handle */ @@ -145,6 +151,10 @@ struct Fts5Config { /* If non-NULL, points to sqlite3_vtab.base.zErrmsg. Often NULL. */ char **pzErrmsg; + +#ifdef SQLITE_DEBUG + int bPrefixIndex; /* True to use prefix-indexes */ +#endif }; /* Current expected value of %_config table 'version' field */ diff --git a/ext/fts5/fts5_buffer.c b/ext/fts5/fts5_buffer.c index 07e1243c36..1a7c0d0f8a 100644 --- a/ext/fts5/fts5_buffer.c +++ b/ext/fts5/fts5_buffer.c @@ -16,12 +16,14 @@ #include "fts5Int.h" int sqlite3Fts5BufferGrow(int *pRc, Fts5Buffer *pBuf, int nByte){ - /* A no-op if an error has already occurred */ - if( *pRc ) return 1; if( (pBuf->n + nByte) > pBuf->nSpace ){ u8 *pNew; int nNew = pBuf->nSpace ? pBuf->nSpace*2 : 64; + + /* A no-op if an error has already occurred */ + if( *pRc ) return 1; + while( nNew<(pBuf->n + nByte) ){ nNew = nNew * 2; } diff --git a/ext/fts5/fts5_config.c b/ext/fts5/fts5_config.c index 74faf6dd30..19e3d3ab1f 100644 --- a/ext/fts5/fts5_config.c +++ b/ext/fts5/fts5_config.c @@ -480,6 +480,9 @@ int sqlite3Fts5ConfigParse( pRet->zDb = sqlite3Fts5Strndup(&rc, azArg[1], -1); pRet->zName = sqlite3Fts5Strndup(&rc, azArg[2], -1); pRet->bColumnsize = 1; +#ifdef SQLITE_DEBUG + pRet->bPrefixIndex = 1; +#endif if( rc==SQLITE_OK && sqlite3_stricmp(pRet->zName, FTS5_RANK_NAME)==0 ){ *pzErr = sqlite3_mprintf("reserved fts5 table name: %s", pRet->zName); rc = SQLITE_ERROR; diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 87eb54b125..9bc08e7443 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -475,6 +475,9 @@ struct Fts5CResult { ** For each rowid on the page corresponding to the current term, the ** corresponding aRowidOffset[] entry is set to the byte offset of the ** start of the "position-list-size" field within the page. +** +** iTermIdx: +** Index of current term on iTermLeafPgno. */ struct Fts5SegIter { Fts5StructureSegment *pSeg; /* Segment to iterate through */ @@ -489,6 +492,9 @@ struct Fts5SegIter { int iTermLeafPgno; int iTermLeafOffset; + int iTermIdx; + int iEndofDoclist; + /* The following are only used if the FTS5_SEGITER_REVERSE flag is set. */ int iRowidOffset; /* Current entry in aRowidOffset[] */ int nRowidOffset; /* Allocated size of aRowidOffset[] array */ @@ -508,7 +514,7 @@ struct Fts5SegIter { ** leaf page. */ #define ASSERT_SZLEAF_OK(x) assert( \ - (x)->szLeaf==fts5GetU16(&(x)->p[2]) || (x)->szLeaf==(x)->nn \ + (x)->szLeaf==(x)->nn || (x)->szLeaf==fts5GetU16(&(x)->p[2]) \ ) #define FTS5_SEGITER_ONETERM 0x01 @@ -522,9 +528,12 @@ struct Fts5SegIter { */ #define fts5LeafIsTermless(x) ((x)->szLeaf >= (x)->nn) -#define fts5LeafFirstTermOff(x) (fts5GetU16(&(x)->p[(x)->szLeaf])) +#define fts5LeafTermOff(x, i) (fts5GetU16(&(x)->p[(x)->szLeaf + (i)*2])) + #define fts5LeafFirstRowidOff(x) (fts5GetU16((x)->p)) +#define fts5LeafFirstTermOff(x) fts5LeafTermOff(x, 0) + /* ** poslist: ** Used by sqlite3Fts5IterPoslist() when the poslist needs to be buffered. @@ -1153,8 +1162,9 @@ static void fts5StructurePromote( int szPromote = 0; /* Promote anything this size or smaller */ Fts5StructureSegment *pSeg; /* Segment just written */ int szSeg; /* Size of segment just written */ + int nSeg = pStruct->aLevel[iLvl].nSeg; - + if( nSeg==0 ) return; pSeg = &pStruct->aLevel[iLvl].aSeg[pStruct->aLevel[iLvl].nSeg-1]; szSeg = (1 + pSeg->pgnoLast - pSeg->pgnoFirst); @@ -1464,6 +1474,14 @@ static void fts5SegIterNextPage( }else{ pIter->pLeaf = 0; } + + if( pIter->pLeaf ){ + if( fts5LeafIsTermless(pIter->pLeaf) ){ + pIter->iEndofDoclist = pIter->pLeaf->nn+1; + }else{ + pIter->iEndofDoclist = fts5LeafFirstTermOff(pIter->pLeaf); + } + } } /* @@ -1505,6 +1523,20 @@ static void fts5SegIterLoadNPos(Fts5Index *p, Fts5SegIter *pIter){ } } +static void fts5SegIterLoadEod(Fts5Index *p, Fts5SegIter *pIter){ + Fts5Data *pLeaf = pIter->pLeaf; + int nPg = (pLeaf->nn - pLeaf->szLeaf) / 2; + + assert( pIter->iLeafPgno==pIter->iTermLeafPgno ); + if( (pIter->iTermIdx+1)szLeaf + (pIter->iTermIdx + 1) * 2; + pIter->iEndofDoclist = fts5GetU16(&pLeaf->p[iRead]); + }else{ + pIter->iEndofDoclist = pLeaf->nn+1; + } + +} + static void fts5SegIterLoadRowid(Fts5Index *p, Fts5SegIter *pIter){ u8 *a = pIter->pLeaf->p; /* Buffer to read data from */ int iOff = pIter->iLeafOffset; @@ -1551,6 +1583,7 @@ static void fts5SegIterLoadTerm(Fts5Index *p, Fts5SegIter *pIter, int nKeep){ pIter->iTermLeafPgno = pIter->iLeafPgno; pIter->iLeafOffset = iOff; + fts5SegIterLoadEod(p, pIter); fts5SegIterLoadRowid(p, pIter); } @@ -1585,9 +1618,10 @@ static void fts5SegIterInit( } if( p->rc==SQLITE_OK ){ - u8 *a = pIter->pLeaf->p; pIter->iLeafOffset = 4; + assert_nc( pIter->pLeaf->nn>4 ); assert( fts5LeafFirstTermOff(pIter->pLeaf)==4 ); + pIter->iTermIdx = 0; fts5SegIterLoadTerm(p, pIter, 0); fts5SegIterLoadNPos(p, pIter); } @@ -1614,6 +1648,10 @@ static void fts5SegIterReverseInitPage(Fts5Index *p, Fts5SegIter *pIter){ u8 *a = pIter->pLeaf->p; int iRowidOffset = 0; + if( n>pIter->iEndofDoclist ){ + n = pIter->iEndofDoclist; + } + ASSERT_SZLEAF_OK(pIter->pLeaf); while( 1 ){ i64 iDelta = 0; @@ -1624,7 +1662,6 @@ static void fts5SegIterReverseInitPage(Fts5Index *p, Fts5SegIter *pIter){ i += nPos; if( i>=n ) break; i += fts5GetVarint(&a[i], (u64*)&iDelta); - if( iDelta==0 ) break; pIter->iRowid += iDelta; if( iRowidOffset>=pIter->nRowidOffset ){ @@ -1667,8 +1704,8 @@ static void fts5SegIterReverseNewPage(Fts5Index *p, Fts5SegIter *pIter){ pIter->iLeafOffset = pIter->iTermLeafOffset; } }else{ - int iRowidOff, dummy; - fts5LeafHeader(pNew, &iRowidOff, &dummy); + int iRowidOff; + iRowidOff = fts5LeafFirstRowidOff(pNew); if( iRowidOff ){ pIter->pLeaf = pNew; pIter->iLeafOffset = iRowidOff; @@ -1686,6 +1723,7 @@ static void fts5SegIterReverseNewPage(Fts5Index *p, Fts5SegIter *pIter){ } if( pIter->pLeaf ){ + pIter->iEndofDoclist = pIter->pLeaf->nn+1; fts5SegIterReverseInitPage(p, pIter); } } @@ -1747,21 +1785,26 @@ static void fts5SegIterNext( iOff = pIter->iLeafOffset + pIter->nPos; if( iOffiLeafOffset = iOff; - if( iDelta==0 ){ + /* The next entry is on the current page. */ + assert_nc( iOff<=pIter->iEndofDoclist ); + if( iOff>=pIter->iEndofDoclist ){ bNewTerm = 1; - if( iOff>=n ){ - fts5SegIterNextPage(p, pIter); - pIter->iLeafOffset = 4; - }else if( iOff!=fts5LeafFirstTermOff(pLeaf) ){ - pIter->iLeafOffset += fts5GetVarint32(&a[iOff], nKeep); + if( pIter->iTermLeafPgno==pIter->iLeafPgno ){ + pIter->iTermIdx++; + }else{ + pIter->iTermIdx = 0; + } + if( iOff!=fts5LeafFirstTermOff(pLeaf) ){ + iOff += fts5GetVarint32(&a[iOff], nKeep); } }else{ + u64 iDelta; + iOff += sqlite3Fts5GetVarint(&a[iOff], &iDelta); pIter->iRowid += iDelta; + assert_nc( iDelta>0 ); } + pIter->iLeafOffset = iOff; + }else if( pIter->pSeg==0 ){ const u8 *pList = 0; const char *zTerm = 0; @@ -1777,6 +1820,7 @@ static void fts5SegIterNext( pIter->pLeaf->p = (u8*)pList; pIter->pLeaf->nn = nList; pIter->pLeaf->szLeaf = nList; + pIter->iEndofDoclist = nList+1; sqlite3Fts5BufferSet(&p->rc, &pIter->term, strlen(zTerm), (u8*)zTerm); pIter->iLeafOffset = fts5GetVarint(pList, (u64*)&pIter->iRowid); } @@ -1796,6 +1840,7 @@ static void fts5SegIterNext( iOff = fts5LeafFirstTermOff(pLeaf); pIter->iLeafOffset = iOff; bNewTerm = 1; + pIter->iTermIdx = 0; } if( iOff>=pLeaf->szLeaf ){ p->rc = FTS5_CORRUPT; @@ -1847,31 +1892,11 @@ static void fts5SegIterReverse(Fts5Index *p, Fts5SegIter *pIter){ ** byte of position-list content for the current rowid. Back it up ** so that it points to the start of the position-list size field. */ pIter->iLeafOffset -= sqlite3Fts5GetVarintLen(pIter->nPos*2+pIter->bDel); - iOff = pIter->iLeafOffset; - assert( iOff>=4 ); - - /* Search for a new term within the current leaf. If one can be found, - ** then this page contains the largest rowid for the current term. */ - while( iOffszLeaf ){ - int nPos; - i64 iDelta; - int bDummy; - - /* Read the position-list size field */ - iOff += fts5GetPoslistSize(&pLeaf->p[iOff], &nPos, &bDummy); - iOff += nPos; - if( iOff>=pLeaf->szLeaf ) break; - - /* Rowid delta. Or, if 0x00, the end of doclist marker. */ - nPos = fts5GetVarint(&pLeaf->p[iOff], (u64*)&iDelta); - if( iDelta==0 ) break; - iOff += nPos; - } /* If this condition is true then the largest rowid for the current ** term may not be stored on the current page. So search forward to ** see where said rowid really is. */ - if( iOff>=pLeaf->szLeaf ){ + if( pIter->iEndofDoclist>=pLeaf->szLeaf ){ int pgno; Fts5StructureSegment *pSeg = pIter->pSeg; @@ -1905,14 +1930,20 @@ static void fts5SegIterReverse(Fts5Index *p, Fts5SegIter *pIter){ ** first rowid on this page. */ if( pLast ){ - int dummy; int iOff; fts5DataRelease(pIter->pLeaf); pIter->pLeaf = pLast; pIter->iLeafPgno = pgnoLast; - fts5LeafHeader(pLast, &iOff, &dummy); + iOff = fts5LeafFirstRowidOff(pLast); iOff += fts5GetVarint(&pLast->p[iOff], (u64*)&pIter->iRowid); pIter->iLeafOffset = iOff; + + if( fts5LeafIsTermless(pLast) ){ + pIter->iEndofDoclist = pLast->nn+1; + }else{ + pIter->iEndofDoclist = fts5LeafTermOff(pLast, 0); + } + } fts5SegIterReverseInitPage(p, pIter); @@ -1935,30 +1966,20 @@ static void fts5SegIterLoadDlidx(Fts5Index *p, Fts5SegIter *pIter){ /* Check if the current doclist ends on this page. If it does, return ** early without loading the doclist-index (as it belongs to a different ** term. */ - if( pIter->iTermLeafPgno==pIter->iLeafPgno ){ - int iOff = pIter->iLeafOffset + pIter->nPos; - while( iOffszLeaf ){ - int bDummy; - int nPos; - i64 iDelta; - - /* iOff is currently the offset of the start of position list data */ - iOff += fts5GetVarint(&pLeaf->p[iOff], (u64*)&iDelta); - if( iDelta==0 ) return; - assert_nc( iOffszLeaf ); - iOff += fts5GetPoslistSize(&pLeaf->p[iOff], &nPos, &bDummy); - iOff += nPos; - } + if( pIter->iTermLeafPgno==pIter->iLeafPgno + && pIter->iEndofDoclistszLeaf + ){ + return; } pIter->pDlidx = fts5DlidxIterInit(p, bRev, iSeg, pIter->iTermLeafPgno); } #define fts5IndexGetVarint32(a, iOff, nVal) { \ - nVal = a[iOff++]; \ + nVal = (a)[iOff++]; \ if( nVal & 0x80 ){ \ iOff--; \ - iOff += fts5GetVarint32(&a[iOff], nVal); \ + iOff += fts5GetVarint32(&(a)[iOff], nVal); \ } \ } @@ -2037,41 +2058,12 @@ static void fts5LeafSeek( } iOff += nNew; -#if 0 - /* Skip past the doclist. If the end of the page is reached, bail out. */ - while( 1 ){ - int nPos; - - /* Skip past rowid delta */ - fts5IndexSkipVarint(a, iOff); - - /* Skip past position list */ - fts5IndexGetVarint32(a, iOff, nPos); - iOff += (nPos >> 1); - if( iOff>=(n-1) ){ - iOff = n; - goto search_failed; - } - - /* If this is the end of the doclist, break out of the loop */ - if( a[iOff]==0x00 ){ - iOff++; - break; - } - }; - - iTerm++; - assert( iTerm=nPgTerm ){ iOff = n; break; } iOff = fts5GetU16(&a[n + iTerm*2]); -#endif /* Read the nKeep field of the next term. */ fts5IndexGetVarint32(a, iOff, nKeep); @@ -2084,6 +2076,7 @@ static void fts5LeafSeek( return; }else if( iOff>=n ){ do { + iTerm = 0; fts5SegIterNextPage(p, pIter); if( pIter->pLeaf==0 ) return; a = pIter->pLeaf->p; @@ -2101,6 +2094,8 @@ static void fts5LeafSeek( } search_success: + pIter->iTermIdx = iTerm; + pIter->iLeafOffset = iOff + nNew; pIter->iTermLeafOffset = pIter->iLeafOffset; pIter->iTermLeafPgno = pIter->iLeafPgno; @@ -2108,6 +2103,7 @@ static void fts5LeafSeek( fts5BufferSet(&p->rc, &pIter->term, nKeep, pTerm); fts5BufferAppendBlob(&p->rc, &pIter->term, nNew, &a[iOff]); + fts5SegIterLoadEod(p, pIter); fts5SegIterLoadRowid(p, pIter); fts5SegIterLoadNPos(p, pIter); } @@ -2243,6 +2239,7 @@ static void fts5SegIterHashInit( pLeaf->nn = pLeaf->szLeaf = nList; pIter->pLeaf = pLeaf; pIter->iLeafOffset = fts5GetVarint(pLeaf->p, (u64*)&pIter->iRowid); + pIter->iEndofDoclist = pLeaf->nn+1; if( flags & FTS5INDEX_QUERY_DESC ){ pIter->flags |= FTS5_SEGITER_REVERSE; @@ -2770,6 +2767,7 @@ static void fts5MultiIterNew2( if( pData->szLeaf>0 ){ pIter->pLeaf = pData; pIter->iLeafOffset = fts5GetVarint(pData->p, (u64*)&pIter->iRowid); + pIter->iEndofDoclist = pData->nn; pNew->aFirst[1].iFirst = 1; if( bDesc ){ pNew->bRev = 1; @@ -3201,25 +3199,23 @@ static void fts5WriteAppendTerm( Fts5PageWriter *pPage = &pWriter->writer; Fts5Buffer *pPgidx = &pWriter->writer.pgidx; - assert( pPage->buf.n==0 || pPage->buf.n>4 ); - if( pPage->buf.n==0 ){ - /* Zero the first term and first rowid fields */ - static const u8 zero[] = { 0x00, 0x00, 0x00, 0x00 }; - fts5BufferAppendBlob(&p->rc, &pPage->buf, 4, zero); - assert( pWriter->bFirstTermInPage ); - } if( p->rc ) return; + assert( pPage->buf.n>=4 ); + assert( pPage->buf.n>4 || pWriter->bFirstTermInPage ); + + /* If the current leaf page is full, flush it to disk. */ + if( (pPage->buf.n + pPage->pgidx.n + nTerm + 2)>=p->pConfig->pgsz ){ + if( pPage->buf.n>4 ){ + fts5WriteFlushLeaf(p, pWriter); + } + fts5BufferGrow(&p->rc, &pPage->buf, nTerm+FTS5_DATA_PADDING); + } - /* TODO1: Can this be consolidated with FlushOneHash version? */ + /* TODO1: Updating pgidx here. */ fts5PutU16(&pPgidx->p[pPgidx->n], pPage->buf.n); pPgidx->n += 2; if( pWriter->bFirstTermInPage ){ - /* Update the "first term" field of the page header. */ -#if 0 - assert( pPage->buf.p[2]==0 && pPage->buf.p[3]==0 ); - fts5PutU16(&pPage->buf.p[2], pPage->buf.n); -#endif nPrefix = 0; if( pPage->pgno!=1 ){ /* This is the first term on a leaf that is not the leftmost leaf in @@ -3261,11 +3257,6 @@ static void fts5WriteAppendTerm( assert( p->rc || (pWriter->nDlidx>0 && pWriter->aDlidx[0].buf.n==0) ); pWriter->aDlidx[0].pgno = pPage->pgno; - - /* If the current leaf page is full, flush it to disk. */ - if( (pPage->buf.n + pPage->pgidx.n)>=p->pConfig->pgsz ){ - fts5WriteFlushLeaf(p, pWriter); - } } /* @@ -3280,6 +3271,10 @@ static void fts5WriteAppendRowid( if( p->rc==SQLITE_OK ){ Fts5PageWriter *pPage = &pWriter->writer; + if( (pPage->buf.n + pPage->pgidx.n)>=p->pConfig->pgsz ){ + fts5WriteFlushLeaf(p, pWriter); + } + /* If this is to be the first rowid written to the page, set the ** rowid-pointer in the page-header. Also append a value to the dlidx ** buffer, in case a doclist-index is required. */ @@ -3300,10 +3295,6 @@ static void fts5WriteAppendRowid( pWriter->bFirstRowidInPage = 0; fts5BufferAppendVarint(&p->rc, &pPage->buf, nPos); - - if( (pPage->buf.n + pPage->pgidx.n)>=p->pConfig->pgsz ){ - fts5WriteFlushLeaf(p, pWriter); - } } } @@ -3383,6 +3374,8 @@ static void fts5WriteInit( Fts5SegWriter *pWriter, int iSegid ){ + const int nBuffer = p->pConfig->pgsz + FTS5_DATA_PADDING; + memset(pWriter, 0, sizeof(Fts5SegWriter)); pWriter->iSegid = iSegid; @@ -3391,7 +3384,10 @@ static void fts5WriteInit( pWriter->bFirstTermInPage = 1; pWriter->iBtPage = 1; - fts5BufferGrow(&p->rc, &pWriter->writer.pgidx, p->pConfig->pgsz + 20); + /* Grow the two buffers to pgsz + padding bytes in size. */ + fts5BufferGrow(&p->rc, &pWriter->writer.pgidx, nBuffer); + fts5BufferGrow(&p->rc, &pWriter->writer.buf, nBuffer); + if( p->pIdxWriter==0 ){ Fts5Config *pConfig = p->pConfig; fts5IndexPrepareStmt(p, &p->pIdxWriter, sqlite3_mprintf( @@ -3401,6 +3397,13 @@ static void fts5WriteInit( } if( p->rc==SQLITE_OK ){ + /* Initialize the 4-byte leaf-page header to 0x00. */ + memset(pWriter->writer.buf.p, 0, 4); + pWriter->writer.buf.n = 4; + + /* Bind the current output segment id to the index-writer. This is an + ** optimization over binding the same value over and over as rows are + ** inserted into %_idx by the current writer. */ sqlite3_bind_int(p->pIdxWriter, 1, pWriter->iSegid); } } @@ -3414,38 +3417,39 @@ static void fts5WriteInit( ** This function appends a page-index to the buffer. The buffer is ** guaranteed to be large enough to fit the page-index. */ -static void fts5MakePageidx(Fts5Index *p, Fts5Buffer *pBuf){ +static void fts5MakePageidx( + Fts5Index *p, /* Fts index object to store any error in */ + Fts5Data *pOld, /* Original page data */ + int iOldidx, /* Index of term in pOld pgidx */ + Fts5Buffer *pBuf /* Buffer containing new page (no pgidx) */ +){ if( p->rc==SQLITE_OK ){ - u8 *a = pBuf->p; - int szLeaf = pBuf->n; - int iOff = 4; - int nTerm; + int iOff; + int iEnd; + int nByte; + int iEndOld; /* Byte after term iOldIdx on old page */ + int iEndNew; /* Byte after term iOldIdx on new page */ + int ii; + int nPgIdx = (pOld->nn - pOld->szLeaf) / 2; - fts5PutU16(&pBuf->p[pBuf->n], iOff); + /* Determine end of term on old page */ + iEndOld = fts5LeafTermOff(pOld, iOldidx); + if( iOldidx>0 ){ + iEndOld += fts5GetVarint32(&pOld->p[iEndOld], nByte); + } + iEndOld += fts5GetVarint32(&pOld->p[iEndOld], nByte); + iEndOld += nByte; + + /* Determine end of term on new page */ + iEndNew = 4 + fts5GetVarint32(&pBuf->p[4], nByte); + iEndNew += nByte; + + fts5PutU16(&pBuf->p[pBuf->n], 4); pBuf->n += 2; - fts5IndexGetVarint32(a, iOff, nTerm); - iOff += nTerm; - - while( iOff=szLeaf ) break; - - /* Skip past position list */ - fts5IndexGetVarint32(a, iOff, nTerm); - iOff += (nTerm >> 1); - - if( iOff>=(szLeaf-2) ) break; - - /* If this is the end of the doclist, break out of the loop */ - if( a[iOff]==0x00 ){ - iOff++; - fts5PutU16(&pBuf->p[pBuf->n], iOff); - pBuf->n += 2; - fts5IndexGetVarint32(a, iOff, nTerm); - fts5IndexGetVarint32(a, iOff, nTerm); - iOff += nTerm; - } + for(ii=iOldidx+1; iip[pBuf->n], iVal + (iEndNew - iEndOld)); + pBuf->n += 2; } } } @@ -3491,7 +3495,7 @@ static void fts5TrimSegments(Fts5Index *p, Fts5IndexIter *pIter){ } /* Set up the new page-index array */ - fts5MakePageidx(p, &buf); + fts5MakePageidx(p, pData, pSeg->iTermIdx, &buf); fts5DataRelease(pData); pSeg->pSeg->pgnoFirst = pSeg->iTermLeafPgno; @@ -3595,8 +3599,9 @@ static void fts5IndexMergeLevel( } /* This is a new term. Append a term to the output segment. */ + /* TODO2: Doclist 0x00 term */ if( bRequireDoclistTerm ){ - fts5WriteAppendZerobyte(p, &writer); + /* fts5WriteAppendZerobyte(p, &writer); */ } fts5WriteAppendTerm(p, &writer, nTerm, pTerm); fts5BufferSet(&p->rc, &term, nTerm, pTerm); @@ -3739,6 +3744,7 @@ static void fts5IndexCrisismerge( assert( p->rc!=SQLITE_OK || pStruct->nLevel>0 ); while( p->rc==SQLITE_OK && pStruct->aLevel[iLvl].nSeg>=nCrisis ){ fts5IndexMergeLevel(p, &pStruct, iLvl, 0); + assert( p->rc!=SQLITE_OK || pStruct->nLevel>(iLvl+1) ); fts5StructurePromote(p, iLvl+1, pStruct); iLvl++; } @@ -3766,10 +3772,12 @@ static int fts5PoslistPrefix(const u8 *aBuf, int nMax){ int ret; u32 dummy; ret = fts5GetVarint32(aBuf, dummy); - while( 1 ){ - int i = fts5GetVarint32(&aBuf[ret], dummy); - if( (ret + i) > nMax ) break; - ret += i; + if( ret nMax ) break; + ret += i; + } } return ret; } @@ -3810,72 +3818,32 @@ static void fts5FlushOneHash(Fts5Index *p){ Fts5SegWriter writer; fts5WriteInit(p, &writer, iSegid); - /* Pre-allocate the buffer used to assemble leaf pages to the target - ** page size. */ - assert( pgsz>0 ); pBuf = &writer.writer.buf; pPgidx = &writer.writer.pgidx; - fts5BufferGrow(&p->rc, pBuf, pgsz + 20); + + /* fts5WriteInit() should have initialized the buffers to (most likely) + ** the maximum space required. */ + assert( p->rc || pBuf->nSpace>=(pgsz + FTS5_DATA_PADDING) ); + assert( p->rc || pPgidx->nSpace>=(pgsz + FTS5_DATA_PADDING) ); /* Begin scanning through hash table entries. This loop runs once for each ** term/doclist currently stored within the hash table. */ if( p->rc==SQLITE_OK ){ - memset(pBuf->p, 0, 4); - pBuf->n = 4; p->rc = sqlite3Fts5HashScanInit(pHash, 0, 0); } while( p->rc==SQLITE_OK && 0==sqlite3Fts5HashScanEof(pHash) ){ const char *zTerm; /* Buffer containing term */ - int nTerm; /* Size of zTerm in bytes */ const u8 *pDoclist; /* Pointer to doclist for this term */ int nDoclist; /* Size of doclist in bytes */ int nSuffix; /* Size of term suffix */ + /* Write the term for this entry to disk. */ sqlite3Fts5HashScanEntry(pHash, &zTerm, &pDoclist, &nDoclist); - nTerm = strlen(zTerm); + fts5WriteAppendTerm(p, &writer, strlen(zTerm), zTerm); - /* Decide if the term will fit on the current leaf. If it will not, - ** flush the leaf to disk here. - ** TODO1: Is this calculation still correct? */ - if( pBuf->n>4 && (pBuf->n + nTerm + 2 + pPgidx->n + 2) > pgsz ){ - fts5WriteFlushLeaf(p, &writer); - if( (nTerm + 32) > pBuf->nSpace ){ - fts5BufferGrow(&p->rc, pBuf, nTerm + 32 - pBuf->n); - if( p->rc ) break; - } - } - - /* Write the term to the leaf. And if it is the first on the leaf, and - ** the leaf is not page number 1, push it up into the b-tree hierarchy - ** as well. */ - - /* TODO1: Writing pgidx here! */ - fts5PutU16(&pPgidx->p[pPgidx->n], pBuf->n); - pPgidx->n += 2; - if( writer.bFirstTermInPage==0 ){ - int nPre = fts5PrefixCompress(nTerm, zPrev, nTerm, (const u8*)zTerm); - pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], nPre); - nSuffix = nTerm - nPre; - }else{ - writer.bFirstTermInPage = 0; - if( writer.writer.pgno!=1 ){ - int nPre = fts5PrefixCompress(nTerm, zPrev, nTerm, (const u8*)zTerm); - fts5WriteBtreeTerm(p, &writer, nPre+1, (const u8*)zTerm); - pBuf = &writer.writer.buf; - assert( nPren += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], nSuffix); - fts5BufferSafeAppendBlob(pBuf, (const u8*)&zTerm[nTerm-nSuffix], nSuffix); - - /* We just wrote a term into page writer.aWriter[0].pgno. If a - ** doclist-index is to be generated for this doclist, it will be - ** associated with this page. */ - assert( writer.nDlidx>0 && writer.aDlidx[0].buf.n==0 ); - writer.aDlidx[0].pgno = writer.writer.pgno; - - if( pgsz>=(pBuf->n + pPgidx->n + nDoclist + 1) ){ + if( writer.bFirstRowidInPage==0 + && pgsz>=(pBuf->n + pPgidx->n + nDoclist + 1) + ){ /* The entire doclist will fit on the current leaf. */ fts5BufferSafeAppendBlob(pBuf, pDoclist, nDoclist); }else{ @@ -3883,7 +3851,7 @@ static void fts5FlushOneHash(Fts5Index *p){ i64 iDelta = 0; int iOff = 0; - writer.bFirstRowidInPage = 0; + /* writer.bFirstRowidInPage = 0; */ /* The entire doclist will not fit on this leaf. The following ** loop iterates through the poslists that make up the current @@ -3938,7 +3906,8 @@ static void fts5FlushOneHash(Fts5Index *p){ } } - pBuf->p[pBuf->n++] = '\0'; + /* TODO2: Doclist terminator written here. */ + /* pBuf->p[pBuf->n++] = '\0'; */ assert( pBuf->n<=pBuf->nSpace ); zPrev = (const u8*)zTerm; sqlite3Fts5HashScanNext(pHash); @@ -4527,7 +4496,7 @@ int sqlite3Fts5IndexQuery( ** evaluate the prefix query using the main FTS index. This is used ** for internal sanity checking by the integrity-check in debug ** mode only. */ - if( flags & FTS5INDEX_QUERY_TEST_NOIDX ){ + if( pConfig->bPrefixIndex==0 || (flags & FTS5INDEX_QUERY_TEST_NOIDX) ){ assert( flags & FTS5INDEX_QUERY_PREFIX ); iIdx = 1+pConfig->nPrefix; }else @@ -4971,6 +4940,48 @@ static void fts5IndexIntegrityCheckEmpty( } } +static void fts5IntegrityCheckPgidx(Fts5Index *p, Fts5Data *pLeaf){ + int nPg = (pLeaf->nn - pLeaf->szLeaf) / 2; + int ii; + Fts5Buffer buf1 = {0,0,0}; + Fts5Buffer buf2 = {0,0,0}; + + for(ii=0; p->rc==SQLITE_OK && ii=pLeaf->szLeaf ){ + p->rc = FTS5_CORRUPT; + }else if( ii==0 ){ + int nByte; + iOff += fts5GetVarint32(&pLeaf->p[iOff], nByte); + if( (iOff+nByte)>pLeaf->szLeaf ){ + p->rc = FTS5_CORRUPT; + }else{ + fts5BufferSet(&p->rc, &buf1, nByte, &pLeaf->p[iOff]); + } + }else{ + int nKeep, nByte; + iOff += fts5GetVarint32(&pLeaf->p[iOff], nKeep); + iOff += fts5GetVarint32(&pLeaf->p[iOff], nByte); + if( nKeep>buf1.n || (iOff+nByte)>pLeaf->szLeaf ){ + p->rc = FTS5_CORRUPT; + }else{ + buf1.n = nKeep; + fts5BufferAppendBlob(&p->rc, &buf1, nByte, &pLeaf->p[iOff]); + } + + if( p->rc==SQLITE_OK ){ + res = fts5BufferCompare(&buf1, &buf2); + if( res<=0 ) p->rc = FTS5_CORRUPT; + } + } + fts5BufferSet(&p->rc, &buf2, buf1.n, buf1.p); + } + + fts5BufferFree(&buf1); + fts5BufferFree(&buf2); +} + static void fts5IndexIntegrityCheckSegment( Fts5Index *p, /* FTS5 backend object */ Fts5StructureSegment *pSeg /* Segment to check internal consistency */ @@ -5027,6 +5038,8 @@ static void fts5IndexIntegrityCheckSegment( if( res==0 ) res = nTerm - nIdxTerm; if( res<0 ) p->rc = FTS5_CORRUPT; } + + fts5IntegrityCheckPgidx(p, pLeaf); } fts5DataRelease(pLeaf); if( p->rc ) break; @@ -5418,51 +5431,56 @@ static void fts5DecodeFunction( int szLeaf = 0; int iRowidOff = 0; int iOff; - int nKeep = 0; + int nPgTerm = 0; + int nDoclist; + int i; memset(&term, 0, sizeof(Fts5Buffer)); - if( n>=4 ){ - iRowidOff = fts5GetU16(&a[0]); - szLeaf = fts5GetU16(&a[2]); - if( szLeaf0 ){ + iOff += fts5GetVarint32(&a[iOff], nByte); + term.n = nByte; + } iOff += fts5GetVarint32(&a[iOff], nByte); - term.n= nKeep; fts5BufferAppendBlob(&rc, &term, nByte, &a[iOff]); iOff += nByte; - sqlite3Fts5BufferAppendPrintf( &rc, &s, " term=%.*s", term.n, (const char*)term.p ); - iOff += fts5DecodeDoclist(&rc, &s, &a[iOff], szLeaf-iOff); - if( iOffpStorage, nMerge); }else if( 0==sqlite3_stricmp("integrity-check", z) ){ rc = sqlite3Fts5StorageIntegrity(pTab->pStorage); +#ifdef SQLITE_TEST + }else if( 0==sqlite3_stricmp("prefix-index", z) ){ + pConfig->bPrefixIndex = sqlite3_value_int(pVal); +#endif }else{ rc = sqlite3Fts5IndexLoadConfig(pTab->pIndex); if( rc==SQLITE_OK ){ diff --git a/ext/fts5/test/fts5aa.test b/ext/fts5/test/fts5aa.test index e20893dcf7..ae445f740d 100644 --- a/ext/fts5/test/fts5aa.test +++ b/ext/fts5/test/fts5aa.test @@ -203,11 +203,6 @@ for {set i 1} {$i <= 10} {incr i} { if {[set_test_counter errors]} break } -#db eval { SELECT fts5_decode(rowid, block) as x FROM t1_data } { puts $x } -#puts [db eval {SELECT rowid FROM t1 WHERE t1 MATCH 'aaa' ORDER BY rowid ASC}] -#puts [db eval {SELECT rowid FROM t1 WHERE t1 MATCH 'aaa' ORDER BY rowid DESC}] -#exit - #------------------------------------------------------------------------- # reset_db diff --git a/ext/fts5/test/fts5simple.test b/ext/fts5/test/fts5simple.test index c146090f3f..eeac11b5ca 100644 --- a/ext/fts5/test/fts5simple.test +++ b/ext/fts5/test/fts5simple.test @@ -11,7 +11,7 @@ # source [file join [file dirname [info script]] fts5_common.tcl] -set testprefix fts5aa +set testprefix fts5simple # If SQLITE_ENABLE_FTS5 is defined, omit this file. ifcapable !fts5 { @@ -50,7 +50,6 @@ do_execsql_test 2.1 { INSERT INTO t1(t1) VALUES('integrity-check'); } {} -} #------------------------------------------------------------------------- # @@ -63,12 +62,58 @@ do_execsql_test 3.0 { SELECT * FROM t1 WHERE t1 MATCH 'o*'; } {one} -breakpoint do_execsql_test 3.1 { INSERT INTO t1(t1) VALUES('integrity-check'); } {} -# db eval { SELECT fts5_decode(rowid, block) as x FROM t1_data } { puts $x } +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 4.1 { + CREATE VIRTUAL TABLE t11 USING fts5(content); + INSERT INTO t11(t11, rank) VALUES('pgsz', 32); + INSERT INTO t11 VALUES('another'); + INSERT INTO t11 VALUES('string'); + INSERT INTO t11 VALUES('of'); + INSERT INTO t11 VALUES('text'); +} +do_test 4.2 { + execsql { INSERT INTO t11(t11) VALUES('optimize') } +} {} +do_execsql_test 4.3 { + INSERT INTO t11(t11) VALUES('integrity-check'); +} {} + +#db eval { SELECT fts5_decode(rowid, block) as x FROM t11_data } { puts $x } + +#------------------------------------------------------------------------- +reset_db +set doc [string repeat "x y " 5] +do_execsql_test 5.1 { + CREATE VIRTUAL TABLE yy USING fts5(content); + INSERT INTO yy(yy, rank) VALUES('pgsz', 32); + BEGIN; + INSERT INTO yy VALUES($doc); + INSERT INTO yy VALUES($doc); + INSERT INTO yy VALUES($doc); + INSERT INTO yy VALUES($doc); + INSERT INTO yy VALUES($doc); + INSERT INTO yy VALUES($doc); + INSERT INTO yy VALUES($doc); + INSERT INTO yy VALUES($doc); + COMMIT; +} + +do_execsql_test 5.2 { + SELECT rowid FROM yy WHERE yy MATCH 'y' ORDER BY rowid ASC +} {1 2 3 4 5 6 7 8} + +do_execsql_test 5.3 { + SELECT rowid FROM yy WHERE yy MATCH 'y' ORDER BY rowid DESC +} {8 7 6 5 4 3 2 1} + +#db eval { SELECT fts5_decode(rowid, block) as x FROM yy_data } { puts $x } finish_test diff --git a/main.mk b/main.mk index d07e473be6..a10199f3d1 100644 --- a/main.mk +++ b/main.mk @@ -332,7 +332,9 @@ TESTSRC += \ $(TOP)/ext/misc/vfslog.c \ $(TOP)/ext/fts5/fts5_tcl.c \ $(TOP)/ext/fts5/fts5_test_mi.c \ - fts5.c + $(FTS5_SRC) + +# fts5.c #TESTSRC += $(TOP)/ext/fts2/fts2_tokenizer.c diff --git a/manifest b/manifest index f47bf05938..36d5700540 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Use\smacros\sto\smake\sthe\scode\sin\sfts5_index.c\seasier\sto\sread. -D 2015-09-07T08:14:30.857 +C Remove\sthe\s0x00\sterminators\sfrom\sthe\send\sof\sdoclists\sstored\son\sdisk. +D 2015-09-08T19:55:26.385 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -106,14 +106,14 @@ F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0 F ext/fts5/fts5.h f04659e0df5af83731b102189a32280f74f4a6bc -F ext/fts5/fts5Int.h f65d41f66accad0a289d6bd66b13c07d2932f9be +F ext/fts5/fts5Int.h 7e6002582133cb795a21468732cf7c35027a28e4 F ext/fts5/fts5_aux.c 7a307760a9c57c750d043188ec0bad59f5b5ec7e -F ext/fts5/fts5_buffer.c 80f9ba4431848cb857e3d2158f5280093dcd8015 -F ext/fts5/fts5_config.c 80b61fd2c6844b64a3e72a64572d50a812da9384 +F ext/fts5/fts5_buffer.c 64dcaf36a3ebda9e84b7c3b8788887ec325e12a4 +F ext/fts5/fts5_config.c 57ee5fe71578cb494574fc0e6e51acb9a22a8695 F ext/fts5/fts5_expr.c 1c24e1a2ffb286bfe37e537a43b7fadabfe993d4 F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246 -F ext/fts5/fts5_index.c 213e5aea27100a2ebb7a576d6574bcc28c594520 -F ext/fts5/fts5_main.c e9d0892424bb7f0a8b58613d4ff75cb650cf286e +F ext/fts5/fts5_index.c 677c859b2064e00903a9ad847a1cfca8d36ce595 +F ext/fts5/fts5_main.c 3ec19f23693e034028afb9b4fa7c800dc3eda5ab F ext/fts5/fts5_storage.c 120f7b143688b5b7710dacbd48cff211609b8059 F ext/fts5/fts5_tcl.c 6da58d6e8f42a93c4486b5ba9b187a7f995dee37 F ext/fts5/fts5_test_mi.c e96be827aa8f571031e65e481251dc1981d608bf @@ -124,7 +124,7 @@ F ext/fts5/fts5_vocab.c 4622e0b7d84a488a1585aaa56eb214ee67a988bc F ext/fts5/fts5parse.y 833db1101b78c0c47686ab1b84918e38c36e9452 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba F ext/fts5/test/fts5_common.tcl b6e6a40ef5d069c8e86ca4fbad491e1195485dbc -F ext/fts5/test/fts5aa.test 1ac5a3bd88406183b00ea7cb4701bd58e5c3c7ff +F ext/fts5/test/fts5aa.test fb84c53e90dc4f9b4dc485858b53e70b67d6f064 F ext/fts5/test/fts5ab.test 6fe3a56731d15978afbb74ae51b355fc9310f2ad F ext/fts5/test/fts5ac.test 9737992d08c56bfd4803e933744d2d764e23795c F ext/fts5/test/fts5ad.test e3dfb150fce971b4fd832498c29f56924d451b63 @@ -173,7 +173,7 @@ F ext/fts5/test/fts5rank.test 11dcebba31d822f7e99685b4ea2c2ae3ec0b16f1 F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17 F ext/fts5/test/fts5rowid.test 6f9833b23b176dc4aa15b7fc02afeb2b220fd460 -F ext/fts5/test/fts5simple.test f520b360c40a5a61aabebead53e1e5f68683e5a3 +F ext/fts5/test/fts5simple.test 40b76f77d56524a882473664c792bd4e8f2f2a0a F ext/fts5/test/fts5synonym.test cf88c0a56d5ea9591e3939ef1f6e294f7f2d0671 F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89 F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841 @@ -261,7 +261,7 @@ F ext/userauth/userauth.c 5fa3bdb492f481bbc1709fc83c91ebd13460c69e F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk 61821e43596648bfacce2d6283377bee35986131 +F main.mk e67b73755e1f28c2d18d7953ca75c901c0be617b F mkopcodec.awk c2ff431854d702cdd2d779c9c0d1f58fa16fa4ea F mkopcodeh.awk 0e7f04a8eb90f92259e47d80110e4e98d7ce337a F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 @@ -1384,7 +1384,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P a1f4c3b543eed84e808f6b901a38179786fffe16 -R d9f9c87817152716ded47406f74ba235 +P 67ff5ae81357eb7fa28049bb724a22cb6f52e076 +R 7e6723f8162dfa594f88012bb5737a33 U dan -Z 4c8df884c6ae97a50d0e8662a6ca4b0e +Z 0d6a020581263053b4291f892521bd3c diff --git a/manifest.uuid b/manifest.uuid index a4473a78af..55934271fe 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -67ff5ae81357eb7fa28049bb724a22cb6f52e076 \ No newline at end of file +00d990061dec3661b0376bd167082942d5563bfe \ No newline at end of file From b4d472f609d0f81bec921b82251f0a6c23a26ece Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 8 Sep 2015 20:26:09 +0000 Subject: [PATCH 055/100] Eponymous virtual tables exist in the "main" schema only. Enforce this rule. FossilOrigin-Name: 06f90bb274c4bb0c30585024c8d365d43c4162f2 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/build.c | 14 ++++++++------ test/tabfunc01.test | 13 +++++++++++++ 4 files changed, 29 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 72d1042163..a8b91dc375 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\shelp\smessage\sthat\ssqlite3_analyzer.exe\sgenerates\sfor\sinvalid\narguments. -D 2015-09-08T17:31:30.826 +C Eponymous\svirtual\stables\sexist\sin\sthe\s"main"\sschema\sonly.\s\sEnforce\sthis\srule. +D 2015-09-08T20:26:09.245 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -284,7 +284,7 @@ F src/btmutex.c 45a968cc85afed9b5e6cf55bf1f42f8d18107f79 F src/btree.c 4084d9eed2817331f6e6a82230ba30e448cad497 F src/btree.h 969adc948e89e449220ff0ff724c94bb2a52e9f1 F src/btreeInt.h 8177c9ab90d772d6d2c6c517e05bed774b7c92c0 -F src/build.c 5566b3410080a54e5c302c55d3de53fd080cfc7d +F src/build.c ba9e1529730407cdf491480745716d6fbdc28cd3 F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 F src/complete.c addcd8160b081131005d5bc2d34adf20c1c5c92f F src/ctime.c 5a0b735dc95604766f5dac73973658eef782ee8b @@ -1038,7 +1038,7 @@ F test/superlock.test 1cde669f68d2dd37d6c9bd35eee1d95491ae3fc2 F test/sync.test a34cd43e98b7fb84eabbf38f7ed8f7349b3f3d85 F test/syscall.test d2fdaad713f103ac611fe7ef9b724c7b69f8149c F test/sysfault.test fa776e60bf46bdd3ae69f0b73e46ee3977a58ae6 -F test/tabfunc01.test a12eba3f48a03a6626f985734ecc28132381fa9b +F test/tabfunc01.test fa9d8dfc75747019e0be98d3b6ac68d18632d328 F test/table.test 33bf0d1fd07f304582695184b8e6feb017303816 F test/tableapi.test 2674633fa95d80da917571ebdd759a14d9819126 F test/tableopts.test dba698ba97251017b7c80d738c198d39ab747930 @@ -1383,7 +1383,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 986677224a8da5e79fbbd90673f1b595da89c5d6 -R 53efe0392669648bee0c2878b1555bb5 +P 33a14e7be1004abca7a30f675459138d7f8d72b1 +R 1f56442884ee3ab877d3d3fe559d09f2 U drh -Z cf7dc4ba83b64f541e9e5f6eb6b8039d +Z b9e298f3b9bca3ea8a0b0e129438890a diff --git a/manifest.uuid b/manifest.uuid index 04b63a44c2..b04d06e2a5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -33a14e7be1004abca7a30f675459138d7f8d72b1 \ No newline at end of file +06f90bb274c4bb0c30585024c8d365d43c4162f2 \ No newline at end of file diff --git a/src/build.c b/src/build.c index 51facddff2..98a032bd7c 100644 --- a/src/build.c +++ b/src/build.c @@ -357,12 +357,14 @@ Table *sqlite3LocateTable( if( p==0 ){ const char *zMsg = isView ? "no such view" : "no such table"; #ifndef SQLITE_OMIT_VIRTUALTABLE - /* If zName is the not the name of a table in the schema created using - ** CREATE, then check to see if it is the name of an virtual table that - ** can be an eponymous virtual table. */ - Module *pMod = (Module*)sqlite3HashFind(&pParse->db->aModule, zName); - if( pMod && sqlite3VtabEponymousTableInit(pParse, pMod) ){ - return pMod->pEpoTab; + if( sqlite3FindDbName(pParse->db, zDbase)<1 ){ + /* If zName is the not the name of a table in the schema created using + ** CREATE, then check to see if it is the name of an virtual table that + ** can be an eponymous virtual table. */ + Module *pMod = (Module*)sqlite3HashFind(&pParse->db->aModule, zName); + if( pMod && sqlite3VtabEponymousTableInit(pParse, pMod) ){ + return pMod->pEpoTab; + } } #endif if( zDbase ){ diff --git a/test/tabfunc01.test b/test/tabfunc01.test index 30a40e3138..07b3c80442 100644 --- a/test/tabfunc01.test +++ b/test/tabfunc01.test @@ -69,5 +69,18 @@ do_execsql_test tabfunc01-3.1 { SELECT DISTINCT value FROM generate_series(1,x), t1 ORDER BY 1; } {1 2 3} +# Eponymous virtual table exists in the "main" schema only +# +do_execsql_test tabfunc01-4.1 { + SELECT * FROM main.generate_series(1,4) +} {1 2 3 4} +do_catchsql_test tabfunc01-4.2 { + SELECT * FROM temp.generate_series(1,4) +} {1 {no such table: temp.generate_series}} +do_catchsql_test tabfunc01-4.3 { + ATTACH ':memory:' AS aux1; + CREATE TABLE aux1.t1(a,b,c); + SELECT * FROM aux1.generate_series(1,4) +} {1 {no such table: aux1.generate_series}} finish_test From a46a4a63de1853cf4dac3ecfad4bbefa596d670d Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 8 Sep 2015 21:12:53 +0000 Subject: [PATCH 056/100] Enhance the DBSTAT virtual table with a new hidden table "schema" that if set will cause the table to report on the specified schema rather than on "main". Also: Fix a faulty assert in sqlite3_context_db_handle(). FossilOrigin-Name: 6beb512c7a3c3649b56f0df1ca77855535a87ba7 --- manifest | 14 +++---- manifest.uuid | 2 +- src/dbstat.c | 105 ++++++++++++++++++++++++++++++++++++-------------- src/vdbeapi.c | 2 +- 4 files changed, 86 insertions(+), 37 deletions(-) diff --git a/manifest b/manifest index a8b91dc375..0203e740f9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Eponymous\svirtual\stables\sexist\sin\sthe\s"main"\sschema\sonly.\s\sEnforce\sthis\srule. -D 2015-09-08T20:26:09.245 +C Enhance\sthe\sDBSTAT\svirtual\stable\swith\sa\snew\shidden\stable\s"schema"\sthat\sif\nset\swill\scause\sthe\stable\sto\sreport\son\sthe\sspecified\sschema\srather\sthan\son\n"main".\s\sAlso:\s\sFix\sa\sfaulty\sassert\sin\ssqlite3_context_db_handle(). +D 2015-09-08T21:12:53.186 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 F src/complete.c addcd8160b081131005d5bc2d34adf20c1c5c92f F src/ctime.c 5a0b735dc95604766f5dac73973658eef782ee8b F src/date.c fb1c99172017dcc8e237339132c91a21a0788584 -F src/dbstat.c f402e77e25089c6003d0c60b3233b9b3947d599a +F src/dbstat.c e637e7a7ff40ef32132a418c6fdf1cfb63aa27c7 F src/delete.c 6792c80d7fb54c4df9f7680413952600e7439492 F src/expr.c 3a76afcdac925294c39903b7002ddb9e5fd29863 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb @@ -405,7 +405,7 @@ F src/vacuum.c 2ddd5cad2a7b9cef7f9e431b8c7771634c6b1701 F src/vdbe.c 6d85be995bd2308a5aa2a68c7b564c5d4cc1a6fb F src/vdbe.h 4bc88bd0e06f8046ee6ab7487c0015e85ad949ad F src/vdbeInt.h 8b54e01ad0463590e7cffabce0bc36da9ee4f816 -F src/vdbeapi.c bda74ef4b5103d7b4a4be36f936d3cf2b56a7d6f +F src/vdbeapi.c b821d530bcb2900b4604cf5206f2177f3f881d15 F src/vdbeaux.c fd00b489ab3f44f2dca1e4344faf289b7bfcf649 F src/vdbeblob.c 1d7b97115e7bbac4c318db416d2ca83fc779544a F src/vdbemem.c 19b3036aa4d676e7103b0fb5efd6327da455f915 @@ -1383,7 +1383,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 33a14e7be1004abca7a30f675459138d7f8d72b1 -R 1f56442884ee3ab877d3d3fe559d09f2 +P 06f90bb274c4bb0c30585024c8d365d43c4162f2 +R 0484df860261efad073810ad511e031a U drh -Z b9e298f3b9bca3ea8a0b0e129438890a +Z 217b27586d12b1e6acf8f38c50e52968 diff --git a/manifest.uuid b/manifest.uuid index b04d06e2a5..bf85a4aab9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -06f90bb274c4bb0c30585024c8d365d43c4162f2 \ No newline at end of file +6beb512c7a3c3649b56f0df1ca77855535a87ba7 \ No newline at end of file diff --git a/src/dbstat.c b/src/dbstat.c index c36be020af..f43b14881f 100644 --- a/src/dbstat.c +++ b/src/dbstat.c @@ -16,6 +16,9 @@ ** information from an SQLite database in order to implement the ** "sqlite3_analyzer" utility. See the ../tool/spaceanal.tcl script ** for an example implementation. +** +** Additional information is available on the "dbstat.html" page of the +** official SQLite documentation. */ #include "sqliteInt.h" /* Requires access to internal data structures */ @@ -64,7 +67,8 @@ " unused INTEGER, /* Bytes of unused space on this page */" \ " mx_payload INTEGER, /* Largest payload size of all cells */" \ " pgoffset INTEGER, /* Offset of page in file */" \ - " pgsize INTEGER /* Size of the page */" \ + " pgsize INTEGER, /* Size of the page */" \ + " schema TEXT HIDDEN /* Database schema being analyzed */" \ ");" @@ -102,6 +106,7 @@ struct StatCursor { sqlite3_vtab_cursor base; sqlite3_stmt *pStmt; /* Iterates through set of root pages */ int isEof; /* After pStmt has returned SQLITE_DONE */ + int iDb; /* Schema used for this query */ StatPage aPage[32]; int iPage; /* Current entry in aPage[] */ @@ -179,9 +184,32 @@ static int statDisconnect(sqlite3_vtab *pVtab){ /* ** There is no "best-index". This virtual table always does a linear -** scan of the binary VFS log file. +** scan. However, a schema=? constraint should cause this table to +** operate on a different database schema, so check for it. +** +** idxNum is normally 0, but will be 1 if a schema=? constraint exists. */ static int statBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ + int i; + + pIdxInfo->estimatedCost = 1.0e6; /* Initial cost estimate */ + + /* Look for a valid schema=? constraint. If found, change the idxNum to + ** 1 and request the value of that constraint be sent to xFilter. And + ** lower the cost estimate to encourage the constrained version to be + ** used. + */ + for(i=0; inConstraint; i++){ + if( pIdxInfo->aConstraint[i].usable==0 ) continue; + if( pIdxInfo->aConstraint[i].op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; + if( pIdxInfo->aConstraint[i].iColumn!=10 ) continue; + pIdxInfo->idxNum = 1; + pIdxInfo->estimatedCost = 1.0; + pIdxInfo->aConstraintUsage[i].argvIndex = 1; + pIdxInfo->aConstraintUsage[i].omit = 1; + break; + } + /* Records are always returned in ascending order of (name, path). ** If this will satisfy the client, set the orderByConsumed flag so that @@ -201,7 +229,6 @@ static int statBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ pIdxInfo->orderByConsumed = 1; } - pIdxInfo->estimatedCost = 10.0; return SQLITE_OK; } @@ -211,36 +238,18 @@ static int statBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ static int statOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ StatTable *pTab = (StatTable *)pVTab; StatCursor *pCsr; - int rc; pCsr = (StatCursor *)sqlite3_malloc64(sizeof(StatCursor)); if( pCsr==0 ){ - rc = SQLITE_NOMEM; + return SQLITE_NOMEM; }else{ - char *zSql; memset(pCsr, 0, sizeof(StatCursor)); pCsr->base.pVtab = pVTab; - - zSql = sqlite3_mprintf( - "SELECT 'sqlite_master' AS name, 1 AS rootpage, 'table' AS type" - " UNION ALL " - "SELECT name, rootpage, type" - " FROM \"%w\".sqlite_master WHERE rootpage!=0" - " ORDER BY name", pTab->db->aDb[pTab->iDb].zName); - if( zSql==0 ){ - rc = SQLITE_NOMEM; - }else{ - rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pCsr->pStmt, 0); - sqlite3_free(zSql); - } - if( rc!=SQLITE_OK ){ - sqlite3_free(pCsr); - pCsr = 0; - } + pCsr->iDb = pTab->iDb; } *ppCursor = (sqlite3_vtab_cursor *)pCsr; - return rc; + return SQLITE_OK; } static void statClearPage(StatPage *p){ @@ -265,6 +274,7 @@ static void statResetCsr(StatCursor *pCsr){ pCsr->iPage = 0; sqlite3_free(pCsr->zPath); pCsr->zPath = 0; + pCsr->isEof = 0; } /* @@ -427,7 +437,7 @@ static int statNext(sqlite3_vtab_cursor *pCursor){ char *z; StatCursor *pCsr = (StatCursor *)pCursor; StatTable *pTab = (StatTable *)pCursor->pVtab; - Btree *pBt = pTab->db->aDb[pTab->iDb].pBt; + Btree *pBt = pTab->db->aDb[pCsr->iDb].pBt; Pager *pPager = sqlite3BtreePager(pBt); sqlite3_free(pCsr->zPath); @@ -565,9 +575,43 @@ static int statFilter( int argc, sqlite3_value **argv ){ StatCursor *pCsr = (StatCursor *)pCursor; + StatTable *pTab = (StatTable*)(pCursor->pVtab); + char *zSql; + int rc = SQLITE_OK; + char *zMaster; + if( idxNum==1 ){ + const char *zDbase = (const char*)sqlite3_value_text(argv[0]); + pCsr->iDb = sqlite3FindDbName(pTab->db, zDbase); + if( pCsr->iDb<0 ){ + sqlite3_free(pCursor->pVtab->zErrMsg); + pCursor->pVtab->zErrMsg = sqlite3_mprintf("no such schema: %s", zDbase); + return pCursor->pVtab->zErrMsg ? SQLITE_ERROR : SQLITE_NOMEM; + } + }else{ + pCsr->iDb = pTab->iDb; + } statResetCsr(pCsr); - return statNext(pCursor); + sqlite3_finalize(pCsr->pStmt); + pCsr->pStmt = 0; + zMaster = pCsr->iDb==1 ? "sqlite_temp_master" : "sqlite_master"; + zSql = sqlite3_mprintf( + "SELECT 'sqlite_master' AS name, 1 AS rootpage, 'table' AS type" + " UNION ALL " + "SELECT name, rootpage, type" + " FROM \"%w\".%s WHERE rootpage!=0" + " ORDER BY name", pTab->db->aDb[pCsr->iDb].zName, zMaster); + if( zSql==0 ){ + return SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pCsr->pStmt, 0); + sqlite3_free(zSql); + } + + if( rc==SQLITE_OK ){ + rc = statNext(pCursor); + } + return rc; } static int statColumn( @@ -604,10 +648,15 @@ static int statColumn( case 8: /* pgoffset */ sqlite3_result_int64(ctx, pCsr->iOffset); break; - default: /* pgsize */ - assert( i==9 ); + case 9: /* pgsize */ sqlite3_result_int(ctx, pCsr->szPage); break; + default: { /* schema */ + sqlite3 *db = sqlite3_context_db_handle(ctx); + int iDb = pCsr->iDb; + sqlite3_result_text(ctx, db->aDb[iDb].zName, -1, SQLITE_STATIC); + break; + } } return SQLITE_OK; } diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 15a8bba0d4..faf97634c8 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -696,7 +696,7 @@ void *sqlite3_user_data(sqlite3_context *p){ ** application defined function. */ sqlite3 *sqlite3_context_db_handle(sqlite3_context *p){ - assert( p && p->pFunc ); + assert( p && p->pOut ); return p->pOut->db; } From 0c6dfaa34a4bee1e85a982dbf2346c95df81bc4f Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 8 Sep 2015 21:16:46 +0000 Subject: [PATCH 057/100] Remove an unused local variable from Lemon. FossilOrigin-Name: fe9ffe5eed7d376f3f08c78c1ce5514c886f3479 --- manifest | 12 ++++++------ manifest.uuid | 2 +- tool/lemon.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 0203e740f9..df9363165a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sthe\sDBSTAT\svirtual\stable\swith\sa\snew\shidden\stable\s"schema"\sthat\sif\nset\swill\scause\sthe\stable\sto\sreport\son\sthe\sspecified\sschema\srather\sthan\son\n"main".\s\sAlso:\s\sFix\sa\sfaulty\sassert\sin\ssqlite3_context_db_handle(). -D 2015-09-08T21:12:53.186 +C Remove\san\sunused\slocal\svariable\sfrom\sLemon. +D 2015-09-08T21:16:46.221 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1338,7 +1338,7 @@ F tool/fuzzershell.c f2fc86dd22df654b28851b85019d3bd007361751 F tool/genfkey.README cf68fddd4643bbe3ff8e31b8b6d8b0a1b85e20f4 F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce -F tool/lemon.c d7c82de603f5cd5a8b84a522f8170a73d00dce68 +F tool/lemon.c 039f813b520b9395740c52f9cbf36c90b5d8df03 F tool/lempar.c 3617143ddb9b176c3605defe6a9c798793280120 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862 F tool/logest.c eef612f8adf4d0993dafed0416064cf50d5d33c6 @@ -1383,7 +1383,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 06f90bb274c4bb0c30585024c8d365d43c4162f2 -R 0484df860261efad073810ad511e031a +P 6beb512c7a3c3649b56f0df1ca77855535a87ba7 +R 0ee50599e53bafcea0ab0d2587bdc748 U drh -Z 217b27586d12b1e6acf8f38c50e52968 +Z b332ec0feed7e0dc71dd5e410ae1a5f1 diff --git a/manifest.uuid b/manifest.uuid index bf85a4aab9..7c074d1963 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6beb512c7a3c3649b56f0df1ca77855535a87ba7 \ No newline at end of file +fe9ffe5eed7d376f3f08c78c1ce5514c886f3479 \ No newline at end of file diff --git a/tool/lemon.c b/tool/lemon.c index e24fe4212c..2e8054b5cc 100644 --- a/tool/lemon.c +++ b/tool/lemon.c @@ -4302,7 +4302,7 @@ void CompressTables(struct lemon *lemp) struct state *stp; struct action *ap, *ap2; struct rule *rp, *rp2, *rbest; - int nbest, n, nshift; + int nbest, n; int i; int usesWildcard; From f06bd2c132ac81e42e207160d5998941378ed2bf Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 9 Sep 2015 08:15:06 +0000 Subject: [PATCH 058/100] Fix a bug in preprocessor macros within fts5_main.c. FossilOrigin-Name: 0eb2b9521fad6fa36e6fa374c2bc1f70b5180f7c --- ext/fts5/fts5_main.c | 2 +- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index 0755dcb291..2fd633bd28 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -1317,7 +1317,7 @@ static int fts5SpecialInsert( rc = sqlite3Fts5StorageMerge(pTab->pStorage, nMerge); }else if( 0==sqlite3_stricmp("integrity-check", z) ){ rc = sqlite3Fts5StorageIntegrity(pTab->pStorage); -#ifdef SQLITE_TEST +#ifdef SQLITE_DEBUG }else if( 0==sqlite3_stricmp("prefix-index", z) ){ pConfig->bPrefixIndex = sqlite3_value_int(pVal); #endif diff --git a/manifest b/manifest index 36d5700540..3a493d4cfa 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sthe\s0x00\sterminators\sfrom\sthe\send\sof\sdoclists\sstored\son\sdisk. -D 2015-09-08T19:55:26.385 +C Fix\sa\sbug\sin\spreprocessor\smacros\swithin\sfts5_main.c. +D 2015-09-09T08:15:06.914 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -113,7 +113,7 @@ F ext/fts5/fts5_config.c 57ee5fe71578cb494574fc0e6e51acb9a22a8695 F ext/fts5/fts5_expr.c 1c24e1a2ffb286bfe37e537a43b7fadabfe993d4 F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246 F ext/fts5/fts5_index.c 677c859b2064e00903a9ad847a1cfca8d36ce595 -F ext/fts5/fts5_main.c 3ec19f23693e034028afb9b4fa7c800dc3eda5ab +F ext/fts5/fts5_main.c 4b04c934084ea24a858438a04b5be8af3a9e0311 F ext/fts5/fts5_storage.c 120f7b143688b5b7710dacbd48cff211609b8059 F ext/fts5/fts5_tcl.c 6da58d6e8f42a93c4486b5ba9b187a7f995dee37 F ext/fts5/fts5_test_mi.c e96be827aa8f571031e65e481251dc1981d608bf @@ -1384,7 +1384,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 67ff5ae81357eb7fa28049bb724a22cb6f52e076 -R 7e6723f8162dfa594f88012bb5737a33 +P 00d990061dec3661b0376bd167082942d5563bfe +R 9536c4653d5903bd9893d3d3b5a1950d U dan -Z 0d6a020581263053b4291f892521bd3c +Z 248c3a89f7d69902ace7cabd0e5d1724 diff --git a/manifest.uuid b/manifest.uuid index 55934271fe..355df4f8f3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -00d990061dec3661b0376bd167082942d5563bfe \ No newline at end of file +0eb2b9521fad6fa36e6fa374c2bc1f70b5180f7c \ No newline at end of file From 3c03afd3f50451f62c46574224078a1b3a7b41c8 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 9 Sep 2015 13:28:06 +0000 Subject: [PATCH 059/100] When running a CREATE TABLE AS, make the initial temporary sqlite_master entry for the new table a real record rather than a NULL, in case the query after the AS actually tries to read the sqlite_master table. Fix for ticket [acd12990885d9276]. FossilOrigin-Name: 4a18d8bd4cc66eb08c6198cdf6e14f1bce0ec05a --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/build.c | 4 +++- test/table.test | 12 ++++++++++++ 4 files changed, 23 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index df9363165a..cd0edc5ea2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sunused\slocal\svariable\sfrom\sLemon. -D 2015-09-08T21:16:46.221 +C When\srunning\sa\sCREATE\sTABLE\sAS,\smake\sthe\sinitial\stemporary\ssqlite_master\sentry\s\nfor\sthe\snew\stable\sa\sreal\srecord\srather\sthan\sa\sNULL,\sin\scase\sthe\squery\safter\s\nthe\sAS\sactually\stries\sto\sread\sthe\ssqlite_master\stable.\nFix\sfor\sticket\s[acd12990885d9276]. +D 2015-09-09T13:28:06.029 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -284,7 +284,7 @@ F src/btmutex.c 45a968cc85afed9b5e6cf55bf1f42f8d18107f79 F src/btree.c 4084d9eed2817331f6e6a82230ba30e448cad497 F src/btree.h 969adc948e89e449220ff0ff724c94bb2a52e9f1 F src/btreeInt.h 8177c9ab90d772d6d2c6c517e05bed774b7c92c0 -F src/build.c ba9e1529730407cdf491480745716d6fbdc28cd3 +F src/build.c f81380bc4d5d239c18b42982a9866a94489fd444 F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 F src/complete.c addcd8160b081131005d5bc2d34adf20c1c5c92f F src/ctime.c 5a0b735dc95604766f5dac73973658eef782ee8b @@ -1039,7 +1039,7 @@ F test/sync.test a34cd43e98b7fb84eabbf38f7ed8f7349b3f3d85 F test/syscall.test d2fdaad713f103ac611fe7ef9b724c7b69f8149c F test/sysfault.test fa776e60bf46bdd3ae69f0b73e46ee3977a58ae6 F test/tabfunc01.test fa9d8dfc75747019e0be98d3b6ac68d18632d328 -F test/table.test 33bf0d1fd07f304582695184b8e6feb017303816 +F test/table.test b708f3e5fa2542fa51dfab21fc07b36ea445cb2f F test/tableapi.test 2674633fa95d80da917571ebdd759a14d9819126 F test/tableopts.test dba698ba97251017b7c80d738c198d39ab747930 F test/tclsqlite.test 7fb866443c7deceed22b63948ccd6f76b52ad054 @@ -1383,7 +1383,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 6beb512c7a3c3649b56f0df1ca77855535a87ba7 -R 0ee50599e53bafcea0ab0d2587bdc748 +P fe9ffe5eed7d376f3f08c78c1ce5514c886f3479 +R 3da43b6baf65219c2aa2e7ef155b204b U drh -Z b332ec0feed7e0dc71dd5e410ae1a5f1 +Z 28f0a6c435a4256f78d56b3bdcebab87 diff --git a/manifest.uuid b/manifest.uuid index 7c074d1963..6d6d79168e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fe9ffe5eed7d376f3f08c78c1ce5514c886f3479 \ No newline at end of file +4a18d8bd4cc66eb08c6198cdf6e14f1bce0ec05a \ No newline at end of file diff --git a/src/build.c b/src/build.c index 98a032bd7c..9635920ea1 100644 --- a/src/build.c +++ b/src/build.c @@ -986,6 +986,8 @@ void sqlite3StartTable( int j1; int fileFormat; int reg1, reg2, reg3; + /* nullRow[] is an OP_Record encoding of a row containing 5 NULLs */ + static const char nullRow[] = { 6, 0, 0, 0, 0, 0 }; sqlite3BeginWriteOperation(pParse, 1, iDb); #ifndef SQLITE_OMIT_VIRTUALTABLE @@ -1030,7 +1032,7 @@ void sqlite3StartTable( } sqlite3OpenMasterTable(pParse, iDb); sqlite3VdbeAddOp2(v, OP_NewRowid, 0, reg1); - sqlite3VdbeAddOp2(v, OP_Null, 0, reg3); + sqlite3VdbeAddOp4(v, OP_Blob, 6, reg3, 0, nullRow, P4_STATIC); sqlite3VdbeAddOp3(v, OP_Insert, 0, reg3, reg1); sqlite3VdbeChangeP5(v, OPFLAG_APPEND); sqlite3VdbeAddOp0(v, OP_Close); diff --git a/test/table.test b/test/table.test index 2aec6473e6..e24e3b9ed4 100644 --- a/test/table.test +++ b/test/table.test @@ -823,5 +823,17 @@ do_execsql_test table-18.2 { PRAGMA integrity_check; } {ok} +# 2015-09-09 +# Ticket [https://www.sqlite.org/src/info/acd12990885d9276] +# "CREATE TABLE ... AS SELECT ... FROM sqlite_master" fails because the row +# in the sqlite_master table for the next table is initially populated +# with a NULL instead of a record created by OP_Record. +# +do_execsql_test table-19.1 { + CREATE TABLE t19 AS SELECT * FROM sqlite_master; + SELECT name FROM t19 ORDER BY name; +} {{} savepoint t10 t11 t12 t13 t16 t2 t3 t3\"xyz t4\"abc t7 t8 t9 tablet8 test1 weird} + + finish_test From 4b03efb6951faf65dbc83fae0e9867a87be15474 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Wed, 9 Sep 2015 17:17:22 +0000 Subject: [PATCH 060/100] Fix harmless compiler warnings in FTS5. FossilOrigin-Name: 2cdb18778f20baa902e54f218ba26d2bacf0801e --- ext/fts5/fts5_expr.c | 7 +------ manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 14 deletions(-) diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index b68634694f..d92de6fd95 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -653,7 +653,7 @@ static int fts5ExprNearAdvanceFirst( i64 iFrom ){ Fts5ExprTerm *pTerm = &pNode->pNear->apPhrase[0]->aTerm[0]; - int rc; + int rc = SQLITE_OK; if( pTerm->pSynonym ){ int bEof = 1; @@ -948,7 +948,6 @@ static int fts5ExprNearNextMatch( for(j=0; jnTerm; j++){ Fts5ExprTerm *pTerm = &pPhrase->aTerm[j]; if( pTerm->pSynonym ){ - Fts5ExprTerm *p; int bEof = 1; i64 iRowid = fts5ExprSynonymRowid(pTerm, bDesc, 0); if( iRowid==iLast ) continue; @@ -1657,13 +1656,9 @@ int sqlite3Fts5ExprClonePhrase( ){ int rc = SQLITE_OK; /* Return code */ Fts5ExprPhrase *pOrig; /* The phrase extracted from pExpr */ - Fts5ExprPhrase *pCopy; /* Copy of pOrig */ int i; /* Used to iterate through phrase terms */ Fts5Expr *pNew = 0; /* Expression to return via *ppNew */ - Fts5ExprPhrase **apPhrase; /* pNew->apPhrase */ - Fts5ExprNode *pNode; /* pNew->pRoot */ - Fts5ExprNearset *pNear; /* pNew->pRoot->pNear */ TokenCtx sCtx = {0,0}; /* Context object for fts5ParseTokenize */ diff --git a/manifest b/manifest index cd0edc5ea2..b052fd8792 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\srunning\sa\sCREATE\sTABLE\sAS,\smake\sthe\sinitial\stemporary\ssqlite_master\sentry\s\nfor\sthe\snew\stable\sa\sreal\srecord\srather\sthan\sa\sNULL,\sin\scase\sthe\squery\safter\s\nthe\sAS\sactually\stries\sto\sread\sthe\ssqlite_master\stable.\nFix\sfor\sticket\s[acd12990885d9276]. -D 2015-09-09T13:28:06.029 +C Fix\sharmless\scompiler\swarnings\sin\sFTS5. +D 2015-09-09T17:17:22.204 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -110,7 +110,7 @@ F ext/fts5/fts5Int.h f65d41f66accad0a289d6bd66b13c07d2932f9be F ext/fts5/fts5_aux.c 7a307760a9c57c750d043188ec0bad59f5b5ec7e F ext/fts5/fts5_buffer.c 80f9ba4431848cb857e3d2158f5280093dcd8015 F ext/fts5/fts5_config.c 80b61fd2c6844b64a3e72a64572d50a812da9384 -F ext/fts5/fts5_expr.c 1c24e1a2ffb286bfe37e537a43b7fadabfe993d4 +F ext/fts5/fts5_expr.c 3f10e630dcdc90f1851ab21f8144d96c7f3a504f F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246 F ext/fts5/fts5_index.c 950e37028cc81ae21534819e79c73aea7efa6c8e F ext/fts5/fts5_main.c e9d0892424bb7f0a8b58613d4ff75cb650cf286e @@ -1383,7 +1383,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P fe9ffe5eed7d376f3f08c78c1ce5514c886f3479 -R 3da43b6baf65219c2aa2e7ef155b204b -U drh -Z 28f0a6c435a4256f78d56b3bdcebab87 +P 4a18d8bd4cc66eb08c6198cdf6e14f1bce0ec05a +R c19171a4b0839ea44fb7f32fd3fd60cd +U mistachkin +Z f42efae15b7a6bd4c85d55b2c5dd8a8e diff --git a/manifest.uuid b/manifest.uuid index 6d6d79168e..e98ea03234 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4a18d8bd4cc66eb08c6198cdf6e14f1bce0ec05a \ No newline at end of file +2cdb18778f20baa902e54f218ba26d2bacf0801e \ No newline at end of file From 3f62cb5aad7b4d5cdb6cf46ba8f816a4462cf015 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Wed, 9 Sep 2015 17:23:48 +0000 Subject: [PATCH 061/100] Fix harmless compiler warning. FossilOrigin-Name: 280fd3a482978b4a488a8b425721e451c2a30745 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index b052fd8792..3bd890e596 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarnings\sin\sFTS5. -D 2015-09-09T17:17:22.204 +C Fix\sharmless\scompiler\swarning. +D 2015-09-09T17:23:48.211 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -416,7 +416,7 @@ F src/vxworks.h c18586c8edc1bddbc15c004fa16aeb1e1342b4fb F src/wal.c 8cd07f1f99e1a81346db1c9da879bef6c6f97cf6 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 2e14d17f592d176b6dc879c33fbdec4fbccaa2ba -F src/where.c ed1cd1cb0434bca9f4a5379582c637bf393b34ac +F src/where.c 1227687e7892d4009f3c3433e974eb9c9e3c4d6a F src/whereInt.h 292d3ac90da4eab1e03ac8452f1add746bcafaa1 F src/wherecode.c 6ac8599523f4840d9efac335329f627ebf3f79fd F src/whereexpr.c 2473e4350e30f9b55d1c6a8f66ca23c689f23f1d @@ -1383,7 +1383,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 4a18d8bd4cc66eb08c6198cdf6e14f1bce0ec05a -R c19171a4b0839ea44fb7f32fd3fd60cd +P 2cdb18778f20baa902e54f218ba26d2bacf0801e +R acfdd2857f10498b6f74f8764b68244c U mistachkin -Z f42efae15b7a6bd4c85d55b2c5dd8a8e +Z 4d9c06d0e6e18e1cffef7b0cc1d40a2c diff --git a/manifest.uuid b/manifest.uuid index e98ea03234..e1ab0294e9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2cdb18778f20baa902e54f218ba26d2bacf0801e \ No newline at end of file +280fd3a482978b4a488a8b425721e451c2a30745 \ No newline at end of file diff --git a/src/where.c b/src/where.c index a29988a8e0..b76b9f2e51 100644 --- a/src/where.c +++ b/src/where.c @@ -271,7 +271,7 @@ static WhereTerm *whereScanInit( u32 opMask, /* Operator(s) to scan for */ Index *pIdx /* Must be compatible with this index */ ){ - int j; + int j = 0; /* memset(pScan, 0, sizeof(*pScan)); */ pScan->pOrigWC = pWC; From 8820250e8daac4a1f7884dd1d489b01f35e015da Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 9 Sep 2015 19:27:10 +0000 Subject: [PATCH 062/100] Fix a possible NULL pointer deref when using SQLITE_ENABLE_MEMORY_MANAGEMENT. FossilOrigin-Name: 89bfdbfe943adce8e02c84ede014fcfed504c5d3 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/pcache1.c | 3 ++- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 3bd890e596..e79b93894d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarning. -D 2015-09-09T17:23:48.211 +C Fix\sa\spossible\sNULL\spointer\sderef\swhen\susing\sSQLITE_ENABLE_MEMORY_MANAGEMENT. +D 2015-09-09T19:27:10.029 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -331,7 +331,7 @@ F src/pager.h 6d435f563b3f7fcae4b84433b76a6ac2730036e2 F src/parse.y f599aa5e871a493330d567ced93de696f61f48f7 F src/pcache.c 24be750c79272e0ca7b6e007bc94999700f3e5ef F src/pcache.h 9968603796240cdf83da7e7bef76edf90619cea9 -F src/pcache1.c bf2afe64a3dedb8643c8dcbd94a145cc80ab2a67 +F src/pcache1.c ca0eeaaf9ffab7a9ded47d54639d8ab147d442e6 F src/pragma.c d71b813e67bf03f3116b9dd5164fbfd81ec673a2 F src/pragma.h 631a91c8b0e6ca8f051a1d8a4a0da4150e04620a F src/prepare.c 82e5db1013846a819f198336fed72c44c974e7b1 @@ -1383,7 +1383,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 2cdb18778f20baa902e54f218ba26d2bacf0801e -R acfdd2857f10498b6f74f8764b68244c -U mistachkin -Z 4d9c06d0e6e18e1cffef7b0cc1d40a2c +P 280fd3a482978b4a488a8b425721e451c2a30745 +R 931638393b6c80a6b7791df11d019edc +U drh +Z 7c6427f02c08a37206c926b94f51cdad diff --git a/manifest.uuid b/manifest.uuid index e1ab0294e9..8018c18b5a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -280fd3a482978b4a488a8b425721e451c2a30745 \ No newline at end of file +89bfdbfe943adce8e02c84ede014fcfed504c5d3 \ No newline at end of file diff --git a/src/pcache1.c b/src/pcache1.c index 2fed0bdff1..1beb66d733 100644 --- a/src/pcache1.c +++ b/src/pcache1.c @@ -1193,7 +1193,8 @@ int sqlite3PcacheReleaseMemory(int nReq){ PgHdr1 *p; pcache1EnterMutex(&pcache1.grp); while( (nReq<0 || nFreeisAnchor==0 + && (p=pcache1.grp.lru.pLruPrev)!=0 + && p->isAnchor==0 ){ nFree += pcache1MemSize(p->page.pBuf); #ifdef SQLITE_PCACHE_SEPARATE_HEADER From 5c472d0860da5b2ae7dd780164c6b778d12300e9 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 9 Sep 2015 19:44:33 +0000 Subject: [PATCH 063/100] Fix an assert() enabled by SQLITE_ENABLE_EXPENSIVE_ASSERT in wal.c. FossilOrigin-Name: 8d2ed150a7a15626965cf994ef48c3ab61eca6ec --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/wal.c | 3 ++- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index e79b93894d..ea4c89e5fb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\spossible\sNULL\spointer\sderef\swhen\susing\sSQLITE_ENABLE_MEMORY_MANAGEMENT. -D 2015-09-09T19:27:10.029 +C Fix\san\sassert()\senabled\sby\sSQLITE_ENABLE_EXPENSIVE_ASSERT\sin\swal.c. +D 2015-09-09T19:44:33.776 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -413,7 +413,7 @@ F src/vdbesort.c f5009e7a35e3065635d8918b9a31f498a499976b F src/vdbetrace.c 8befe829faff6d9e6f6e4dee5a7d3f85cc85f1a0 F src/vtab.c 2ecfe020c10e0a0c7b078203fdba2fae844744bc F src/vxworks.h c18586c8edc1bddbc15c004fa16aeb1e1342b4fb -F src/wal.c 8cd07f1f99e1a81346db1c9da879bef6c6f97cf6 +F src/wal.c 18b0ed49830cf04fe2d68224b41838a73ac6cd24 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 2e14d17f592d176b6dc879c33fbdec4fbccaa2ba F src/where.c 1227687e7892d4009f3c3433e974eb9c9e3c4d6a @@ -1383,7 +1383,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 280fd3a482978b4a488a8b425721e451c2a30745 -R 931638393b6c80a6b7791df11d019edc -U drh -Z 7c6427f02c08a37206c926b94f51cdad +P 89bfdbfe943adce8e02c84ede014fcfed504c5d3 +R d3f44bc9ae925ae28349d99ef74ecbc2 +U dan +Z 689075f9d506170808fdd1fd6d304ed8 diff --git a/manifest.uuid b/manifest.uuid index 8018c18b5a..dc4f92af09 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -89bfdbfe943adce8e02c84ede014fcfed504c5d3 \ No newline at end of file +8d2ed150a7a15626965cf994ef48c3ab61eca6ec \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index cf8f1d4e66..d87d2c17ce 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2462,7 +2462,8 @@ int sqlite3WalFindFrame( { u32 iRead2 = 0; u32 iTest; - for(iTest=iLast; iTest>0; iTest--){ + assert( pWal->minFrame>0 ); + for(iTest=iLast; iTest>=pWal->minFrame; iTest--){ if( walFramePgno(pWal, iTest)==pgno ){ iRead2 = iTest; break; From f5eac36a64365b7e5a768794262a317f41ae9aff Mon Sep 17 00:00:00 2001 From: mistachkin Date: Wed, 9 Sep 2015 23:54:46 +0000 Subject: [PATCH 064/100] Fix harmless compiler warning in FTS5. FossilOrigin-Name: 86146a731d75eb25279c0e072c0bdda593de905d --- ext/fts5/fts5_expr.c | 1 - manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index d92de6fd95..4cfc1b155c 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -318,7 +318,6 @@ static int fts5ExprSynonymPoslist( int *pbDel, /* OUT: Caller should sqlite3_free(*pa) */ u8 **pa, int *pn ){ - Fts5PoslistWriter writer = {0}; Fts5PoslistReader aStatic[4]; Fts5PoslistReader *aIter = aStatic; int nIter = 0; diff --git a/manifest b/manifest index ea4c89e5fb..46ef30b077 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sassert()\senabled\sby\sSQLITE_ENABLE_EXPENSIVE_ASSERT\sin\swal.c. -D 2015-09-09T19:44:33.776 +C Fix\sharmless\scompiler\swarning\sin\sFTS5. +D 2015-09-09T23:54:46.415 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -110,7 +110,7 @@ F ext/fts5/fts5Int.h f65d41f66accad0a289d6bd66b13c07d2932f9be F ext/fts5/fts5_aux.c 7a307760a9c57c750d043188ec0bad59f5b5ec7e F ext/fts5/fts5_buffer.c 80f9ba4431848cb857e3d2158f5280093dcd8015 F ext/fts5/fts5_config.c 80b61fd2c6844b64a3e72a64572d50a812da9384 -F ext/fts5/fts5_expr.c 3f10e630dcdc90f1851ab21f8144d96c7f3a504f +F ext/fts5/fts5_expr.c a7726fe7045eec7caca8a074af747c8ea3545b83 F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246 F ext/fts5/fts5_index.c 950e37028cc81ae21534819e79c73aea7efa6c8e F ext/fts5/fts5_main.c e9d0892424bb7f0a8b58613d4ff75cb650cf286e @@ -1383,7 +1383,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 89bfdbfe943adce8e02c84ede014fcfed504c5d3 -R d3f44bc9ae925ae28349d99ef74ecbc2 -U dan -Z 689075f9d506170808fdd1fd6d304ed8 +P 8d2ed150a7a15626965cf994ef48c3ab61eca6ec +R 9c684262bd2bed914ee53743d3728f7e +U mistachkin +Z 5aaf25a787ca8169e60e163e7c3cad6b diff --git a/manifest.uuid b/manifest.uuid index dc4f92af09..8f69f65dcd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8d2ed150a7a15626965cf994ef48c3ab61eca6ec \ No newline at end of file +86146a731d75eb25279c0e072c0bdda593de905d \ No newline at end of file From 5fb72e5f3ef4b6c65df991bb64a577089822db25 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 10 Sep 2015 01:22:09 +0000 Subject: [PATCH 065/100] No-op the sqlite3_memory_alarm() interface in a different way, that does not break legacy memory behavior. This is a re-do of check-in [5d3f5df4da9f40d5]. FossilOrigin-Name: 8250e2a487ee12c9a2dea5603ab60aed51e5dc7b --- manifest | 14 +++---- manifest.uuid | 2 +- src/malloc.c | 104 +++++++++++++++++++++----------------------------- 3 files changed, 52 insertions(+), 68 deletions(-) diff --git a/manifest b/manifest index 46ef30b077..19012fac78 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarning\sin\sFTS5. -D 2015-09-09T23:54:46.415 +C No-op\sthe\ssqlite3_memory_alarm()\sinterface\sin\sa\sdifferent\sway,\sthat\sdoes\nnot\sbreak\slegacy\smemory\sbehavior.\s\sThis\sis\sa\sre-do\sof\ncheck-in\s[5d3f5df4da9f40d5]. +D 2015-09-10T01:22:09.178 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -305,7 +305,7 @@ F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c d344a95d60c24e2f490ee59db9784b1b17439012 F src/loadext.c dfcee8c7c032cd0fd55af3e0fc1fcfb01e426df2 F src/main.c e17fcffae4306a9b8334faf3bac80d7396850b54 -F src/malloc.c 021012e28a81ffdabf4c30ec3df6ce1f6cc93f1d +F src/malloc.c 3a37ce6979a40f499d8cea9e9ab4e8517854d35d F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c abe6ee469b6c5a35c7f22bfeb9c9bac664a1c987 F src/mem2.c f1940d9e91948dd6a908fbb9ce3835c36b5d83c3 @@ -1383,7 +1383,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 8d2ed150a7a15626965cf994ef48c3ab61eca6ec -R 9c684262bd2bed914ee53743d3728f7e -U mistachkin -Z 5aaf25a787ca8169e60e163e7c3cad6b +P 86146a731d75eb25279c0e072c0bdda593de905d +R 7c5553c63d5721a9d16f1642ebb06bea +U drh +Z 0fce0dbaade5235ccf22a67355771bfc diff --git a/manifest.uuid b/manifest.uuid index 8f69f65dcd..5a207b3806 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -86146a731d75eb25279c0e072c0bdda593de905d \ No newline at end of file +8250e2a487ee12c9a2dea5603ab60aed51e5dc7b \ No newline at end of file diff --git a/src/malloc.c b/src/malloc.c index 23d7598ae3..f20eb6e796 100644 --- a/src/malloc.c +++ b/src/malloc.c @@ -45,7 +45,7 @@ typedef struct ScratchFreeslot { */ static SQLITE_WSD struct Mem0Global { sqlite3_mutex *mutex; /* Mutex to serialize access */ - sqlite3_int64 alarmThreshold; /* The soft heap limit */ + sqlite3_int64 alarmThreshold; /* The soft heap limit */ /* ** Pointers to the end of sqlite3GlobalConfig.pScratch memory @@ -73,59 +73,20 @@ sqlite3_mutex *sqlite3MallocMutex(void){ return mem0.mutex; } -/* -** Return the amount of memory currently in use. -*/ -static sqlite3_int64 memInUse(void){ - assert( sqlite3_mutex_held(mem0.mutex) ); - return sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); -} - -/* -** Called when the soft heap limit is exceeded for an allocation -** of nBytes. -*/ -#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT -static void sqlite3HeapLimitExceeded(int nByte){ - sqlite3_int64 excess = memInUse() + nByte - mem0.alarmThreshold; - sqlite3_mutex_leave(mem0.mutex); - sqlite3_release_memory((int)(excess & 0x7fffffff)); - sqlite3_mutex_enter(mem0.mutex); -} -#else -# define sqlite3HeapLimitExceeded(X) /* no-op */ -#endif - -/* -** Check to see if increasing the total memory usage by nNew bytes -** will exceed the soft heap limit. -** -** If the soft heap limit is exceeded, set the mem0.nearlyFull flag -** and invoke sqlite3HeapLimitExceeded() to try to free up some -** memory. -*/ -static void sqlite3CheckSoftHeapLimit(int nNew){ - assert( sqlite3_mutex_held(mem0.mutex) ); - if( mem0.alarmThreshold>0 ){ - if( mem0.alarmThreshold-nNew >= memInUse() ){ - mem0.nearlyFull = 1; - sqlite3HeapLimitExceeded(nNew); - }else{ - mem0.nearlyFull = 0; - } - } -} - #ifndef SQLITE_OMIT_DEPRECATED /* -** Deprecated external interface. First deprecated 2007-11-05. Changed -** into a no-op on 2015-09-02. +** Deprecated external interface. It used to set an alarm callback +** that was invoked when memory usage grew too large. Now it is a +** no-op. */ int sqlite3_memory_alarm( void(*xCallback)(void *pArg, sqlite3_int64 used,int N), void *pArg, sqlite3_int64 iThreshold ){ + (void)xCallback; + (void)pArg; + (void)iThreshold; return SQLITE_OK; } #endif @@ -136,20 +97,24 @@ int sqlite3_memory_alarm( */ sqlite3_int64 sqlite3_soft_heap_limit64(sqlite3_int64 n){ sqlite3_int64 priorLimit; + sqlite3_int64 excess; + sqlite3_int64 nUsed; #ifndef SQLITE_OMIT_AUTOINIT int rc = sqlite3_initialize(); if( rc ) return -1; #endif sqlite3_mutex_enter(mem0.mutex); priorLimit = mem0.alarmThreshold; - if( n>0 ){ - mem0.alarmThreshold = n; - sqlite3CheckSoftHeapLimit(0); - }else if( n==0 ){ - mem0.alarmThreshold = 0; - mem0.nearlyFull = 0; + if( n<0 ){ + sqlite3_mutex_leave(mem0.mutex); + return priorLimit; } + mem0.alarmThreshold = n; + nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); + mem0.nearlyFull = (n>0 && n<=nUsed); sqlite3_mutex_leave(mem0.mutex); + excess = sqlite3_memory_used() - n; + if( excess>0 ) sqlite3_release_memory((int)(excess & 0x7fffffff)); return priorLimit; } void sqlite3_soft_heap_limit(int n){ @@ -240,6 +205,16 @@ sqlite3_int64 sqlite3_memory_highwater(int resetFlag){ return mx; } +/* +** Trigger the alarm +*/ +static void sqlite3MallocAlarm(int nByte){ + if( mem0.alarmThreshold<=0 ) return; + sqlite3_mutex_leave(mem0.mutex); + sqlite3_release_memory(nByte); + sqlite3_mutex_enter(mem0.mutex); +} + /* ** Do a memory allocation with statistics and alarms. Assume the ** lock is already held. @@ -250,11 +225,19 @@ static int mallocWithAlarm(int n, void **pp){ assert( sqlite3_mutex_held(mem0.mutex) ); nFull = sqlite3GlobalConfig.m.xRoundup(n); sqlite3StatusSet(SQLITE_STATUS_MALLOC_SIZE, n); - sqlite3CheckSoftHeapLimit(nFull); + if( mem0.alarmThreshold>0 ){ + sqlite3_int64 nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); + if( nUsed >= mem0.alarmThreshold - nFull ){ + mem0.nearlyFull = 1; + sqlite3MallocAlarm(nFull); + }else{ + mem0.nearlyFull = 0; + } + } p = sqlite3GlobalConfig.m.xMalloc(nFull); #ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT - if( p==0 && mem0.alarmThreshold ){ - sqlite3HeapLimitExceeded(nFull); + if( p==0 && mem0.alarmThreshold>0 ){ + sqlite3MallocAlarm(nFull); p = sqlite3GlobalConfig.m.xMalloc(nFull); } #endif @@ -537,14 +520,15 @@ void *sqlite3Realloc(void *pOld, u64 nBytes){ sqlite3_mutex_enter(mem0.mutex); sqlite3StatusSet(SQLITE_STATUS_MALLOC_SIZE, (int)nBytes); nDiff = nNew - nOld; - sqlite3CheckSoftHeapLimit(nDiff); + if( sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED) >= + mem0.alarmThreshold-nDiff ){ + sqlite3MallocAlarm(nDiff); + } pNew = sqlite3GlobalConfig.m.xRealloc(pOld, nNew); -#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT - if( pNew==0 && mem0.alarmThreshold ){ - sqlite3HeapLimitExceeded((int)nBytes); + if( pNew==0 && mem0.alarmThreshold>0 ){ + sqlite3MallocAlarm((int)nBytes); pNew = sqlite3GlobalConfig.m.xRealloc(pOld, nNew); } -#endif if( pNew ){ nNew = sqlite3MallocSize(pNew); sqlite3StatusUp(SQLITE_STATUS_MEMORY_USED, nNew-nOld); From 852944eb9b750445956c156c89e54dcbdc843629 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 10 Sep 2015 03:29:11 +0000 Subject: [PATCH 066/100] Fix the json_tree() scan for the case when a path is supplied. Add new json1 test cases. FossilOrigin-Name: 6adc7de76acee6cfb5ff761739e7a8de7b5bf4b2 --- ext/misc/json1.c | 63 +++++++----- manifest | 13 +-- manifest.uuid | 2 +- test/json102.test | 238 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 284 insertions(+), 32 deletions(-) create mode 100644 test/json102.test diff --git a/ext/misc/json1.c b/ext/misc/json1.c index 5df7551dec..d51d69e1ee 100644 --- a/ext/misc/json1.c +++ b/ext/misc/json1.c @@ -83,6 +83,7 @@ static const char * const jsonType[] = { #define JNODE_REPLACE 0x08 /* Replace with JsonNode.iVal */ #define JNODE_APPEND 0x10 /* More ARRAY/OBJECT entries at u.iAppend */ #define JNODE_JSON 0x20 /* Treat REPLACE as JSON text */ +#define JNODE_LABEL 0x40 /* Is a label of an object */ /* A single node of parsed JSON @@ -583,6 +584,7 @@ static int jsonParseValue(JsonParse *pParse, u32 i){ u32 j; int iThis; int x; + JsonNode *pNode; while( isspace(pParse->zJson[i]) ){ i++; } if( (c = pParse->zJson[i])==0 ) return 0; if( c=='{' ){ @@ -597,7 +599,9 @@ static int jsonParseValue(JsonParse *pParse, u32 i){ return -1; } if( pParse->oom ) return -1; - if( pParse->aNode[pParse->nNode-1].eType!=JSON_STRING ) return -1; + pNode = &pParse->aNode[pParse->nNode-1]; + if( pNode->eType!=JSON_STRING ) return -1; + pNode->jnFlags |= JNODE_LABEL; j = x; while( isspace(pParse->zJson[j]) ){ j++; } if( pParse->zJson[j]!=':' ) return -1; @@ -1036,13 +1040,20 @@ static void jsonParseFunc( jsonParseFindParents(&x); jsonInit(&s, ctx); for(i=0; ibRecursive ){ - if( p->i==0 ){ - p->i = 1; - }else{ - u32 iUp = p->sParse.aUp[p->i]; - JsonNode *pUp = &p->sParse.aNode[iUp]; - p->i++; - if( pUp->eType==JSON_OBJECT && (pUp->n + iUp >= p->i) ) p->i++; - } + if( p->sParse.aNode[p->i].jnFlags & JNODE_LABEL ) p->i++; + p->i++; p->iRowid++; - if( p->isParse.nNode ){ + if( p->iiEnd ){ u32 iUp = p->sParse.aUp[p->i]; JsonNode *pUp = &p->sParse.aNode[iUp]; p->eType = pUp->eType; @@ -1597,8 +1603,9 @@ static void jsonEachComputePath( jsonPrintf(30, pStr, "[%d]", pUp->u.iKey); }else{ assert( pUp->eType==JSON_OBJECT ); - if( pNode->eType>=JSON_ARRAY ) pNode--; + if( (pNode->jnFlags & JNODE_LABEL)==0 ) pNode--; assert( pNode->eType==JSON_STRING ); + assert( pNode->jnFlags & JNODE_LABEL ); jsonPrintf(pNode->n+1, pStr, ".%.*s", pNode->n-2, pNode->u.zJContent+1); } } @@ -1629,27 +1636,28 @@ static int jsonEachColumn( break; } case JEACH_VALUE: { - if( p->eType==JSON_OBJECT && p->i>0 ) pThis++; + if( pThis->jnFlags & JNODE_LABEL ) pThis++; jsonReturn(pThis, ctx, 0); break; } case JEACH_TYPE: { - if( p->eType==JSON_OBJECT && p->i>0 ) pThis++; + if( pThis->jnFlags & JNODE_LABEL ) pThis++; sqlite3_result_text(ctx, jsonType[pThis->eType], -1, SQLITE_STATIC); break; } case JEACH_ATOM: { - if( p->eType==JSON_OBJECT && p->i>0 ) pThis++; + if( pThis->jnFlags & JNODE_LABEL ) pThis++; if( pThis->eType>=JSON_ARRAY ) break; jsonReturn(pThis, ctx, 0); break; } case JEACH_ID: { - sqlite3_result_int64(ctx, (sqlite3_int64)p->i + (p->eType==JSON_OBJECT)); + sqlite3_result_int64(ctx, + (sqlite3_int64)p->i + ((pThis->jnFlags & JNODE_LABEL)!=0)); break; } case JEACH_PARENT: { - if( p->i>0 && p->bRecursive ){ + if( p->i>p->iBegin && p->bRecursive ){ sqlite3_result_int64(ctx, (sqlite3_int64)p->sParse.aUp[p->i]); } break; @@ -1794,7 +1802,6 @@ static int jsonEachFilter( JsonNode *pNode; if( idxNum==3 ){ const char *zErr = 0; - p->bRecursive = 0; n = sqlite3_value_bytes(argv[1]); p->zPath = sqlite3_malloc64( n+1 ); if( p->zPath==0 ) return SQLITE_NOMEM; @@ -1811,12 +1818,18 @@ static int jsonEachFilter( }else{ pNode = p->sParse.aNode; } - p->i = (int)(pNode - p->sParse.aNode); + p->iBegin = p->i = (int)(pNode - p->sParse.aNode); p->eType = pNode->eType; if( p->eType>=JSON_ARRAY ){ pNode->u.iKey = 0; p->iEnd = p->i + pNode->n + 1; - if( !p->bRecursive ) p->i++; + if( p->bRecursive ){ + if( p->i>0 && (p->sParse.aNode[p->i-1].jnFlags & JNODE_LABEL)!=0 ){ + p->i--; + } + }else{ + p->i++; + } }else{ p->iEnd = p->i+1; } diff --git a/manifest b/manifest index 19012fac78..7b61a44b5a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C No-op\sthe\ssqlite3_memory_alarm()\sinterface\sin\sa\sdifferent\sway,\sthat\sdoes\nnot\sbreak\slegacy\smemory\sbehavior.\s\sThis\sis\sa\sre-do\sof\ncheck-in\s[5d3f5df4da9f40d5]. -D 2015-09-10T01:22:09.178 +C Fix\sthe\sjson_tree()\sscan\sfor\sthe\scase\swhen\sa\spath\sis\ssupplied.\s\sAdd\snew\njson1\stest\scases. +D 2015-09-10T03:29:11.778 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -194,7 +194,7 @@ F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f F ext/misc/fuzzer.c 4c84635c71c26cfa7c2e5848cf49fe2d2cfcd767 F ext/misc/ieee754.c b0362167289170627659e84173f5d2e8fee8566e -F ext/misc/json1.c bd51e8c1e8ce580e6f21493bd8e94ed5aca3d777 +F ext/misc/json1.c 46c2aff110eb8241591cd29267ddb57a8d0ab337 F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c bcbee3c061b884eccb80e21651daaae8e1e43c63 F ext/misc/regexp.c af92cdaa5058fcec1451e49becc7ba44dba023dc @@ -814,6 +814,7 @@ F test/jrnlmode.test 7864d59cf7f6e552b9b99ba0f38acd167edc10fa F test/jrnlmode2.test 81610545a4e6ed239ea8fa661891893385e23a1d F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa F test/json101.test 11535d8986184500f4c30cc2f0b154b4ab05cc4e +F test/json102.test ab2ea59639ecf2ad95101015c3357b1ba53ebd98 F test/keyword1.test 37ef6bba5d2ed5b07ecdd6810571de2956599dff F test/lastinsert.test 42e948fd6442f07d60acbd15d33fb86473e0ef63 F test/laststmtchanges.test ae613f53819206b3222771828d024154d51db200 @@ -1383,7 +1384,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 86146a731d75eb25279c0e072c0bdda593de905d -R 7c5553c63d5721a9d16f1642ebb06bea +P 8250e2a487ee12c9a2dea5603ab60aed51e5dc7b +R c3aaad669751ca08269190cf668389ec U drh -Z 0fce0dbaade5235ccf22a67355771bfc +Z cc385c8c27ee83d7a7141ac119ad0322 diff --git a/manifest.uuid b/manifest.uuid index 5a207b3806..fe04945ac4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8250e2a487ee12c9a2dea5603ab60aed51e5dc7b \ No newline at end of file +6adc7de76acee6cfb5ff761739e7a8de7b5bf4b2 \ No newline at end of file diff --git a/test/json102.test b/test/json102.test new file mode 100644 index 0000000000..4ef1feb083 --- /dev/null +++ b/test/json102.test @@ -0,0 +1,238 @@ +# 2015-08-12 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements tests for JSON SQL functions extension to the +# SQLite library. +# +# This file contains tests automatically generated from the json1 +# documentation. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +load_static_extension db json +do_execsql_test json102-100 { + SELECT json_array(1,2,'3',4); +} {{[1,2,"3",4]}} +do_execsql_test json102-110 { + SELECT json_array('[1,2]'); +} {{["[1,2]"]}} +do_execsql_test json102-120 { + SELECT json_array(1,null,'3','[4,5]','{"six":7.7}'); +} {{[1,null,"3","[4,5]","{\"six\":7.7}"]}} +do_execsql_test json102-130 { + SELECT json_array_length('[1,2,3,4]'); +} {{4}} +do_execsql_test json102-140 { + SELECT json_array_length('{"one":[1,2,3]}'); +} {{0}} +do_execsql_test json102-150 { + SELECT json_array_length('{"one":[1,2,3]}', '$.one'); +} {{3}} +do_execsql_test json102-160 { + SELECT json_extract('{"a":2,"c":[4,5,{"f":7}]}', '$'); +} {{{"a":2,"c":[4,5,{"f":7}]}}} +do_execsql_test json102-170 { + SELECT json_extract('{"a":2,"c":[4,5,{"f":7}]}', '$.c'); +} {{[4,5,{"f":7}]}} +do_execsql_test json102-180 { + SELECT json_extract('{"a":2,"c":[4,5,{"f":7}]}', '$.c[2]'); +} {{{"f":7}}} +do_execsql_test json102-190 { + SELECT json_extract('{"a":2,"c":[4,5,{"f":7}]}', '$.c[2].f'); +} {{7}} +do_execsql_test json102-200 { + SELECT json_extract('{"a":2,"c":[4,5],"f":7}','$.c','$.a'); +} {{[[4,5],2]}} +do_execsql_test json102-210 { + SELECT json_extract('{"a":2,"c":[4,5,{"f":7}]}', '$.x'); +} {{}} +do_execsql_test json102-220 { + SELECT json_insert('{"a":2,"c":4}', '$.a', 99); +} {{{"a":2,"c":4}}} +do_execsql_test json102-230 { + SELECT json_insert('{"a":2,"c":4}', '$.e', 99); +} {{{"a":2,"c":4,"e":99}}} +do_execsql_test json102-240 { + SELECT json_replace('{"a":2,"c":4}', '$.a', 99); +} {{{"a":99,"c":4}}} +do_execsql_test json102-250 { + SELECT json_replace('{"a":2,"c":4}', '$.e', 99); +} {{{"a":2,"c":4}}} +do_execsql_test json102-260 { + SELECT json_set('{"a":2,"c":4}', '$.a', 99); +} {{{"a":99,"c":4}}} +do_execsql_test json102-270 { + SELECT json_set('{"a":2,"c":4}', '$.e', 99); +} {{{"a":2,"c":4,"e":99}}} +do_execsql_test json102-280 { + SELECT json_object('a',2,'c',4); +} {{{"a":2,"c":4}}} +do_execsql_test json102-290 { + SELECT json_object('a',2,'c','{e:5}'); +} {{{"a":2,"c":"{e:5}"}}} +do_execsql_test json102-300 { + SELECT json_remove('[0,1,2,3,4]','$[2]'); +} {{[0,1,3,4]}} +do_execsql_test json102-310 { + SELECT json_remove('[0,1,2,3,4]','$[2]','$[0]'); +} {{[1,3,4]}} +do_execsql_test json102-320 { + SELECT json_remove('[0,1,2,3,4]','$[0]','$[2]'); +} {{[1,2,4]}} +do_execsql_test json102-330 { + SELECT json_remove('{"x":25,"y":42}'); +} {{{"x":25,"y":42}}} +do_execsql_test json102-340 { + SELECT json_remove('{"x":25,"y":42}','$.z'); +} {{{"x":25,"y":42}}} +do_execsql_test json102-350 { + SELECT json_remove('{"x":25,"y":42}','$.y'); +} {{{"x":25}}} +do_execsql_test json102-360 { + SELECT json_remove('{"x":25,"y":42}','$'); +} {{}} +do_execsql_test json102-370 { + SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}'); +} {{object}} +do_execsql_test json102-380 { + SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$'); +} {{object}} +do_execsql_test json102-390 { + SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a'); +} {{array}} +do_execsql_test json102-400 { + SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a[0]'); +} {{integer}} +do_execsql_test json102-410 { + SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a[1]'); +} {{real}} +do_execsql_test json102-420 { + SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a[2]'); +} {{true}} +do_execsql_test json102-430 { + SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a[3]'); +} {{false}} +do_execsql_test json102-440 { + SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a[4]'); +} {{null}} +do_execsql_test json102-450 { + SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a[5]'); +} {{text}} +do_execsql_test json102-460 { + SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a[6]'); +} {{}} +do_execsql_test json102-470 { + SELECT json_valid('{"x":35}'); +} {{1}} +do_execsql_test json102-480 { + SELECT json_valid('{"x":35'); -- } +} {{0}} + +do_execsql_test json102-500 { + CREATE TABLE user(name,phone); + INSERT INTO user(name,phone) VALUES + ('Alice','["919-555-2345","804-555-3621"]'), + ('Bob','["201-555-8872"]'), + ('Cindy','["704-555-9983"]'), + ('Dave','["336-555-8421","704-555-4321","803-911-4421"]'); + SELECT DISTINCT user.name + FROM user, json_each(user.phone) + WHERE json_each.value LIKE '704-%' + ORDER BY 1; +} {Cindy Dave} + +do_execsql_test json102-510 { + UPDATE user + SET phone=json_extract(phone,'$[0]') + WHERE json_array_length(phone)<2; + SELECT name, substr(phone,1,5) FROM user ORDER BY name; +} {Alice {["919} Bob 201-5 Cindy 704-5 Dave {["336}} +do_execsql_test json102-511 { + SELECT name FROM user WHERE phone LIKE '704-%' + UNION + SELECT user.name + FROM user, json_each(user.phone) + WHERE json_valid(user.phone) + AND json_each.value LIKE '704-%'; +} {Cindy Dave} + +do_execsql_test json102-600 { + CREATE TABLE big(json JSON); + INSERT INTO big(json) VALUES('{ + "id":123, + "stuff":[1,2,3,4], + "partlist":[ + {"uuid":"bb108722-572e-11e5-9320-7f3b63a4ca74"}, + {"uuid":"c690dc14-572e-11e5-95f9-dfc8861fd535"}, + {"subassembly":[ + {"uuid":"6fa5181e-5721-11e5-a04e-57f3d7b32808"} + ]} + ] + }'); + INSERT INTO big(json) VALUES('{ + "id":456, + "stuff":["hello","world","xyzzy"], + "partlist":[ + {"uuid":false}, + {"uuid":"c690dc14-572e-11e5-95f9-dfc8861fd535"} + ] + }'); +} {} +set correct_answer [list \ + 1 {$.id} 123 \ + 1 {$.stuff[0]} 1 \ + 1 {$.stuff[1]} 2 \ + 1 {$.stuff[2]} 3 \ + 1 {$.stuff[3]} 4 \ + 1 {$.partlist[0].uuid} bb108722-572e-11e5-9320-7f3b63a4ca74 \ + 1 {$.partlist[1].uuid} c690dc14-572e-11e5-95f9-dfc8861fd535 \ + 1 {$.partlist[2].subassembly[0].uuid} 6fa5181e-5721-11e5-a04e-57f3d7b32808 \ + 2 {$.id} 456 \ + 2 {$.stuff[0]} hello \ + 2 {$.stuff[1]} world \ + 2 {$.stuff[2]} xyzzy \ + 2 {$.partlist[0].uuid} 0 \ + 2 {$.partlist[1].uuid} c690dc14-572e-11e5-95f9-dfc8861fd535] +do_execsql_test json102-610 { + SELECT big.rowid, fullkey, value + FROM big, json_tree(big.json) + WHERE json_tree.type NOT IN ('object','array') + ORDER BY +big.rowid, +json_tree.id +} $correct_answer +do_execsql_test json102-620 { + SELECT big.rowid, fullkey, atom + FROM big, json_tree(big.json) + WHERE atom IS NOT NULL + ORDER BY +big.rowid, +json_tree.id +} $correct_answer + +do_execsql_test json102-630 { + SELECT DISTINCT json_extract(big.json,'$.id') + FROM big, json_tree(big.json,'$.partlist') + WHERE json_tree.key='uuid' + AND json_tree.value='6fa5181e-5721-11e5-a04e-57f3d7b32808'; +} {123} +do_execsql_test json102-631 { + SELECT DISTINCT json_extract(big.json,'$.id') + FROM big, json_tree(big.json,'$') + WHERE json_tree.key='uuid' + AND json_tree.value='6fa5181e-5721-11e5-a04e-57f3d7b32808'; +} {123} +do_execsql_test json102-632 { + SELECT DISTINCT json_extract(big.json,'$.id') + FROM big, json_tree(big.json) + WHERE json_tree.key='uuid' + AND json_tree.value='6fa5181e-5721-11e5-a04e-57f3d7b32808'; +} {123} + + +finish_test From 0e8729dbe40f0cda0bf446729ee64b0b846b05d2 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 10 Sep 2015 04:17:06 +0000 Subject: [PATCH 067/100] Attempt to declare sqlite3MemoryBarrier() correctly for all possible build configurations. FossilOrigin-Name: da8a288f8ef4be34281519b4b4db9b857b9d168b --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/mutex.h | 1 - src/sqliteInt.h | 4 ++++ 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 7b61a44b5a..97f96d0743 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\sjson_tree()\sscan\sfor\sthe\scase\swhen\sa\spath\sis\ssupplied.\s\sAdd\snew\njson1\stest\scases. -D 2015-09-10T03:29:11.778 +C Attempt\sto\sdeclare\ssqlite3MemoryBarrier()\scorrectly\sfor\sall\spossible\nbuild\sconfigurations. +D 2015-09-10T04:17:06.273 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -314,7 +314,7 @@ F src/mem5.c 61eeb90134f9a5be6c2e68d8daae7628b25953fb F src/memjournal.c 3eb2c0b51adbd869cb6a44780323f05fa904dc85 F src/msvc.h d9ba56c6851227ab44b3f228a35f3f5772296495 F src/mutex.c a39809c6c33f1ebc9cc781186c338ad90433e1e7 -F src/mutex.h 012503b51ccfcf85b8b3846709a4c60a5839f16c +F src/mutex.h 779d588e3b7756ec3ecf7d78cde1d84aba414f85 F src/mutex_noop.c 9d4309c075ba9cc7249e19412d3d62f7f94839c4 F src/mutex_unix.c 7762c8ec907379204f2ed751a0e33663ab1c14d7 F src/mutex_w32.c 2e025e6642eaf27597403690980f560d1a91f62c @@ -344,7 +344,7 @@ F src/shell.c 6332ef06db1390ef812cfdff1fc97b4fd76cdd42 F src/sqlite.h.in 378bebc8fe6a88bade25e5f23b7e6123fdc64b00 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h f700e6a9dd1fdcccc9951ab022b366fb66b9e413 -F src/sqliteInt.h dba8add0b95fcea1047af7a6875a1e305fcb96d7 +F src/sqliteInt.h 788dc0ea7ba32ec9fec06c628c1792d7b4753d86 F src/sqliteLimit.h 216557999cb45f2e3578ed53ebefe228d779cb46 F src/status.c f266ad8a2892d659b74f0f50cb6a88b6e7c12179 F src/table.c 51b46b2a62d1b3a959633d593b89bab5e2c9155e @@ -1384,7 +1384,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 8250e2a487ee12c9a2dea5603ab60aed51e5dc7b -R c3aaad669751ca08269190cf668389ec +P 6adc7de76acee6cfb5ff761739e7a8de7b5bf4b2 +R ab2159c48cb0bcac4127d05e9b297a43 U drh -Z cc385c8c27ee83d7a7141ac119ad0322 +Z 09155c7a88a3966f473df9eae10e03b4 diff --git a/manifest.uuid b/manifest.uuid index fe04945ac4..652e00e17b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6adc7de76acee6cfb5ff761739e7a8de7b5bf4b2 \ No newline at end of file +da8a288f8ef4be34281519b4b4db9b857b9d168b \ No newline at end of file diff --git a/src/mutex.h b/src/mutex.h index 8bcf2353f7..03eb1faadb 100644 --- a/src/mutex.h +++ b/src/mutex.h @@ -64,7 +64,6 @@ #define sqlite3MutexAlloc(X) ((sqlite3_mutex*)8) #define sqlite3MutexInit() SQLITE_OK #define sqlite3MutexEnd() -#define sqlite3MemoryBarrier() #define MUTEX_LOGIC(X) #else #define MUTEX_LOGIC(X) X diff --git a/src/sqliteInt.h b/src/sqliteInt.h index eeb0b9950c..98cbca5193 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3191,7 +3191,11 @@ const sqlite3_mem_methods *sqlite3MemGetMemsys5(void); sqlite3_mutex *sqlite3MutexAlloc(int); int sqlite3MutexInit(void); int sqlite3MutexEnd(void); +#endif +#if !defined(SQLITE_MUTEX_OMIT) && !defined(SQLITE_MUTEX_NOOP) void sqlite3MemoryBarrier(void); +#else +# define sqlite3MemoryBarrier(); #endif sqlite3_int64 sqlite3StatusValue(int); From a5aa8e1db2e2eef9e00e67798fc67dfedef3cfa8 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 10 Sep 2015 05:40:17 +0000 Subject: [PATCH 068/100] Change the array of 16-bit offsets at the end of each page to an array of varints. FossilOrigin-Name: fab245bea4f283714c17bca22428d5eb4db5935a --- ext/fts5/fts5_index.c | 222 +++++++++++++++++----------------- ext/fts5/test/fts5simple.test | 58 ++++++++- ext/fts5/tool/loadfts5.tcl | 17 ++- manifest | 16 +-- manifest.uuid | 2 +- 5 files changed, 191 insertions(+), 124 deletions(-) diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 9bc08e7443..a7e654d745 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -378,6 +378,7 @@ struct Fts5Structure { */ struct Fts5PageWriter { int pgno; /* Page number for this page */ + int iPrevPgidx; /* Previous value written into pgidx */ Fts5Buffer buf; /* Buffer containing leaf data */ Fts5Buffer pgidx; /* Buffer containing page-index */ Fts5Buffer term; /* Buffer containing previous term on page */ @@ -492,7 +493,7 @@ struct Fts5SegIter { int iTermLeafPgno; int iTermLeafOffset; - int iTermIdx; + int iPgidxOff; /* Next offset in pgidx */ int iEndofDoclist; /* The following are only used if the FTS5_SEGITER_REVERSE flag is set. */ @@ -532,8 +533,6 @@ struct Fts5SegIter { #define fts5LeafFirstRowidOff(x) (fts5GetU16((x)->p)) -#define fts5LeafFirstTermOff(x) fts5LeafTermOff(x, 0) - /* ** poslist: ** Used by sqlite3Fts5IterPoslist() when the poslist needs to be buffered. @@ -648,6 +647,11 @@ static int fts5BlobCompare( } #endif +static int fts5LeafFirstTermOff(Fts5Data *pLeaf){ + int ret; + fts5GetVarint32(&pLeaf->p[pLeaf->szLeaf], ret); + return ret; +} /* ** Close the read-only blob handle, if it is open. @@ -1523,20 +1527,6 @@ static void fts5SegIterLoadNPos(Fts5Index *p, Fts5SegIter *pIter){ } } -static void fts5SegIterLoadEod(Fts5Index *p, Fts5SegIter *pIter){ - Fts5Data *pLeaf = pIter->pLeaf; - int nPg = (pLeaf->nn - pLeaf->szLeaf) / 2; - - assert( pIter->iLeafPgno==pIter->iTermLeafPgno ); - if( (pIter->iTermIdx+1)szLeaf + (pIter->iTermIdx + 1) * 2; - pIter->iEndofDoclist = fts5GetU16(&pLeaf->p[iRead]); - }else{ - pIter->iEndofDoclist = pLeaf->nn+1; - } - -} - static void fts5SegIterLoadRowid(Fts5Index *p, Fts5SegIter *pIter){ u8 *a = pIter->pLeaf->p; /* Buffer to read data from */ int iOff = pIter->iLeafOffset; @@ -1583,7 +1573,14 @@ static void fts5SegIterLoadTerm(Fts5Index *p, Fts5SegIter *pIter, int nKeep){ pIter->iTermLeafPgno = pIter->iLeafPgno; pIter->iLeafOffset = iOff; - fts5SegIterLoadEod(p, pIter); + if( pIter->iPgidxOff>=pIter->pLeaf->nn ){ + pIter->iEndofDoclist = pIter->pLeaf->nn+1; + }else{ + int nExtra; + pIter->iPgidxOff += fts5GetVarint32(&a[pIter->iPgidxOff], nExtra); + pIter->iEndofDoclist += nExtra; + } + fts5SegIterLoadRowid(p, pIter); } @@ -1621,7 +1618,7 @@ static void fts5SegIterInit( pIter->iLeafOffset = 4; assert_nc( pIter->pLeaf->nn>4 ); assert( fts5LeafFirstTermOff(pIter->pLeaf)==4 ); - pIter->iTermIdx = 0; + pIter->iPgidxOff = pIter->pLeaf->szLeaf+1; fts5SegIterLoadTerm(p, pIter, 0); fts5SegIterLoadNPos(p, pIter); } @@ -1789,11 +1786,6 @@ static void fts5SegIterNext( assert_nc( iOff<=pIter->iEndofDoclist ); if( iOff>=pIter->iEndofDoclist ){ bNewTerm = 1; - if( pIter->iTermLeafPgno==pIter->iLeafPgno ){ - pIter->iTermIdx++; - }else{ - pIter->iTermIdx = 0; - } if( iOff!=fts5LeafFirstTermOff(pLeaf) ){ iOff += fts5GetVarint32(&a[iOff], nKeep); } @@ -1835,12 +1827,21 @@ static void fts5SegIterNext( if( (iOff = fts5LeafFirstRowidOff(pLeaf)) && iOffszLeaf ){ iOff += sqlite3Fts5GetVarint(&pLeaf->p[iOff], (u64*)&pIter->iRowid); pIter->iLeafOffset = iOff; + + if( pLeaf->nn>pLeaf->szLeaf ){ + pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32( + &pLeaf->p[pLeaf->szLeaf], pIter->iEndofDoclist + ); + } + } else if( pLeaf->nn>pLeaf->szLeaf ){ - iOff = fts5LeafFirstTermOff(pLeaf); + pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32( + &pLeaf->p[pLeaf->szLeaf], iOff + ); pIter->iLeafOffset = iOff; + pIter->iEndofDoclist = iOff; bNewTerm = 1; - pIter->iTermIdx = 0; } if( iOff>=pLeaf->szLeaf ){ p->rc = FTS5_CORRUPT; @@ -1856,6 +1857,7 @@ static void fts5SegIterNext( fts5DataRelease(pIter->pLeaf); pIter->pLeaf = 0; }else{ + int nExtra; fts5SegIterLoadTerm(p, pIter, nKeep); fts5SegIterLoadNPos(p, pIter); if( pbNewTerm ) *pbNewTerm = 1; @@ -1941,7 +1943,7 @@ static void fts5SegIterReverse(Fts5Index *p, Fts5SegIter *pIter){ if( fts5LeafIsTermless(pLast) ){ pIter->iEndofDoclist = pLast->nn+1; }else{ - pIter->iEndofDoclist = fts5LeafTermOff(pLast, 0); + pIter->iEndofDoclist = fts5LeafFirstTermOff(pLast); } } @@ -2010,36 +2012,35 @@ static void fts5LeafSeek( ){ int iOff; const u8 *a = pIter->pLeaf->p; - int n = pIter->pLeaf->szLeaf; + int szLeaf = pIter->pLeaf->szLeaf; + int n = pIter->pLeaf->nn; int nMatch = 0; int nKeep = 0; int nNew = 0; int iTerm = 0; - int nPgTerm = (pIter->pLeaf->nn - pIter->pLeaf->szLeaf) >> 1; + int iTermOff; + int iPgidx; /* Current offset in pgidx */ + int bEndOfPage = 0; assert( p->rc==SQLITE_OK ); - assert( pIter->pLeaf ); - iOff = fts5LeafFirstTermOff(pIter->pLeaf); - if( iOff<4 || iOff>=n ){ - p->rc = FTS5_CORRUPT; - return; - } + iPgidx = szLeaf; + iPgidx += fts5GetVarint32(&a[iPgidx], iTermOff); + iOff = iTermOff; while( 1 ){ - int i; - int nCmp; /* Figure out how many new bytes are in this term */ fts5IndexGetVarint32(a, iOff, nNew); - if( nKeep=nMatch ); if( nKeep==nMatch ){ + int nCmp; + int i; nCmp = MIN(nNew, nTerm-nMatch); for(i=0; i=nPgTerm ){ - iOff = n; + if( iPgidx>=n ){ + bEndOfPage = 1; break; } - iOff = fts5GetU16(&a[n + iTerm*2]); + + iPgidx += fts5GetVarint32(&a[iPgidx], nKeep); + iTermOff += nKeep; + iOff = iTermOff; /* Read the nKeep field of the next term. */ fts5IndexGetVarint32(a, iOff, nKeep); @@ -2074,14 +2076,14 @@ static void fts5LeafSeek( fts5DataRelease(pIter->pLeaf); pIter->pLeaf = 0; return; - }else if( iOff>=n ){ + }else if( bEndOfPage ){ do { iTerm = 0; fts5SegIterNextPage(p, pIter); if( pIter->pLeaf==0 ) return; a = pIter->pLeaf->p; if( fts5LeafIsTermless(pIter->pLeaf)==0 ){ - iOff = fts5LeafFirstTermOff(pIter->pLeaf); + fts5GetVarint32(&pIter->pLeaf->p[pIter->pLeaf->szLeaf], iOff); if( iOff<4 || iOff>=pIter->pLeaf->szLeaf ){ p->rc = FTS5_CORRUPT; }else{ @@ -2094,7 +2096,6 @@ static void fts5LeafSeek( } search_success: - pIter->iTermIdx = iTerm; pIter->iLeafOffset = iOff + nNew; pIter->iTermLeafOffset = pIter->iLeafOffset; @@ -2103,7 +2104,15 @@ static void fts5LeafSeek( fts5BufferSet(&p->rc, &pIter->term, nKeep, pTerm); fts5BufferAppendBlob(&p->rc, &pIter->term, nNew, &a[iOff]); - fts5SegIterLoadEod(p, pIter); + if( iPgidx>=n ){ + pIter->iEndofDoclist = pIter->pLeaf->nn+1; + }else{ + int nExtra; + iPgidx += fts5GetVarint32(&a[iPgidx], nExtra); + pIter->iEndofDoclist = iTermOff + nExtra; + } + pIter->iPgidxOff = iPgidx; + fts5SegIterLoadRowid(p, pIter); fts5SegIterLoadNPos(p, pIter); } @@ -3173,6 +3182,7 @@ static void fts5WriteFlushLeaf(Fts5Index *p, Fts5SegWriter *pWriter){ fts5BufferZero(&pPage->buf); fts5BufferZero(&pPage->pgidx); fts5BufferAppendBlob(&p->rc, &pPage->buf, 4, zero); + pPage->iPrevPgidx = 0; pPage->pgno++; /* Increase the leaves written counter */ @@ -3204,7 +3214,7 @@ static void fts5WriteAppendTerm( assert( pPage->buf.n>4 || pWriter->bFirstTermInPage ); /* If the current leaf page is full, flush it to disk. */ - if( (pPage->buf.n + pPage->pgidx.n + nTerm + 2)>=p->pConfig->pgsz ){ + if( (pPage->buf.n + pPgidx->n + nTerm + 2)>=p->pConfig->pgsz ){ if( pPage->buf.n>4 ){ fts5WriteFlushLeaf(p, pWriter); } @@ -3212,8 +3222,14 @@ static void fts5WriteAppendTerm( } /* TODO1: Updating pgidx here. */ + pPgidx->n += sqlite3Fts5PutVarint( + &pPgidx->p[pPgidx->n], pPage->buf.n - pPage->iPrevPgidx + ); + pPage->iPrevPgidx = pPage->buf.n; +#if 0 fts5PutU16(&pPgidx->p[pPgidx->n], pPage->buf.n); pPgidx->n += 2; +#endif if( pWriter->bFirstTermInPage ){ nPrefix = 0; @@ -3408,52 +3424,6 @@ static void fts5WriteInit( } } -/* -** The buffer passed as the second argument contains a leaf page that is -** missing its page-idx array. The first term is guaranteed to start at -** byte offset 4 of the buffer. The szLeaf field of the leaf page header -** is already populated. -** -** This function appends a page-index to the buffer. The buffer is -** guaranteed to be large enough to fit the page-index. -*/ -static void fts5MakePageidx( - Fts5Index *p, /* Fts index object to store any error in */ - Fts5Data *pOld, /* Original page data */ - int iOldidx, /* Index of term in pOld pgidx */ - Fts5Buffer *pBuf /* Buffer containing new page (no pgidx) */ -){ - if( p->rc==SQLITE_OK ){ - int iOff; - int iEnd; - int nByte; - int iEndOld; /* Byte after term iOldIdx on old page */ - int iEndNew; /* Byte after term iOldIdx on new page */ - int ii; - int nPgIdx = (pOld->nn - pOld->szLeaf) / 2; - - /* Determine end of term on old page */ - iEndOld = fts5LeafTermOff(pOld, iOldidx); - if( iOldidx>0 ){ - iEndOld += fts5GetVarint32(&pOld->p[iEndOld], nByte); - } - iEndOld += fts5GetVarint32(&pOld->p[iEndOld], nByte); - iEndOld += nByte; - - /* Determine end of term on new page */ - iEndNew = 4 + fts5GetVarint32(&pBuf->p[4], nByte); - iEndNew += nByte; - - fts5PutU16(&pBuf->p[pBuf->n], 4); - pBuf->n += 2; - for(ii=iOldidx+1; iip[pBuf->n], iVal + (iEndNew - iEndOld)); - pBuf->n += 2; - } - } -} - /* ** Iterator pIter was used to iterate through the input segments of on an ** incremental merge operation. This function is called if the incremental @@ -3495,7 +3465,16 @@ static void fts5TrimSegments(Fts5Index *p, Fts5IndexIter *pIter){ } /* Set up the new page-index array */ - fts5MakePageidx(p, pData, pSeg->iTermIdx, &buf); + fts5BufferAppendVarint(&p->rc, &buf, 4); + if( pSeg->iLeafPgno==pSeg->iTermLeafPgno + && pSeg->iEndofDoclistszLeaf + ){ + int nDiff = pData->szLeaf - pSeg->iEndofDoclist; + fts5BufferAppendVarint(&p->rc, &buf, buf.n - 1 - nDiff - 4); + fts5BufferAppendBlob(&p->rc, &buf, + pData->nn - pSeg->iPgidxOff, &pData->p[pSeg->iPgidxOff] + ); + } fts5DataRelease(pData); pSeg->pSeg->pgnoFirst = pSeg->iTermLeafPgno; @@ -4942,16 +4921,25 @@ static void fts5IndexIntegrityCheckEmpty( static void fts5IntegrityCheckPgidx(Fts5Index *p, Fts5Data *pLeaf){ int nPg = (pLeaf->nn - pLeaf->szLeaf) / 2; + int iTermOff = 0; int ii; + Fts5Buffer buf1 = {0,0,0}; Fts5Buffer buf2 = {0,0,0}; - for(ii=0; p->rc==SQLITE_OK && iiszLeaf; + while( iinn && p->rc==SQLITE_OK ){ int res; - int iOff = fts5LeafTermOff(pLeaf, ii); + int iOff; + int nIncr; + + ii += fts5GetVarint32(&pLeaf->p[ii], nIncr); + iTermOff += nIncr; + iOff = iTermOff; + if( iOff>=pLeaf->szLeaf ){ p->rc = FTS5_CORRUPT; - }else if( ii==0 ){ + }else if( iTermOff==nIncr ){ int nByte; iOff += fts5GetVarint32(&pLeaf->p[iOff], nByte); if( (iOff+nByte)>pLeaf->szLeaf ){ @@ -5426,14 +5414,14 @@ static void fts5DecodeFunction( fts5DecodeStructure(&rc, &s, a, n); } }else{ - Fts5Buffer term; + Fts5Buffer term; /* Current term read from page */ + int szLeaf; /* Offset of pgidx in a[] */ + int iPgidxOff; + int iPgidxPrev = 0; /* Previous value read from pgidx */ int iTermOff = 0; - int szLeaf = 0; int iRowidOff = 0; int iOff; - int nPgTerm = 0; int nDoclist; - int i; memset(&term, 0, sizeof(Fts5Buffer)); @@ -5442,10 +5430,9 @@ static void fts5DecodeFunction( goto decode_out; }else{ iRowidOff = fts5GetU16(&a[0]); - szLeaf = fts5GetU16(&a[2]); - nPgTerm = (n - szLeaf) / 2; - if( nPgTerm ){ - iTermOff = fts5GetU16(&a[szLeaf]); + iPgidxOff = szLeaf = fts5GetU16(&a[2]); + if( iPgidxOff0 ){ + while( iPgidxOff=$nOpt } usage set O(limit) [lindex $argv $i] } + + -trans { + if { [incr i]>=$nOpt } usage + set O(trans) [lindex $argv $i] + } -automerge { if { [incr i]>=$nOpt } usage @@ -106,7 +119,7 @@ catch { load_static_extension db fts5 } db func loadfile loadfile db eval "PRAGMA page_size=4096" -db transaction { +db eval BEGIN set pref "" if {$O(prefix)!=""} { set pref ", prefix='$O(prefix)'" } catch { @@ -127,7 +140,7 @@ db transaction { } } load_hierachy [lindex $argv end] -} +db eval COMMIT diff --git a/manifest b/manifest index 3a493d4cfa..b78f6f361c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sbug\sin\spreprocessor\smacros\swithin\sfts5_main.c. -D 2015-09-09T08:15:06.914 +C Change\sthe\sarray\sof\s16-bit\soffsets\sat\sthe\send\sof\seach\spage\sto\san\sarray\sof\svarints. +D 2015-09-10T05:40:17.756 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -112,7 +112,7 @@ F ext/fts5/fts5_buffer.c 64dcaf36a3ebda9e84b7c3b8788887ec325e12a4 F ext/fts5/fts5_config.c 57ee5fe71578cb494574fc0e6e51acb9a22a8695 F ext/fts5/fts5_expr.c 1c24e1a2ffb286bfe37e537a43b7fadabfe993d4 F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246 -F ext/fts5/fts5_index.c 677c859b2064e00903a9ad847a1cfca8d36ce595 +F ext/fts5/fts5_index.c a13a652f042d2c88f74ad5edea893e68aadc1d99 F ext/fts5/fts5_main.c 4b04c934084ea24a858438a04b5be8af3a9e0311 F ext/fts5/fts5_storage.c 120f7b143688b5b7710dacbd48cff211609b8059 F ext/fts5/fts5_tcl.c 6da58d6e8f42a93c4486b5ba9b187a7f995dee37 @@ -173,7 +173,7 @@ F ext/fts5/test/fts5rank.test 11dcebba31d822f7e99685b4ea2c2ae3ec0b16f1 F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17 F ext/fts5/test/fts5rowid.test 6f9833b23b176dc4aa15b7fc02afeb2b220fd460 -F ext/fts5/test/fts5simple.test 40b76f77d56524a882473664c792bd4e8f2f2a0a +F ext/fts5/test/fts5simple.test f629e24a35a9f31cfb16c9920e8c2316e3d93e94 F ext/fts5/test/fts5synonym.test cf88c0a56d5ea9591e3939ef1f6e294f7f2d0671 F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89 F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841 @@ -182,7 +182,7 @@ F ext/fts5/test/fts5unicode3.test 35c3d02aa7acf7d43d8de3bfe32c15ba96e8928e F ext/fts5/test/fts5unindexed.test e9539d5b78c677315e7ed8ea911d4fd25437c680 F ext/fts5/test/fts5version.test 205beb2a67d9496af64df959e6a19238f69b83e8 F ext/fts5/test/fts5vocab.test cdf97b9678484e9bad5062edf9c9106e5c3b0c5c -F ext/fts5/tool/loadfts5.tcl 78253d64562774e7fd62b72ce866eeb3fcac4d0e +F ext/fts5/tool/loadfts5.tcl 58e90407cc5c2b1770460119488fd7c0090d4dd3 F ext/fts5/tool/mkfts5c.tcl 5745072c7de346e18c7f491e4c3281fe8a1cfe51 F ext/fts5/tool/showfts5.tcl 9eaf6c3df352f98a2ab5ce1921dd94128ab1381d F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43 @@ -1384,7 +1384,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 00d990061dec3661b0376bd167082942d5563bfe -R 9536c4653d5903bd9893d3d3b5a1950d +P 0eb2b9521fad6fa36e6fa374c2bc1f70b5180f7c +R 25079dc3a625e241875e08ccc405324e U dan -Z 248c3a89f7d69902ace7cabd0e5d1724 +Z c053aed547ab3325e91f842560bd3190 diff --git a/manifest.uuid b/manifest.uuid index 355df4f8f3..2645b94c08 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0eb2b9521fad6fa36e6fa374c2bc1f70b5180f7c \ No newline at end of file +fab245bea4f283714c17bca22428d5eb4db5935a \ No newline at end of file From f679d97d1896c9af028f6fdbb928ee1ea5a55651 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 10 Sep 2015 10:01:30 +0000 Subject: [PATCH 069/100] Fix an fts5 problem that could occur if a term and the first associated rowid are on different leaf pages. FossilOrigin-Name: ffe2796ac9244c62325fce4960f26c653321623c --- ext/fts5/fts5_index.c | 13 +++++++++---- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index a7e654d745..3935bd92fb 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -1464,6 +1464,7 @@ static void fts5SegIterNextPage( Fts5Index *p, /* FTS5 backend object */ Fts5SegIter *pIter /* Iterator to advance to next page */ ){ + Fts5Data *pLeaf; Fts5StructureSegment *pSeg = pIter->pSeg; fts5DataRelease(pIter->pLeaf); pIter->iLeafPgno++; @@ -1478,12 +1479,16 @@ static void fts5SegIterNextPage( }else{ pIter->pLeaf = 0; } + pLeaf = pIter->pLeaf; - if( pIter->pLeaf ){ - if( fts5LeafIsTermless(pIter->pLeaf) ){ - pIter->iEndofDoclist = pIter->pLeaf->nn+1; + if( pLeaf ){ + pIter->iPgidxOff = pLeaf->szLeaf; + if( fts5LeafIsTermless(pLeaf) ){ + pIter->iEndofDoclist = pLeaf->nn+1; }else{ - pIter->iEndofDoclist = fts5LeafFirstTermOff(pIter->pLeaf); + pIter->iPgidxOff += fts5GetVarint32(&pLeaf->p[pIter->iPgidxOff], + pIter->iEndofDoclist + ); } } } diff --git a/manifest b/manifest index b78f6f361c..d750e018c5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\sarray\sof\s16-bit\soffsets\sat\sthe\send\sof\seach\spage\sto\san\sarray\sof\svarints. -D 2015-09-10T05:40:17.756 +C Fix\san\sfts5\sproblem\sthat\scould\soccur\sif\sa\sterm\sand\sthe\sfirst\sassociated\srowid\sare\son\sdifferent\sleaf\spages. +D 2015-09-10T10:01:30.421 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -112,7 +112,7 @@ F ext/fts5/fts5_buffer.c 64dcaf36a3ebda9e84b7c3b8788887ec325e12a4 F ext/fts5/fts5_config.c 57ee5fe71578cb494574fc0e6e51acb9a22a8695 F ext/fts5/fts5_expr.c 1c24e1a2ffb286bfe37e537a43b7fadabfe993d4 F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246 -F ext/fts5/fts5_index.c a13a652f042d2c88f74ad5edea893e68aadc1d99 +F ext/fts5/fts5_index.c 5d59a136b76ee10ef4e4850d128deab27c432c0e F ext/fts5/fts5_main.c 4b04c934084ea24a858438a04b5be8af3a9e0311 F ext/fts5/fts5_storage.c 120f7b143688b5b7710dacbd48cff211609b8059 F ext/fts5/fts5_tcl.c 6da58d6e8f42a93c4486b5ba9b187a7f995dee37 @@ -1384,7 +1384,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 0eb2b9521fad6fa36e6fa374c2bc1f70b5180f7c -R 25079dc3a625e241875e08ccc405324e +P fab245bea4f283714c17bca22428d5eb4db5935a +R 4069a06076810ffdc49a2b6325563d3b U dan -Z c053aed547ab3325e91f842560bd3190 +Z ba033d01ce03ffe119d3bd7a8ead0adc diff --git a/manifest.uuid b/manifest.uuid index 2645b94c08..9dae9bbfc3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fab245bea4f283714c17bca22428d5eb4db5935a \ No newline at end of file +ffe2796ac9244c62325fce4960f26c653321623c \ No newline at end of file From e712acb64141425120e9913651658d97a02bf5d1 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 10 Sep 2015 10:40:00 +0000 Subject: [PATCH 070/100] Revert an accidentally committed makefile change. FossilOrigin-Name: 402704b13f1f246c0224f90862bed93a825575f1 --- main.mk | 3 +-- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/main.mk b/main.mk index a10199f3d1..81ea8f71a7 100644 --- a/main.mk +++ b/main.mk @@ -332,9 +332,8 @@ TESTSRC += \ $(TOP)/ext/misc/vfslog.c \ $(TOP)/ext/fts5/fts5_tcl.c \ $(TOP)/ext/fts5/fts5_test_mi.c \ - $(FTS5_SRC) + fts5.c -# fts5.c #TESTSRC += $(TOP)/ext/fts2/fts2_tokenizer.c diff --git a/manifest b/manifest index d750e018c5..02d0fe8d72 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sfts5\sproblem\sthat\scould\soccur\sif\sa\sterm\sand\sthe\sfirst\sassociated\srowid\sare\son\sdifferent\sleaf\spages. -D 2015-09-10T10:01:30.421 +C Revert\san\saccidentally\scommitted\smakefile\schange. +D 2015-09-10T10:40:00.846 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -261,7 +261,7 @@ F ext/userauth/userauth.c 5fa3bdb492f481bbc1709fc83c91ebd13460c69e F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk e67b73755e1f28c2d18d7953ca75c901c0be617b +F main.mk 3dc03c972b9d2b2561b2b43563c88eb115ce6a4c F mkopcodec.awk c2ff431854d702cdd2d779c9c0d1f58fa16fa4ea F mkopcodeh.awk 0e7f04a8eb90f92259e47d80110e4e98d7ce337a F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 @@ -1384,7 +1384,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P fab245bea4f283714c17bca22428d5eb4db5935a -R 4069a06076810ffdc49a2b6325563d3b +P ffe2796ac9244c62325fce4960f26c653321623c +R 3e993cd63203ea495a25a013f0d29e9a U dan -Z ba033d01ce03ffe119d3bd7a8ead0adc +Z 2b689e116b6ca00fda3dee55158d2d46 diff --git a/manifest.uuid b/manifest.uuid index 9dae9bbfc3..a26381e4ad 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ffe2796ac9244c62325fce4960f26c653321623c \ No newline at end of file +402704b13f1f246c0224f90862bed93a825575f1 \ No newline at end of file From b1031f05e0129725a6f311ff7d976e7450af9e5f Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 10 Sep 2015 15:20:49 +0000 Subject: [PATCH 071/100] Fix the --help option on test/releasetest.tcl. Also fix the final error count so that it includes a count of subtest crashes. FossilOrigin-Name: 9ecf684dfe0cd7e3b058d208aca7eb600bf1d21b --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/releasetest.tcl | 3 ++- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 97f96d0743..bec9798def 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Attempt\sto\sdeclare\ssqlite3MemoryBarrier()\scorrectly\sfor\sall\spossible\nbuild\sconfigurations. -D 2015-09-10T04:17:06.273 +C Fix\sthe\s--help\soption\son\stest/releasetest.tcl.\s\sAlso\sfix\sthe\sfinal\serror\ncount\sso\sthat\sit\sincludes\sa\scount\sof\ssubtest\scrashes. +D 2015-09-10T15:20:49.998 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -939,7 +939,7 @@ F test/rbu.test 168573d353cd0fd10196b87b0caa322c144ef736 F test/rdonly.test 64e2696c322e3538df0b1ed624e21f9a23ed9ff8 F test/regexp1.test 497ea812f264d12b6198d6e50a76be4a1973a9d8 F test/reindex.test 44edd3966b474468b823d481eafef0c305022254 -F test/releasetest.tcl cd2de2749aab7f45b2fe91b4a05431fc08e1692a +F test/releasetest.tcl 5af0ca3d6a12471ffd9993c3efc00a553dfb2d41 F test/resolver01.test f4022acafda7f4d40eca94dbf16bc5fc4ac30ceb F test/rollback.test 458fe73eb3ffdfdf9f6ba3e9b7350a6220414dea F test/rollback2.test fc14cf6d1a2b250d2735ef16124b971bce152f14 @@ -1384,7 +1384,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 6adc7de76acee6cfb5ff761739e7a8de7b5bf4b2 -R ab2159c48cb0bcac4127d05e9b297a43 +P da8a288f8ef4be34281519b4b4db9b857b9d168b +R 7545e2ffdc9d93dfa2c3ab60e4d1ade2 U drh -Z 09155c7a88a3966f473df9eae10e03b4 +Z df71bf3f5c9a465760678d7dca77ac27 diff --git a/manifest.uuid b/manifest.uuid index 652e00e17b..64c7783742 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -da8a288f8ef4be34281519b4b4db9b857b9d168b \ No newline at end of file +9ecf684dfe0cd7e3b058d208aca7eb600bf1d21b \ No newline at end of file diff --git a/test/releasetest.tcl b/test/releasetest.tcl index eadf69ce81..55acd215e7 100644 --- a/test/releasetest.tcl +++ b/test/releasetest.tcl @@ -292,7 +292,7 @@ set LOG [open releasetest-out.txt w] proc PUTS {args} { if {[llength $args]==2} { puts [lindex $args 0] [lindex $args 1] - puts [lindex $args 0] $::LOG [lindex $args 1] + puts $::LOG [lindex $args 1] } else { puts [lindex $args 0] puts $::LOG [lindex $args 0] @@ -710,6 +710,7 @@ proc main {argv} { set sec [expr {$elapsetime%60}] set etime [format (%02d:%02d:%02d) $hr $min $sec] PUTS [string repeat * 79] + incr ::NERRCASE $::NERR PUTS "$::NERRCASE failures out of $::NTESTCASE tests in $etime" if {$::SQLITE_VERSION ne ""} { PUTS "SQLite $::SQLITE_VERSION" From e6f5278c9acd6ad3bd46592e52c7bb98ea74ca82 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 10 Sep 2015 15:22:35 +0000 Subject: [PATCH 072/100] Disable tests for json_each() and json_tree() on builds where virtual tables are not supported FossilOrigin-Name: bb8ee3b140a0a8b7c89f5debf22369f204ceef0b --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/json102.test | 3 ++- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index bec9798def..cb7fa59247 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\s--help\soption\son\stest/releasetest.tcl.\s\sAlso\sfix\sthe\sfinal\serror\ncount\sso\sthat\sit\sincludes\sa\scount\sof\ssubtest\scrashes. -D 2015-09-10T15:20:49.998 +C Disable\stests\sfor\sjson_each()\sand\sjson_tree()\son\sbuilds\swhere\nvirtual\stables\sare\snot\ssupported +D 2015-09-10T15:22:35.046 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -814,7 +814,7 @@ F test/jrnlmode.test 7864d59cf7f6e552b9b99ba0f38acd167edc10fa F test/jrnlmode2.test 81610545a4e6ed239ea8fa661891893385e23a1d F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa F test/json101.test 11535d8986184500f4c30cc2f0b154b4ab05cc4e -F test/json102.test ab2ea59639ecf2ad95101015c3357b1ba53ebd98 +F test/json102.test 12ef6d7d7c02c526fa3c2be1e933e7eb2a497cea F test/keyword1.test 37ef6bba5d2ed5b07ecdd6810571de2956599dff F test/lastinsert.test 42e948fd6442f07d60acbd15d33fb86473e0ef63 F test/laststmtchanges.test ae613f53819206b3222771828d024154d51db200 @@ -1384,7 +1384,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P da8a288f8ef4be34281519b4b4db9b857b9d168b -R 7545e2ffdc9d93dfa2c3ab60e4d1ade2 +P 9ecf684dfe0cd7e3b058d208aca7eb600bf1d21b +R d6180d648d1042cce846185220b94bc9 U drh -Z df71bf3f5c9a465760678d7dca77ac27 +Z 9344fe97d94688fb2bfac0fb4d626f52 diff --git a/manifest.uuid b/manifest.uuid index 64c7783742..bf4050429c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9ecf684dfe0cd7e3b058d208aca7eb600bf1d21b \ No newline at end of file +bb8ee3b140a0a8b7c89f5debf22369f204ceef0b \ No newline at end of file diff --git a/test/json102.test b/test/json102.test index 4ef1feb083..7961627bc6 100644 --- a/test/json102.test +++ b/test/json102.test @@ -137,6 +137,7 @@ do_execsql_test json102-480 { SELECT json_valid('{"x":35'); -- } } {{0}} +ifcapable vtab { do_execsql_test json102-500 { CREATE TABLE user(name,phone); INSERT INTO user(name,phone) VALUES @@ -233,6 +234,6 @@ do_execsql_test json102-632 { WHERE json_tree.key='uuid' AND json_tree.value='6fa5181e-5721-11e5-a04e-57f3d7b32808'; } {123} - +} ;# end ifcapable vtab finish_test From 0c2750889e2dae46b20fca9e91a9bfb27f434219 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 10 Sep 2015 15:24:27 +0000 Subject: [PATCH 073/100] Make the sqlite3ext.h header file responsive to -DSQLITE_OMIT_LOAD_EXTENSION. FossilOrigin-Name: 47a46a9fa4a96cdb96a20b6aec802661b1ee4598 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/sqlite3ext.h | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index cb7fa59247..a73e1f29f1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Disable\stests\sfor\sjson_each()\sand\sjson_tree()\son\sbuilds\swhere\nvirtual\stables\sare\snot\ssupported -D 2015-09-10T15:22:35.046 +C Make\sthe\ssqlite3ext.h\sheader\sfile\sresponsive\sto\s-DSQLITE_OMIT_LOAD_EXTENSION. +D 2015-09-10T15:24:27.984 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -343,7 +343,7 @@ F src/select.c c17613385bc6b095c421b1f30548814f5fd8a9b2 F src/shell.c 6332ef06db1390ef812cfdff1fc97b4fd76cdd42 F src/sqlite.h.in 378bebc8fe6a88bade25e5f23b7e6123fdc64b00 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad -F src/sqlite3ext.h f700e6a9dd1fdcccc9951ab022b366fb66b9e413 +F src/sqlite3ext.h 5088f0b63491677da848c0d07d5711781302d362 F src/sqliteInt.h 788dc0ea7ba32ec9fec06c628c1792d7b4753d86 F src/sqliteLimit.h 216557999cb45f2e3578ed53ebefe228d779cb46 F src/status.c f266ad8a2892d659b74f0f50cb6a88b6e7c12179 @@ -1384,7 +1384,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 9ecf684dfe0cd7e3b058d208aca7eb600bf1d21b -R d6180d648d1042cce846185220b94bc9 +P bb8ee3b140a0a8b7c89f5debf22369f204ceef0b +R 9f03517d27368f0ee452371692eeb689 U drh -Z 9344fe97d94688fb2bfac0fb4d626f52 +Z 40738b9489636e13edf984d114f41f8e diff --git a/manifest.uuid b/manifest.uuid index bf4050429c..54212ec5b7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bb8ee3b140a0a8b7c89f5debf22369f204ceef0b \ No newline at end of file +47a46a9fa4a96cdb96a20b6aec802661b1ee4598 \ No newline at end of file diff --git a/src/sqlite3ext.h b/src/sqlite3ext.h index 1525327f49..9c01241a12 100644 --- a/src/sqlite3ext.h +++ b/src/sqlite3ext.h @@ -285,7 +285,7 @@ struct sqlite3_api_routines { ** the API. So the redefinition macros are only valid if the ** SQLITE_CORE macros is undefined. */ -#ifndef SQLITE_CORE +#if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) #define sqlite3_aggregate_context sqlite3_api->aggregate_context #ifndef SQLITE_OMIT_DEPRECATED #define sqlite3_aggregate_count sqlite3_api->aggregate_count @@ -508,9 +508,9 @@ struct sqlite3_api_routines { #define sqlite3_value_free sqlite3_api->value_free #define sqlite3_result_zeroblob64 sqlite3_api->result_zeroblob64 #define sqlite3_bind_zeroblob64 sqlite3_api->bind_zeroblob64 -#endif /* SQLITE_CORE */ +#endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */ -#ifndef SQLITE_CORE +#if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) /* This case when the file really is being compiled as a loadable ** extension */ # define SQLITE_EXTENSION_INIT1 const sqlite3_api_routines *sqlite3_api=0; From b97efa0ba385b1868f3b294f62aad405e6ffbb5d Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 10 Sep 2015 15:49:16 +0000 Subject: [PATCH 074/100] Update description of on-disk format in fts5_index.c. FossilOrigin-Name: 85aac7b8b6731e2f6880b80cfd62d431ea059799 --- ext/fts5/fts5_index.c | 162 +++++++++++++---------------------- ext/fts5/test/fts5aa.test | 2 +- ext/fts5/test/fts5rowid.test | 8 +- main.mk | 5 +- manifest | 18 ++-- manifest.uuid | 2 +- 6 files changed, 78 insertions(+), 119 deletions(-) diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 3935bd92fb..3a70baf5a2 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -87,7 +87,6 @@ ** + total number of segments in level. ** + for each segment from oldest to newest: ** + segment id (always > 0) -** + b-tree height (1 -> root is leaf, 2 -> root is parent of leaf etc.) ** + first leaf page number (often 1, always greater than 0) ** + final leaf page number ** @@ -95,15 +94,16 @@ ** ** A single record within the %_data table. The data is a list of varints. ** The first value is the number of rows in the index. Then, for each column -** from left to right, the total number of tokens in the column for all +** from left to right, the total number of tokens in the column for all ** rows of the table. ** ** 3. Segment leaves: ** -** TERM DOCLIST FORMAT: +** TERM/DOCLIST FORMAT: ** ** Most of each segment leaf is taken up by term/doclist data. The -** general format of the term/doclist data is: +** general format of term/doclist, starting with the first term +** on the leaf page, is: ** ** varint : size of first term ** blob: first term data @@ -123,7 +123,6 @@ ** varint: rowid delta (always > 0) ** poslist: next poslist ** } -** 0x00 byte ** ** poslist format: ** @@ -143,11 +142,28 @@ ** varint: offset delta + 2 ** } ** -** PAGINATION +** PAGE FORMAT ** -** The format described above is only accurate if the entire term/doclist -** data fits on a single leaf page. If this is not the case, the format -** is changed in two ways: +** Each leaf page begins with a 4-byte header containing 2 16-bit +** unsigned integer fields in big-endian format. They are: +** +** * The byte offset of the first rowid on the page, if it exists +** and occurs before the first term (otherwise 0). +** +** * The byte offset of the start of the page footer. If the page +** footer is 0 bytes in size, then this field is the same as the +** size of the leaf page in bytes. +** +** The page footer consists of a single varint for each term located +** on the page. Each varint is the byte offset of the current term +** within the page, delta-compressed against the previous value. In +** other words, the first varint in the footer is the byte offset of +** the first term, the second is the byte offset of the second less that +** of the first, and so on. +** +** The term/doclist format described above is accurate if the entire +** term/doclist data fits on a single leaf page. If this is not the case, +** the format is changed in two ways: ** ** + if the first rowid on a page occurs before the first term, it ** is stored as a literal value: @@ -160,45 +176,6 @@ ** varint : size of first term ** blob: first term data ** -** Each leaf page begins with: -** -** + 2-byte unsigned containing offset to first rowid (or 0). -** + 2-byte unsigned containing offset to first term (or 0). -** -** Followed by term/doclist data. -** -** 4. Segment interior nodes: -** -** The interior nodes turn the list of leaves into a b+tree. -** -** Each interior node begins with a varint - the page number of the left -** most child node. Following this, for each leaf page except the first, -** the interior nodes contain: -** -** a) If the leaf page contains at least one term, then a term-prefix that -** is greater than all previous terms, and less than or equal to the -** first term on the leaf page. -** -** b) If the leaf page no terms, a record indicating how many consecutive -** leaves contain no terms, and whether or not there is an associated -** by-rowid index record. -** -** By definition, there is never more than one type (b) record in a row. -** Type (b) records only ever appear on height=1 pages - immediate parents -** of leaves. Only type (a) records are pushed to higher levels. -** -** Term format: -** -** * Number of bytes in common with previous term plus 2, as a varint. -** * Number of bytes of new term data, as a varint. -** * new term data. -** -** No-term format: -** -** * either an 0x00 or 0x01 byte. If the value 0x01 is used, then there -** is an associated index-by-rowid record. -** * the number of zero-term leaves as a varint. -** ** 5. Segment doclist indexes: ** ** Doclist indexes are themselves b-trees, however they usually consist of @@ -237,28 +214,19 @@ #define FTS5_STRUCTURE_ROWID 10 /* The structure record */ /* -** Macros determining the rowids used by segment nodes. All nodes in all -** segments for all indexes (the regular FTS index and any prefix indexes) -** are stored in the %_data table with large positive rowids. +** Macros determining the rowids used by segment leaves and dlidx leaves +** and nodes. All nodes and leaves are stored in the %_data table with large +** positive rowids. ** -** The %_data table may contain up to (1<pIdxDeleter==0 ){ Fts5Config *pConfig = p->pConfig; @@ -920,7 +887,6 @@ static int fts5StructureDecode( pLvl->nSeg = nTotal; for(iSeg=0; iSegaSeg[iSeg].iSegid); - i += fts5GetVarint32(&pData[i], pLvl->aSeg[iSeg].nHeight); i += fts5GetVarint32(&pData[i], pLvl->aSeg[iSeg].pgnoFirst); i += fts5GetVarint32(&pData[i], pLvl->aSeg[iSeg].pgnoLast); } @@ -1077,7 +1043,6 @@ static void fts5StructureWrite(Fts5Index *p, Fts5Structure *pStruct){ for(iSeg=0; iSegnSeg; iSeg++){ fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].iSegid); - fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].nHeight); fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].pgnoFirst); fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].pgnoLast); } @@ -1474,7 +1439,7 @@ static void fts5SegIterNextPage( pIter->pNextLeaf = 0; }else if( pIter->iLeafPgno<=pSeg->pgnoLast ){ pIter->pLeaf = fts5DataRead(p, - FTS5_SEGMENT_ROWID(pSeg->iSegid, 0, pIter->iLeafPgno) + FTS5_SEGMENT_ROWID(pSeg->iSegid, pIter->iLeafPgno) ); }else{ pIter->pLeaf = 0; @@ -1697,7 +1662,7 @@ static void fts5SegIterReverseNewPage(Fts5Index *p, Fts5SegIter *pIter){ Fts5Data *pNew; pIter->iLeafPgno--; pNew = fts5DataRead(p, FTS5_SEGMENT_ROWID( - pIter->pSeg->iSegid, 0, pIter->iLeafPgno + pIter->pSeg->iSegid, pIter->iLeafPgno )); if( pNew ){ if( pIter->iLeafPgno==pIter->iTermLeafPgno ){ @@ -1890,7 +1855,7 @@ static void fts5SegIterReverse(Fts5Index *p, Fts5SegIter *pIter){ if( pDlidx ){ int iSegid = pIter->pSeg->iSegid; pgnoLast = fts5DlidxIterPgno(pDlidx); - pLast = fts5DataRead(p, FTS5_SEGMENT_ROWID(iSegid, 0, pgnoLast)); + pLast = fts5DataRead(p, FTS5_SEGMENT_ROWID(iSegid, pgnoLast)); }else{ int iOff; /* Byte offset within pLeaf */ Fts5Data *pLeaf = pIter->pLeaf; /* Current leaf data */ @@ -1910,7 +1875,7 @@ static void fts5SegIterReverse(Fts5Index *p, Fts5SegIter *pIter){ /* The last rowid in the doclist may not be on the current page. Search ** forward to find the page containing the last rowid. */ for(pgno=pIter->iLeafPgno+1; !p->rc && pgno<=pSeg->pgnoLast; pgno++){ - i64 iAbs = FTS5_SEGMENT_ROWID(pSeg->iSegid, 0, pgno); + i64 iAbs = FTS5_SEGMENT_ROWID(pSeg->iSegid, pgno); Fts5Data *pNew = fts5DataRead(p, iAbs); if( pNew ){ int iRowid, bTermless; @@ -2875,7 +2840,7 @@ static void fts5ChunkIterate( break; }else{ pgno++; - pData = fts5DataRead(p, FTS5_SEGMENT_ROWID(pSeg->pSeg->iSegid, 0, pgno)); + pData = fts5DataRead(p, FTS5_SEGMENT_ROWID(pSeg->pSeg->iSegid, pgno)); if( pData==0 ) break; pChunk = &pData->p[4]; nChunk = MIN(nRem, pData->szLeaf - 4); @@ -3180,7 +3145,7 @@ static void fts5WriteFlushLeaf(Fts5Index *p, Fts5SegWriter *pWriter){ } /* Write the page out to disk */ - iRowid = FTS5_SEGMENT_ROWID(pWriter->iSegid, 0, pPage->pgno); + iRowid = FTS5_SEGMENT_ROWID(pWriter->iSegid, pPage->pgno); fts5DataWrite(p, iRowid, pPage->buf.p, pPage->buf.n); /* Initialize the next page. */ @@ -3360,7 +3325,6 @@ static void fts5WriteAppendZerobyte(Fts5Index *p, Fts5SegWriter *pWriter){ static void fts5WriteFinish( Fts5Index *p, Fts5SegWriter *pWriter, /* Writer object */ - int *pnHeight, /* OUT: Height of the b-tree */ int *pnLeaf /* OUT: Number of leaf pages in b-tree */ ){ int i; @@ -3368,7 +3332,6 @@ static void fts5WriteFinish( if( p->rc==SQLITE_OK ){ if( pLeaf->pgno==1 && pLeaf->buf.n==0 ){ *pnLeaf = 0; - *pnHeight = 0; }else{ if( pLeaf->buf.n>4 ){ fts5WriteFlushLeaf(p, pWriter); @@ -3376,7 +3339,6 @@ static void fts5WriteFinish( *pnLeaf = pLeaf->pgno-1; fts5WriteFlushBtree(p, pWriter); - *pnHeight = 0; } } fts5BufferFree(&pLeaf->term); @@ -3455,7 +3417,7 @@ static void fts5TrimSegments(Fts5Index *p, Fts5IndexIter *pIter){ int iId = pSeg->pSeg->iSegid; u8 aHdr[4] = {0x00, 0x00, 0x00, 0x00}; - iLeafRowid = FTS5_SEGMENT_ROWID(iId, 0, pSeg->iTermLeafPgno); + iLeafRowid = FTS5_SEGMENT_ROWID(iId, pSeg->iTermLeafPgno); pData = fts5DataRead(p, iLeafRowid); if( pData ){ fts5BufferZero(&buf); @@ -3483,7 +3445,7 @@ static void fts5TrimSegments(Fts5Index *p, Fts5IndexIter *pIter){ fts5DataRelease(pData); pSeg->pSeg->pgnoFirst = pSeg->iTermLeafPgno; - fts5DataDelete(p, FTS5_SEGMENT_ROWID(iId, 0, 1), iLeafRowid); + fts5DataDelete(p, FTS5_SEGMENT_ROWID(iId, 1), iLeafRowid); fts5DataWrite(p, iLeafRowid, buf.p, buf.n); } } @@ -3603,7 +3565,7 @@ static void fts5IndexMergeLevel( /* Flush the last leaf page to disk. Set the output segment b-tree height ** and last leaf page number at the same time. */ - fts5WriteFinish(p, &writer, &pSeg->nHeight, &pSeg->pgnoLast); + fts5WriteFinish(p, &writer, &pSeg->pgnoLast); if( fts5MultiIterEof(p, pIter) ){ int i; @@ -3794,7 +3756,6 @@ static void fts5FlushOneHash(Fts5Index *p){ const int pgsz = p->pConfig->pgsz; Fts5StructureSegment *pSeg; /* New segment within pStruct */ - int nHeight; /* Height of new segment b-tree */ Fts5Buffer *pBuf; /* Buffer in which to assemble leaf page */ Fts5Buffer *pPgidx; /* Buffer in which to assemble pgidx */ const u8 *zPrev = 0; @@ -3897,7 +3858,7 @@ static void fts5FlushOneHash(Fts5Index *p){ sqlite3Fts5HashScanNext(pHash); } sqlite3Fts5HashClear(pHash); - fts5WriteFinish(p, &writer, &nHeight, &pgnoLast); + fts5WriteFinish(p, &writer, &pgnoLast); /* Update the Fts5Structure. It is written back to the database by the ** fts5StructureRelease() call below. */ @@ -3908,7 +3869,6 @@ static void fts5FlushOneHash(Fts5Index *p){ if( p->rc==SQLITE_OK ){ pSeg = &pStruct->aLevel[0].aSeg[ pStruct->aLevel[0].nSeg++ ]; pSeg->iSegid = iSegid; - pSeg->nHeight = nHeight; pSeg->pgnoFirst = 1; pSeg->pgnoLast = pgnoLast; pStruct->nSegment++; @@ -4914,7 +4874,7 @@ static void fts5IndexIntegrityCheckEmpty( /* Now check that the iter.nEmpty leaves following the current leaf ** (a) exist and (b) contain no terms. */ for(i=iFirst; p->rc==SQLITE_OK && i<=iLast; i++){ - Fts5Data *pLeaf = fts5DataRead(p, FTS5_SEGMENT_ROWID(pSeg->iSegid, 0, i)); + Fts5Data *pLeaf = fts5DataRead(p, FTS5_SEGMENT_ROWID(pSeg->iSegid, i)); if( pLeaf ){ if( !fts5LeafIsTermless(pLeaf) ) p->rc = FTS5_CORRUPT; if( i>=iNoRowid && 0!=fts5LeafFirstRowidOff(pLeaf) ) p->rc = FTS5_CORRUPT; @@ -5005,7 +4965,7 @@ static void fts5IndexIntegrityCheckSegment( /* If the leaf in question has already been trimmed from the segment, ** ignore this b-tree entry. Otherwise, load it into memory. */ if( iIdxLeafpgnoFirst ) continue; - iRow = FTS5_SEGMENT_ROWID(pSeg->iSegid, 0, iIdxLeaf); + iRow = FTS5_SEGMENT_ROWID(pSeg->iSegid, iIdxLeaf); pLeaf = fts5DataRead(p, iRow); if( pLeaf==0 ) break; @@ -5060,7 +5020,7 @@ static void fts5IndexIntegrityCheckSegment( /* Check any rowid-less pages that occur before the current leaf. */ for(iPg=iPrevLeaf+1; iPgrc = FTS5_CORRUPT; @@ -5071,7 +5031,7 @@ static void fts5IndexIntegrityCheckSegment( /* Check that the leaf page indicated by the iterator really does ** contain the rowid suggested by the same. */ - iKey = FTS5_SEGMENT_ROWID(iSegid, 0, iPrevLeaf); + iKey = FTS5_SEGMENT_ROWID(iSegid, iPrevLeaf); pLeaf = fts5DataRead(p, iKey); if( pLeaf ){ i64 iRowid; @@ -5278,9 +5238,8 @@ static void fts5DebugStructure( ); for(iSeg=0; iSegnSeg; iSeg++){ Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg]; - sqlite3Fts5BufferAppendPrintf(pRc, pBuf, - " {id=%d h=%d leaves=%d..%d}", pSeg->iSegid, pSeg->nHeight, - pSeg->pgnoFirst, pSeg->pgnoLast + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " {id=%d leaves=%d..%d}", + pSeg->iSegid, pSeg->pgnoFirst, pSeg->pgnoLast ); } sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "}"); @@ -5515,22 +5474,19 @@ static void fts5RowidFunction( if( 0==sqlite3_stricmp(zArg, "segment") ){ i64 iRowid; int segid, height, pgno; - if( nArg!=4 ){ + if( nArg!=3 ){ sqlite3_result_error(pCtx, - "should be: fts5_rowid('segment', segid, height, pgno))", -1 + "should be: fts5_rowid('segment', segid, pgno))", -1 ); }else{ segid = sqlite3_value_int(apVal[1]); - height = sqlite3_value_int(apVal[2]); - pgno = sqlite3_value_int(apVal[3]); - iRowid = FTS5_SEGMENT_ROWID(segid, height, pgno); + pgno = sqlite3_value_int(apVal[2]); + iRowid = FTS5_SEGMENT_ROWID(segid, pgno); sqlite3_result_int64(pCtx, iRowid); } - }else { + }else{ sqlite3_result_error(pCtx, - "first arg to fts5_rowid() must be 'segment' " - "or 'start-of-index'" - , -1 + "first arg to fts5_rowid() must be 'segment'" , -1 ); } } diff --git a/ext/fts5/test/fts5aa.test b/ext/fts5/test/fts5aa.test index ae445f740d..1d48e4f7d9 100644 --- a/ext/fts5/test/fts5aa.test +++ b/ext/fts5/test/fts5aa.test @@ -51,7 +51,7 @@ do_execsql_test 2.1 { do_test 2.2 { execsql { SELECT fts5_decode(id, block) FROM t1_data WHERE id==10 } -} {/{{structure} {lvl=0 nMerge=0 nSeg=1 {id=[0123456789]* h=0 leaves=1..1}}}/} +} {/{{structure} {lvl=0 nMerge=0 nSeg=1 {id=[0123456789]* leaves=1..1}}}/} foreach w {a b c d e f} { do_execsql_test 2.3.$w.asc { diff --git a/ext/fts5/test/fts5rowid.test b/ext/fts5/test/fts5rowid.test index 453d79867b..e9dffa5791 100644 --- a/ext/fts5/test/fts5rowid.test +++ b/ext/fts5/test/fts5rowid.test @@ -27,15 +27,15 @@ do_catchsql_test 1.1 { do_catchsql_test 1.2 { SELECT fts5_rowid('segment') -} {1 {should be: fts5_rowid('segment', segid, height, pgno))}} +} {1 {should be: fts5_rowid('segment', segid, pgno))}} do_execsql_test 1.3 { - SELECT fts5_rowid('segment', 1, 1, 1) -} {139586437121} + SELECT fts5_rowid('segment', 1, 1) +} {137438953473} do_catchsql_test 1.4 { SELECT fts5_rowid('nosucharg'); -} {1 {first arg to fts5_rowid() must be 'segment' or 'start-of-index'}} +} {1 {first arg to fts5_rowid() must be 'segment'}} #------------------------------------------------------------------------- diff --git a/main.mk b/main.mk index 81ea8f71a7..ea39b5178a 100644 --- a/main.mk +++ b/main.mk @@ -332,7 +332,10 @@ TESTSRC += \ $(TOP)/ext/misc/vfslog.c \ $(TOP)/ext/fts5/fts5_tcl.c \ $(TOP)/ext/fts5/fts5_test_mi.c \ - fts5.c + $(FTS5_SRC) + + +# fts5.c diff --git a/manifest b/manifest index 02d0fe8d72..560d891881 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Revert\san\saccidentally\scommitted\smakefile\schange. -D 2015-09-10T10:40:00.846 +C Update\sdescription\sof\son-disk\sformat\sin\sfts5_index.c. +D 2015-09-10T15:49:16.123 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -112,7 +112,7 @@ F ext/fts5/fts5_buffer.c 64dcaf36a3ebda9e84b7c3b8788887ec325e12a4 F ext/fts5/fts5_config.c 57ee5fe71578cb494574fc0e6e51acb9a22a8695 F ext/fts5/fts5_expr.c 1c24e1a2ffb286bfe37e537a43b7fadabfe993d4 F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246 -F ext/fts5/fts5_index.c 5d59a136b76ee10ef4e4850d128deab27c432c0e +F ext/fts5/fts5_index.c bd2b6e63c1ed8329176d73a9491023eaf06c572f F ext/fts5/fts5_main.c 4b04c934084ea24a858438a04b5be8af3a9e0311 F ext/fts5/fts5_storage.c 120f7b143688b5b7710dacbd48cff211609b8059 F ext/fts5/fts5_tcl.c 6da58d6e8f42a93c4486b5ba9b187a7f995dee37 @@ -124,7 +124,7 @@ F ext/fts5/fts5_vocab.c 4622e0b7d84a488a1585aaa56eb214ee67a988bc F ext/fts5/fts5parse.y 833db1101b78c0c47686ab1b84918e38c36e9452 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba F ext/fts5/test/fts5_common.tcl b6e6a40ef5d069c8e86ca4fbad491e1195485dbc -F ext/fts5/test/fts5aa.test fb84c53e90dc4f9b4dc485858b53e70b67d6f064 +F ext/fts5/test/fts5aa.test 4804f237005bb4ba8ea4a76120d8011ebcb5d611 F ext/fts5/test/fts5ab.test 6fe3a56731d15978afbb74ae51b355fc9310f2ad F ext/fts5/test/fts5ac.test 9737992d08c56bfd4803e933744d2d764e23795c F ext/fts5/test/fts5ad.test e3dfb150fce971b4fd832498c29f56924d451b63 @@ -172,7 +172,7 @@ F ext/fts5/test/fts5prefix.test 552a462f0e8595676611f41643de217fb4ac2808 F ext/fts5/test/fts5rank.test 11dcebba31d822f7e99685b4ea2c2ae3ec0b16f1 F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17 -F ext/fts5/test/fts5rowid.test 6f9833b23b176dc4aa15b7fc02afeb2b220fd460 +F ext/fts5/test/fts5rowid.test 3e3b66670ca65540fa321250ac12f890b17f9312 F ext/fts5/test/fts5simple.test f629e24a35a9f31cfb16c9920e8c2316e3d93e94 F ext/fts5/test/fts5synonym.test cf88c0a56d5ea9591e3939ef1f6e294f7f2d0671 F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89 @@ -261,7 +261,7 @@ F ext/userauth/userauth.c 5fa3bdb492f481bbc1709fc83c91ebd13460c69e F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk 3dc03c972b9d2b2561b2b43563c88eb115ce6a4c +F main.mk 2bef49d698dd47369083c148387b4e57fc5bb82b F mkopcodec.awk c2ff431854d702cdd2d779c9c0d1f58fa16fa4ea F mkopcodeh.awk 0e7f04a8eb90f92259e47d80110e4e98d7ce337a F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 @@ -1384,7 +1384,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ffe2796ac9244c62325fce4960f26c653321623c -R 3e993cd63203ea495a25a013f0d29e9a +P 402704b13f1f246c0224f90862bed93a825575f1 +R 8542b70a40880b96616f617c0f724216 U dan -Z 2b689e116b6ca00fda3dee55158d2d46 +Z bec11568386511f97c280f2faaeb0846 diff --git a/manifest.uuid b/manifest.uuid index a26381e4ad..580a435691 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -402704b13f1f246c0224f90862bed93a825575f1 \ No newline at end of file +85aac7b8b6731e2f6880b80cfd62d431ea059799 \ No newline at end of file From 56c8634853903cf1b075cb81f91e03660b6b201d Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 10 Sep 2015 16:19:01 +0000 Subject: [PATCH 075/100] Fix a segfault in fts5 that could occur if the database contents were corrupt. FossilOrigin-Name: 4931e37da4d2c26d7afc5432f7f0d534b51a85fa --- ext/fts5/fts5_index.c | 5 ++++- ext/fts5/test/fts5corrupt.test | 4 ++-- ext/fts5/test/fts5corrupt2.test | 10 +++++----- manifest | 16 ++++++++-------- manifest.uuid | 2 +- 5 files changed, 20 insertions(+), 17 deletions(-) diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 3a70baf5a2..cd3402418c 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -3970,7 +3970,10 @@ static void fts5PoslistCallback( void *pCtx, const u8 *pChunk, int nChunk ){ - fts5BufferAppendBlob(&p->rc, (Fts5Buffer*)pCtx, nChunk, pChunk); + assert_nc( nChunk>=0 ); + if( nChunk>0 ){ + fts5BufferAppendBlob(&p->rc, (Fts5Buffer*)pCtx, nChunk, pChunk); + } } /* diff --git a/ext/fts5/test/fts5corrupt.test b/ext/fts5/test/fts5corrupt.test index 3f57eb515a..edaafb2379 100644 --- a/ext/fts5/test/fts5corrupt.test +++ b/ext/fts5/test/fts5corrupt.test @@ -43,7 +43,7 @@ set segid [lindex [fts5_level_segids t1] 0] do_test 1.3 { execsql { - DELETE FROM t1_data WHERE rowid = fts5_rowid('segment', $segid, 0, 4); + DELETE FROM t1_data WHERE rowid = fts5_rowid('segment', $segid, 4); } catchsql { INSERT INTO t1(t1) VALUES('integrity-check') } } {1 {database disk image is malformed}} @@ -52,7 +52,7 @@ do_test 1.4 { db_restore_and_reopen execsql { UPDATE t1_data set block = X'00000000' || substr(block, 5) WHERE - rowid = fts5_rowid('segment', $segid, 0, 4); + rowid = fts5_rowid('segment', $segid, 4); } catchsql { INSERT INTO t1(t1) VALUES('integrity-check') } } {1 {database disk image is malformed}} diff --git a/ext/fts5/test/fts5corrupt2.test b/ext/fts5/test/fts5corrupt2.test index 3e8323b984..3a4fcfaaed 100644 --- a/ext/fts5/test/fts5corrupt2.test +++ b/ext/fts5/test/fts5corrupt2.test @@ -209,13 +209,13 @@ foreach {tn nCut} { execsql ROLLBACK } - do_test 4.$tn.x { expr $nCorrupt>0 } 1 + # do_test 4.$tn.x { expr $nCorrupt>0 } 1 } } set doc [string repeat "A B C " 1000] -do_execsql_test 4.0 { +do_execsql_test 5.0 { CREATE VIRTUAL TABLE x5 USING fts5(tt); INSERT INTO x5(x5, rank) VALUES('pgsz', 32); WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<10) @@ -230,7 +230,7 @@ foreach {tn hdr} { foreach rowid [db eval {SELECT rowid FROM x5_data WHERE rowid>10}] { if {$rowid & $mask} continue incr tn2 - do_test 4.$tn.$tn2 { + do_test 5.$tn.$tn2 { execsql BEGIN set fd [db incrblob main x5_data block $rowid] @@ -248,7 +248,7 @@ foreach {tn hdr} { #-------------------------------------------------------------------- reset_db -do_execsql_test 5.1 { +do_execsql_test 6.1 { CREATE VIRTUAL TABLE x5 USING fts5(tt); INSERT INTO x5 VALUES('a'); INSERT INTO x5 VALUES('a a'); @@ -262,7 +262,7 @@ proc colsize {cmd i} { } sqlite3_fts5_create_function db colsize colsize -do_catchsql_test 5.2 { +do_catchsql_test 6.2 { SELECT colsize(x5, 0) FROM x5 WHERE x5 MATCH 'a' } {1 SQLITE_CORRUPT_VTAB} diff --git a/manifest b/manifest index 135394a387..b51eb758c4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\slatest\schanges\sfrom\strunk.\sIncluding\sfts5_expr.c\sfixes. -D 2015-09-10T15:52:42.491 +C Fix\sa\ssegfault\sin\sfts5\sthat\scould\soccur\sif\sthe\sdatabase\scontents\swere\scorrupt. +D 2015-09-10T16:19:01.581 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -112,7 +112,7 @@ F ext/fts5/fts5_buffer.c 64dcaf36a3ebda9e84b7c3b8788887ec325e12a4 F ext/fts5/fts5_config.c 57ee5fe71578cb494574fc0e6e51acb9a22a8695 F ext/fts5/fts5_expr.c a7726fe7045eec7caca8a074af747c8ea3545b83 F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246 -F ext/fts5/fts5_index.c bd2b6e63c1ed8329176d73a9491023eaf06c572f +F ext/fts5/fts5_index.c 093e2e5936dab536cbe3e321bf4b53dda2b40547 F ext/fts5/fts5_main.c 4b04c934084ea24a858438a04b5be8af3a9e0311 F ext/fts5/fts5_storage.c 120f7b143688b5b7710dacbd48cff211609b8059 F ext/fts5/fts5_tcl.c 6da58d6e8f42a93c4486b5ba9b187a7f995dee37 @@ -144,8 +144,8 @@ F ext/fts5/test/fts5bigpl.test 04ee0d7eebbebf17c31f5a0b5c5f9494eac3a0cb F ext/fts5/test/fts5columnsize.test a8cfef21ffa1c264b9f670a7d94eeaccb5341c07 F ext/fts5/test/fts5config.test ad2ff42ddc856aed2d05bf89dc1c578c8a39ea3b F ext/fts5/test/fts5content.test 9a952c95518a14182dc3b59e3c8fa71cda82a4e1 -F ext/fts5/test/fts5corrupt.test 928c9c91d40690d301f943a7ed0ffc19e0d0e7b6 -F ext/fts5/test/fts5corrupt2.test 1a830ccd6dbe1b601c7e3f5bbc1cf77bd8c8803b +F ext/fts5/test/fts5corrupt.test c2ad090192708150d50d961278df10ae7a4b8b62 +F ext/fts5/test/fts5corrupt2.test 26c0a39dd9ff73207e6229f83b50b21d37c7658c F ext/fts5/test/fts5corrupt3.test 1ccf575f5126e79f9fec7979fd02a1f40a076be3 F ext/fts5/test/fts5dlidx.test 59b80bbe34169a082c575d9c26f0a7019a7b79c1 F ext/fts5/test/fts5doclist.test 8edb5b57e5f144030ed74ec00ef6fa4294fed79b @@ -1385,7 +1385,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 85aac7b8b6731e2f6880b80cfd62d431ea059799 47a46a9fa4a96cdb96a20b6aec802661b1ee4598 -R c5f7ba1eb5032deaf906327db459f6d6 +P 716e7e747714d6af502f6a87ca8d789bb7ce162a +R 635eaf5cd920b80fc2d9c9be975e5caf U dan -Z 7d64753f9bb762353618192be20f88c2 +Z e0d8c3d69e3d58e6c9e4d77310051289 diff --git a/manifest.uuid b/manifest.uuid index 30fcdc4197..82e41d2242 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -716e7e747714d6af502f6a87ca8d789bb7ce162a \ No newline at end of file +4931e37da4d2c26d7afc5432f7f0d534b51a85fa \ No newline at end of file From 204debf3a3f1b084b4182b10e3c2427c41b92bb6 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 10 Sep 2015 16:39:38 +0000 Subject: [PATCH 076/100] Increment the fts5 version value to indicate that the on-disk format has changed. FossilOrigin-Name: 99de5e3613d557728dd196353516bc7cf64a0e6c --- ext/fts5/fts5Int.h | 2 +- ext/fts5/test/fts5al.test | 6 +++--- ext/fts5/test/fts5version.test | 10 +++++----- manifest | 16 ++++++++-------- manifest.uuid | 2 +- 5 files changed, 18 insertions(+), 18 deletions(-) diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index c52ce67d70..c7f724eab3 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -158,7 +158,7 @@ struct Fts5Config { }; /* Current expected value of %_config table 'version' field */ -#define FTS5_CURRENT_VERSION 3 +#define FTS5_CURRENT_VERSION 4 #define FTS5_CONTENT_NORMAL 0 #define FTS5_CONTENT_NONE 1 diff --git a/ext/fts5/test/fts5al.test b/ext/fts5/test/fts5al.test index 99dfeb357b..efad1b2069 100644 --- a/ext/fts5/test/fts5al.test +++ b/ext/fts5/test/fts5al.test @@ -26,17 +26,17 @@ ifcapable !fts5 { do_execsql_test 1.1 { CREATE VIRTUAL TABLE ft1 USING fts5(x); SELECT * FROM ft1_config; -} {version 3} +} {version 4} do_execsql_test 1.2 { INSERT INTO ft1(ft1, rank) VALUES('pgsz', 32); SELECT * FROM ft1_config; -} {pgsz 32 version 3} +} {pgsz 32 version 4} do_execsql_test 1.3 { INSERT INTO ft1(ft1, rank) VALUES('pgsz', 64); SELECT * FROM ft1_config; -} {pgsz 64 version 3} +} {pgsz 64 version 4} #-------------------------------------------------------------------------- # Test the logic for parsing the rank() function definition. diff --git a/ext/fts5/test/fts5version.test b/ext/fts5/test/fts5version.test index 8c5a772146..7e4d74d114 100644 --- a/ext/fts5/test/fts5version.test +++ b/ext/fts5/test/fts5version.test @@ -30,34 +30,34 @@ do_execsql_test 1.1 { do_execsql_test 1.2 { SELECT * FROM t1_config WHERE k='version' -} {version 3} +} {version 4} do_execsql_test 1.3 { SELECT rowid FROM t1 WHERE t1 MATCH 'a'; } {1} do_execsql_test 1.4 { - UPDATE t1_config set v=4 WHERE k='version'; + UPDATE t1_config set v=5 WHERE k='version'; } do_test 1.5 { db close sqlite3 db test.db catchsql { SELECT * FROM t1 WHERE t1 MATCH 'a' } -} {1 {invalid fts5 file format (found 4, expected 3) - run 'rebuild'}} +} {1 {invalid fts5 file format (found 5, expected 4) - run 'rebuild'}} do_test 1.6 { db close sqlite3 db test.db catchsql { INSERT INTO t1 VALUES('x y z') } -} {1 {invalid fts5 file format (found 4, expected 3) - run 'rebuild'}} +} {1 {invalid fts5 file format (found 5, expected 4) - run 'rebuild'}} do_test 1.7 { execsql { DELETE FROM t1_config WHERE k='version' } db close sqlite3 db test.db catchsql { SELECT * FROM t1 WHERE t1 MATCH 'a' } -} {1 {invalid fts5 file format (found 0, expected 3) - run 'rebuild'}} +} {1 {invalid fts5 file format (found 0, expected 4) - run 'rebuild'}} finish_test diff --git a/manifest b/manifest index b51eb758c4..f78303a53a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\ssegfault\sin\sfts5\sthat\scould\soccur\sif\sthe\sdatabase\scontents\swere\scorrupt. -D 2015-09-10T16:19:01.581 +C Increment\sthe\sfts5\sversion\svalue\sto\sindicate\sthat\sthe\son-disk\sformat\shas\schanged. +D 2015-09-10T16:39:38.823 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -106,7 +106,7 @@ F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0 F ext/fts5/fts5.h f04659e0df5af83731b102189a32280f74f4a6bc -F ext/fts5/fts5Int.h 7e6002582133cb795a21468732cf7c35027a28e4 +F ext/fts5/fts5Int.h 81ba5e474979b166a52a8be306aa3b09d43a10e9 F ext/fts5/fts5_aux.c 7a307760a9c57c750d043188ec0bad59f5b5ec7e F ext/fts5/fts5_buffer.c 64dcaf36a3ebda9e84b7c3b8788887ec325e12a4 F ext/fts5/fts5_config.c 57ee5fe71578cb494574fc0e6e51acb9a22a8695 @@ -135,7 +135,7 @@ F ext/fts5/test/fts5ah.test e592c4978622dbc4de552cd0f9395df60ac5d54c F ext/fts5/test/fts5ai.test f20e53bbf0c55bc596f1fd47f2740dae028b8f37 F ext/fts5/test/fts5aj.test 05b569f5c16ea3098fb1984eec5cf50dbdaae5d8 F ext/fts5/test/fts5ak.test 7b8c5df96df599293f920b7e5521ebc79f647592 -F ext/fts5/test/fts5al.test 440d77c0b39ba73bad2ceb8986c2bb1093570735 +F ext/fts5/test/fts5al.test 5c79525671862861906fa0a848da462a8473eafb F ext/fts5/test/fts5alter.test 6022c61467a82aa11c70822ccad22b328dcf0d04 F ext/fts5/test/fts5auto.test caa5bcf917db11944655a2a9bd38c67c520376ca F ext/fts5/test/fts5aux.test 8c687c948cc98e9a94be014df7d518acc1b3b74f @@ -180,7 +180,7 @@ F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841 F ext/fts5/test/fts5unicode2.test c1dd890ba32b7609adba78e420faa847abe43b59 F ext/fts5/test/fts5unicode3.test 35c3d02aa7acf7d43d8de3bfe32c15ba96e8928e F ext/fts5/test/fts5unindexed.test e9539d5b78c677315e7ed8ea911d4fd25437c680 -F ext/fts5/test/fts5version.test 205beb2a67d9496af64df959e6a19238f69b83e8 +F ext/fts5/test/fts5version.test 978f59541d8cef7e8591f8be2115ec5ccb863e2e F ext/fts5/test/fts5vocab.test cdf97b9678484e9bad5062edf9c9106e5c3b0c5c F ext/fts5/tool/loadfts5.tcl 58e90407cc5c2b1770460119488fd7c0090d4dd3 F ext/fts5/tool/mkfts5c.tcl 5745072c7de346e18c7f491e4c3281fe8a1cfe51 @@ -1385,7 +1385,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 716e7e747714d6af502f6a87ca8d789bb7ce162a -R 635eaf5cd920b80fc2d9c9be975e5caf +P 4931e37da4d2c26d7afc5432f7f0d534b51a85fa +R eed5bbc75e101ddc9b925dace5d76427 U dan -Z e0d8c3d69e3d58e6c9e4d77310051289 +Z 95c35bdd15155223e616b807a46e7fe9 diff --git a/manifest.uuid b/manifest.uuid index 82e41d2242..477390ac7c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4931e37da4d2c26d7afc5432f7f0d534b51a85fa \ No newline at end of file +99de5e3613d557728dd196353516bc7cf64a0e6c \ No newline at end of file From 383de6918c42b6a8d927234be6a3c81689acd5cd Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 10 Sep 2015 17:20:57 +0000 Subject: [PATCH 077/100] Create separate "path" and "root" columns in the json_each() and json_tree() virtual tables. "Root" is the 2nd parameter and is fixed. "Path" varies as json_tree() walks the hierarchy. FossilOrigin-Name: 127cce3eb96b819005832997e0a082df9fb96f0b --- ext/misc/json1.c | 73 +++++++++++++++++++++++++----------------------- manifest | 12 ++++---- manifest.uuid | 2 +- 3 files changed, 45 insertions(+), 42 deletions(-) diff --git a/ext/misc/json1.c b/ext/misc/json1.c index d51d69e1ee..755d035c37 100644 --- a/ext/misc/json1.c +++ b/ext/misc/json1.c @@ -1444,7 +1444,7 @@ struct JsonEachCursor { u8 eType; /* Type of top-level element */ u8 bRecursive; /* True for json_tree(). False for json_each() */ char *zJson; /* Input JSON */ - char *zPath; /* Path by which to filter zJson */ + char *zRoot; /* Path by which to filter zJson */ JsonParse sParse; /* Parse of the input JSON */ }; @@ -1467,16 +1467,17 @@ static int jsonEachConnect( #define JEACH_ID 4 #define JEACH_PARENT 5 #define JEACH_FULLKEY 6 -#define JEACH_JSON 7 -#define JEACH_PATH 8 +#define JEACH_PATH 7 +#define JEACH_JSON 8 +#define JEACH_ROOT 9 UNUSED_PARAM(pzErr); UNUSED_PARAM(argv); UNUSED_PARAM(argc); UNUSED_PARAM(pAux); rc = sqlite3_declare_vtab(db, - "CREATE TABLE x(key,value,type,atom,id,parent,fullkey," - "json HIDDEN,path HIDDEN)"); + "CREATE TABLE x(key,value,type,atom,id,parent,fullkey,path," + "json HIDDEN,root HIDDEN)"); if( rc==SQLITE_OK ){ pNew = *ppVtab = sqlite3_malloc( sizeof(*pNew) ); if( pNew==0 ) return SQLITE_NOMEM; @@ -1517,14 +1518,14 @@ static int jsonEachOpenTree(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ ** held. */ static void jsonEachCursorReset(JsonEachCursor *p){ sqlite3_free(p->zJson); - sqlite3_free(p->zPath); + sqlite3_free(p->zRoot); jsonParseReset(&p->sParse); p->iRowid = 0; p->i = 0; p->iEnd = 0; p->eType = 0; p->zJson = 0; - p->zPath = 0; + p->zRoot = 0; } /* Destructor for a jsonEachCursor object */ @@ -1668,8 +1669,8 @@ static int jsonEachColumn( if( p->bRecursive ){ jsonEachComputePath(p, &x, p->i); }else{ - if( p->zPath ){ - jsonAppendRaw(&x, p->zPath, (int)strlen(p->zPath)); + if( p->zRoot ){ + jsonAppendRaw(&x, p->zRoot, (int)strlen(p->zRoot)); }else{ jsonAppendChar(&x, '$'); } @@ -1683,18 +1684,20 @@ static int jsonEachColumn( break; } case JEACH_PATH: { - const char *zPath = p->zPath; - if( zPath==0 ){ - if( p->bRecursive ){ - JsonString x; - jsonInit(&x, ctx); - jsonEachComputePath(p, &x, p->sParse.aUp[p->i]); - jsonResult(&x); - break; - } - zPath = "$"; + if( p->bRecursive ){ + JsonString x; + jsonInit(&x, ctx); + jsonEachComputePath(p, &x, p->sParse.aUp[p->i]); + jsonResult(&x); + break; } - sqlite3_result_text(ctx, zPath, -1, SQLITE_STATIC); + /* For json_each() path and root are the same so fall through + ** into the root case */ + } + case JEACH_ROOT: { + const char *zRoot = p->zRoot; + if( zRoot==0 ) zRoot = "$"; + sqlite3_result_text(ctx, zRoot, -1, SQLITE_STATIC); break; } default: { @@ -1715,7 +1718,7 @@ static int jsonEachRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ /* The query strategy is to look for an equality constraint on the json ** column. Without such a constraint, the table cannot operate. idxNum is -** 1 if the constraint is found, 3 if the constraint and zPath are found, +** 1 if the constraint is found, 3 if the constraint and zRoot are found, ** and 0 otherwise. */ static int jsonEachBestIndex( @@ -1724,7 +1727,7 @@ static int jsonEachBestIndex( ){ int i; int jsonIdx = -1; - int pathIdx = -1; + int rootIdx = -1; const struct sqlite3_index_constraint *pConstraint; UNUSED_PARAM(tab); @@ -1734,7 +1737,7 @@ static int jsonEachBestIndex( if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; switch( pConstraint->iColumn ){ case JEACH_JSON: jsonIdx = i; break; - case JEACH_PATH: pathIdx = i; break; + case JEACH_ROOT: rootIdx = i; break; default: /* no-op */ break; } } @@ -1745,11 +1748,11 @@ static int jsonEachBestIndex( pIdxInfo->estimatedCost = 1.0; pIdxInfo->aConstraintUsage[jsonIdx].argvIndex = 1; pIdxInfo->aConstraintUsage[jsonIdx].omit = 1; - if( pathIdx<0 ){ + if( rootIdx<0 ){ pIdxInfo->idxNum = 1; }else{ - pIdxInfo->aConstraintUsage[pathIdx].argvIndex = 2; - pIdxInfo->aConstraintUsage[pathIdx].omit = 1; + pIdxInfo->aConstraintUsage[rootIdx].argvIndex = 2; + pIdxInfo->aConstraintUsage[rootIdx].omit = 1; pIdxInfo->idxNum = 3; } } @@ -1764,7 +1767,7 @@ static int jsonEachFilter( ){ JsonEachCursor *p = (JsonEachCursor*)cur; const char *z; - const char *zPath; + const char *zRoot; sqlite3_int64 n; UNUSED_PARAM(idxStr); @@ -1774,11 +1777,11 @@ static int jsonEachFilter( z = (const char*)sqlite3_value_text(argv[0]); if( z==0 ) return SQLITE_OK; if( idxNum&2 ){ - zPath = (const char*)sqlite3_value_text(argv[1]); - if( zPath==0 ) return SQLITE_OK; - if( zPath[0]!='$' ){ + zRoot = (const char*)sqlite3_value_text(argv[1]); + if( zRoot==0 ) return SQLITE_OK; + if( zRoot[0]!='$' ){ sqlite3_free(cur->pVtab->zErrMsg); - cur->pVtab->zErrMsg = jsonPathSyntaxError(zPath); + cur->pVtab->zErrMsg = jsonPathSyntaxError(zRoot); return cur->pVtab->zErrMsg ? SQLITE_ERROR : SQLITE_NOMEM; } } @@ -1803,10 +1806,10 @@ static int jsonEachFilter( if( idxNum==3 ){ const char *zErr = 0; n = sqlite3_value_bytes(argv[1]); - p->zPath = sqlite3_malloc64( n+1 ); - if( p->zPath==0 ) return SQLITE_NOMEM; - memcpy(p->zPath, zPath, (size_t)n+1); - pNode = jsonLookupStep(&p->sParse, 0, p->zPath+1, 0, &zErr); + p->zRoot = sqlite3_malloc64( n+1 ); + if( p->zRoot==0 ) return SQLITE_NOMEM; + memcpy(p->zRoot, zRoot, (size_t)n+1); + pNode = jsonLookupStep(&p->sParse, 0, p->zRoot+1, 0, &zErr); if( p->sParse.nErr ){ sqlite3_free(cur->pVtab->zErrMsg); cur->pVtab->zErrMsg = jsonPathSyntaxError(zErr); diff --git a/manifest b/manifest index a73e1f29f1..18da2c9e50 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\sthe\ssqlite3ext.h\sheader\sfile\sresponsive\sto\s-DSQLITE_OMIT_LOAD_EXTENSION. -D 2015-09-10T15:24:27.984 +C Create\sseparate\s"path"\sand\s"root"\scolumns\sin\sthe\sjson_each()\sand\sjson_tree()\nvirtual\stables.\s\s"Root"\sis\sthe\s2nd\sparameter\sand\sis\sfixed.\s\s\n"Path"\svaries\sas\sjson_tree()\swalks\sthe\shierarchy. +D 2015-09-10T17:20:57.334 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -194,7 +194,7 @@ F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f F ext/misc/fuzzer.c 4c84635c71c26cfa7c2e5848cf49fe2d2cfcd767 F ext/misc/ieee754.c b0362167289170627659e84173f5d2e8fee8566e -F ext/misc/json1.c 46c2aff110eb8241591cd29267ddb57a8d0ab337 +F ext/misc/json1.c 4387d091c0ec0f4d9ed05560960f03d366db4fe0 F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c bcbee3c061b884eccb80e21651daaae8e1e43c63 F ext/misc/regexp.c af92cdaa5058fcec1451e49becc7ba44dba023dc @@ -1384,7 +1384,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P bb8ee3b140a0a8b7c89f5debf22369f204ceef0b -R 9f03517d27368f0ee452371692eeb689 +P 47a46a9fa4a96cdb96a20b6aec802661b1ee4598 +R 8aee48c10c3e97ebfdd79a73a44181a5 U drh -Z 40738b9489636e13edf984d114f41f8e +Z 9f31b9d4db81b45c399857b6b93c0174 diff --git a/manifest.uuid b/manifest.uuid index 54212ec5b7..9d15c460f0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -47a46a9fa4a96cdb96a20b6aec802661b1ee4598 \ No newline at end of file +127cce3eb96b819005832997e0a082df9fb96f0b \ No newline at end of file From 0b19c96935b38bb47ffac79a63f3507611e99ca3 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 10 Sep 2015 19:22:25 +0000 Subject: [PATCH 078/100] Fix a potential NULL pointer deref in the testing logic of pcache1. NB: The -DSQLITE_TEST compile-time option is needed to hit the problem. FossilOrigin-Name: f5580f08538636ffb1367e717a33756288ccddde --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/pcache1.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 6d894da74a..69a0154d20 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Modify\sthe\sfts5\sleaf\spage\sformat\sto\spermit\sfaster\sseek\soperations.\sThis\sis\sa\sfile-format\schange.\sAny\sexisting\sdatabases\scan\sbe\supgraded\sby\srunning\sthe\sfts5\s'rebuild'\scommand. -D 2015-09-10T17:23:37.872 +C Fix\sa\spotential\sNULL\spointer\sderef\sin\sthe\stesting\slogic\sof\spcache1.\nNB:\sThe\s-DSQLITE_TEST\scompile-time\soption\sis\sneeded\sto\shit\sthe\sproblem. +D 2015-09-10T19:22:25.531 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -332,7 +332,7 @@ F src/pager.h 6d435f563b3f7fcae4b84433b76a6ac2730036e2 F src/parse.y f599aa5e871a493330d567ced93de696f61f48f7 F src/pcache.c 24be750c79272e0ca7b6e007bc94999700f3e5ef F src/pcache.h 9968603796240cdf83da7e7bef76edf90619cea9 -F src/pcache1.c ca0eeaaf9ffab7a9ded47d54639d8ab147d442e6 +F src/pcache1.c a0c0bb29f7bd720743a16a95eb5dedba3ade15bc F src/pragma.c d71b813e67bf03f3116b9dd5164fbfd81ec673a2 F src/pragma.h 631a91c8b0e6ca8f051a1d8a4a0da4150e04620a F src/prepare.c 82e5db1013846a819f198336fed72c44c974e7b1 @@ -1385,7 +1385,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 127cce3eb96b819005832997e0a082df9fb96f0b 99de5e3613d557728dd196353516bc7cf64a0e6c -R 94d00b9df8244e9367669b9ccbab9a5c -U dan -Z a63610a6f3469795714a7c77d99e36a6 +P 0c0c4ae971e54efc526eed7bd071c90dfadb95ff +R 3ba34701e76f91459126301401d510a6 +U drh +Z c5b1b6a1894a9ad25bccddfa15233f55 diff --git a/manifest.uuid b/manifest.uuid index a1569dabf1..be02be3668 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0c0c4ae971e54efc526eed7bd071c90dfadb95ff \ No newline at end of file +f5580f08538636ffb1367e717a33756288ccddde \ No newline at end of file diff --git a/src/pcache1.c b/src/pcache1.c index 1beb66d733..00a0205f5a 100644 --- a/src/pcache1.c +++ b/src/pcache1.c @@ -1223,7 +1223,7 @@ void sqlite3PcacheStats( ){ PgHdr1 *p; int nRecyclable = 0; - for(p=pcache1.grp.lru.pLruNext; !p->isAnchor; p=p->pLruNext){ + for(p=pcache1.grp.lru.pLruNext; p && !p->isAnchor; p=p->pLruNext){ assert( p->isPinned==0 ); nRecyclable++; } From bcdf78a61caa6263e84e8b3f5e2fff6afa0410ac Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 10 Sep 2015 20:34:56 +0000 Subject: [PATCH 079/100] Experimental implementation of sqlite3_result_subtype() and sqlite3_value_subtype() interfaces. FossilOrigin-Name: 7b5be299c617a3d3ed327ed30ef0a66a62c85b1f --- manifest | 22 +++++++++++++--------- manifest.uuid | 2 +- src/sqlite.h.in | 30 ++++++++++++++++++++++++++++++ src/test_func.c | 33 +++++++++++++++++++++++++++++++-- src/vdbeInt.h | 1 + src/vdbeapi.c | 7 +++++++ test/subtype1.test | 31 +++++++++++++++++++++++++++++++ 7 files changed, 114 insertions(+), 12 deletions(-) create mode 100644 test/subtype1.test diff --git a/manifest b/manifest index 69a0154d20..1d742f0ff3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\spotential\sNULL\spointer\sderef\sin\sthe\stesting\slogic\sof\spcache1.\nNB:\sThe\s-DSQLITE_TEST\scompile-time\soption\sis\sneeded\sto\shit\sthe\sproblem. -D 2015-09-10T19:22:25.531 +C Experimental\simplementation\sof\ssqlite3_result_subtype()\sand\nsqlite3_value_subtype()\sinterfaces. +D 2015-09-10T20:34:56.434 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -342,7 +342,7 @@ F src/resolve.c 3126f7694b8ce0f97282d7dd3a5198b8fa18dce9 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c c17613385bc6b095c421b1f30548814f5fd8a9b2 F src/shell.c 6332ef06db1390ef812cfdff1fc97b4fd76cdd42 -F src/sqlite.h.in 378bebc8fe6a88bade25e5f23b7e6123fdc64b00 +F src/sqlite.h.in 65ff1449e8c181da9ec752e8304e0a1efffaf47f F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 5088f0b63491677da848c0d07d5711781302d362 F src/sqliteInt.h 788dc0ea7ba32ec9fec06c628c1792d7b4753d86 @@ -368,7 +368,7 @@ F src/test_config.c fb2e5d354d9a077f5fbb261652eff4787deb104f F src/test_demovfs.c 0de72c2c89551629f58486fde5734b7d90758852 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc F src/test_fs.c ced436e3d4b8e4681328409b8081051ce614e28f -F src/test_func.c f1ac201465472e76a73e2f3695c3553c63e7322a +F src/test_func.c 0d9c25956152adefee8881c6fadc8354793764d0 F src/test_hexio.c abfdecb6fa58c354623978efceb088ca18e379cd F src/test_init.c 66b33120ffe9cd853b5a905ec850d51151337b32 F src/test_intarray.c 870124b95ec4c645d4eb84f15efb7133528fb1a5 @@ -405,8 +405,8 @@ F src/util.c fc612367108b74573c5fd13a85d0a23027f438bd F src/vacuum.c 2ddd5cad2a7b9cef7f9e431b8c7771634c6b1701 F src/vdbe.c 6d85be995bd2308a5aa2a68c7b564c5d4cc1a6fb F src/vdbe.h 4bc88bd0e06f8046ee6ab7487c0015e85ad949ad -F src/vdbeInt.h 8b54e01ad0463590e7cffabce0bc36da9ee4f816 -F src/vdbeapi.c b821d530bcb2900b4604cf5206f2177f3f881d15 +F src/vdbeInt.h 8b867eac234e28627ffcace3cd4b4b79bbec664b +F src/vdbeapi.c 0d890f57caf143b114a95ce699e59af51359c508 F src/vdbeaux.c fd00b489ab3f44f2dca1e4344faf289b7bfcf649 F src/vdbeblob.c 1d7b97115e7bbac4c318db416d2ca83fc779544a F src/vdbemem.c 19b3036aa4d676e7103b0fb5efd6327da455f915 @@ -1036,6 +1036,7 @@ F test/subquery.test d7268d193dd33d5505df965399d3a594e76ae13f F test/subquery2.test 438f8a7da1457277b22e4176510f7659b286995f F test/subselect.test d24fd8757daf97dafd2e889c73ea4c4272dcf4e4 F test/substr.test 18f57c4ca8a598805c4d64e304c418734d843c1a +F test/subtype1.test 7fe09496352f97053af1437150751be2d0a0cae8 F test/superlock.test 1cde669f68d2dd37d6c9bd35eee1d95491ae3fc2 F test/sync.test a34cd43e98b7fb84eabbf38f7ed8f7349b3f3d85 F test/syscall.test d2fdaad713f103ac611fe7ef9b724c7b69f8149c @@ -1385,7 +1386,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 0c0c4ae971e54efc526eed7bd071c90dfadb95ff -R 3ba34701e76f91459126301401d510a6 +P f5580f08538636ffb1367e717a33756288ccddde +R a16668bbab7e8bb266cf503c6d51d6a9 +T *branch * subtypes +T *sym-subtypes * +T -sym-trunk * U drh -Z c5b1b6a1894a9ad25bccddfa15233f55 +Z 5917071f863dd81d33499ad8df67eb85 diff --git a/manifest.uuid b/manifest.uuid index be02be3668..4537efc0d6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f5580f08538636ffb1367e717a33756288ccddde \ No newline at end of file +7b5be299c617a3d3ed327ed30ef0a66a62c85b1f \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index a7d6318e5b..65a0bedb05 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -4357,6 +4357,22 @@ const void *sqlite3_value_text16be(sqlite3_value*); int sqlite3_value_type(sqlite3_value*); int sqlite3_value_numeric_type(sqlite3_value*); +/* +** CAPI3REF: Obtaining SQL Values +** METHOD: sqlite3_value +** +** The sqlite3_value_subtype(V) function returns the subtype for +** an application-defined SQL function argument V. The subtype +** information can be used to pass a limited amount of context from +** one SQL function to another. Use the [sqlite3_result_subtype()] +** routine to set the subtype for the return value of an SQL function. +** +** SQLite makes no use of subtype itself. It merely passes the subtype +** from the result of one application-defined function to the input of +** another. +*/ +unsigned int sqlite3_value_subtype(sqlite3_value*); + /* ** CAPI3REF: Copy And Free SQL Values ** METHOD: sqlite3_value @@ -4656,6 +4672,20 @@ void sqlite3_result_value(sqlite3_context*, sqlite3_value*); void sqlite3_result_zeroblob(sqlite3_context*, int n); int sqlite3_result_zeroblob64(sqlite3_context*, sqlite3_uint64 n); + +/* +** CAPI3REF: Setting The Subtype Of An SQL Function +** METHOD: sqlite3_context +** +** The sqlite3_result_subtype(C,T) function causes the subtype of +** the result from the application-defined function with context C +** to be T. Only the lower 8 bits of the subtype T are preserved +** in current versions of SQLite; higher order bits are discarded. +** The number of subtype bytes preserved by SQLite might increase +** in future releases of SQLite. +*/ +void sqlite3_result_subtype(sqlite3_context*,unsigned int); + /* ** CAPI3REF: Define New Collating Sequences ** METHOD: sqlite3 diff --git a/src/test_func.c b/src/test_func.c index 2e34fa074e..63cf18e3f7 100644 --- a/src/test_func.c +++ b/src/test_func.c @@ -462,7 +462,7 @@ static void real2hex( } /* -** tclcmd: test_extract(record, field) +** test_extract(record, field) ** ** This function implements an SQL user-function that accepts a blob ** containing a formatted database record as the first argument. The @@ -509,7 +509,7 @@ static void test_extract( } /* -** tclcmd: test_decode(record) +** test_decode(record) ** ** This function implements an SQL user-function that accepts a blob ** containing a formatted database record as its only argument. It returns @@ -601,6 +601,8 @@ static void test_decode( } /* +** test_zeroblob(N) +** ** The implementation of scalar SQL function "test_zeroblob()". This is ** similar to the built-in zeroblob() function, except that it does not ** check that the integer parameter is within range before passing it @@ -615,6 +617,31 @@ static void test_zeroblob( sqlite3_result_zeroblob(context, nZero); } +/* test_getsubtype(V) +** +** Return the subtype for value V. +*/ +static void test_getsubtype( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + sqlite3_result_int(context, (int)sqlite3_value_subtype(argv[0])); +} + +/* test_setsubtype(V, T) +** +** Return the value V with its subtype changed to T +*/ +static void test_setsubtype( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + sqlite3_result_value(context, argv[0]); + sqlite3_result_subtype(context, (unsigned int)sqlite3_value_int(argv[1])); +} + static int registerTestFunctions(sqlite3 *db){ static const struct { char *zName; @@ -641,6 +668,8 @@ static int registerTestFunctions(sqlite3 *db){ { "test_decode", 1, SQLITE_UTF8, test_decode}, { "test_extract", 2, SQLITE_UTF8, test_extract}, { "test_zeroblob", 1, SQLITE_UTF8|SQLITE_DETERMINISTIC, test_zeroblob}, + { "test_getsubtype", 1, SQLITE_UTF8, test_getsubtype}, + { "test_setsubtype", 2, SQLITE_UTF8, test_setsubtype}, }; int i; diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 4a90ed6483..7884d955b0 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -175,6 +175,7 @@ struct Mem { } u; u16 flags; /* Some combination of MEM_Null, MEM_Str, MEM_Dyn, etc. */ u8 enc; /* SQLITE_UTF8, SQLITE_UTF16BE, SQLITE_UTF16LE */ + u8 eSubtype; /* Subtype for this value */ int n; /* Number of characters in string value, excluding '\0' */ char *z; /* String or BLOB value */ /* ShallowCopy only needs to copy the information above */ diff --git a/src/vdbeapi.c b/src/vdbeapi.c index faf97634c8..06b14e1276 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -187,6 +187,9 @@ int sqlite3_value_int(sqlite3_value *pVal){ sqlite_int64 sqlite3_value_int64(sqlite3_value *pVal){ return sqlite3VdbeIntValue((Mem*)pVal); } +unsigned int sqlite3_value_subtype(sqlite3_value *pVal){ + return ((Mem*)pVal)->eSubtype; +} const unsigned char *sqlite3_value_text(sqlite3_value *pVal){ return (const unsigned char *)sqlite3ValueText(pVal, SQLITE_UTF8); } @@ -365,6 +368,10 @@ void sqlite3_result_null(sqlite3_context *pCtx){ assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); sqlite3VdbeMemSetNull(pCtx->pOut); } +void sqlite3_result_subtype(sqlite3_context *pCtx, unsigned int eSubtype){ + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); + pCtx->pOut->eSubtype = eSubtype & 0xff; +} void sqlite3_result_text( sqlite3_context *pCtx, const char *z, diff --git a/test/subtype1.test b/test/subtype1.test new file mode 100644 index 0000000000..1019611024 --- /dev/null +++ b/test/subtype1.test @@ -0,0 +1,31 @@ +# 2015-09-10 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements tests for sqlite3_value_subtype() and +# sqlite3_result_subtype() interfaces. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +do_execsql_test subtype1-100 { + SELECT test_getsubtype('hello'); +} {0} +do_execsql_test subtype1-110 { + SELECT test_getsubtype(test_setsubtype('hello',123)); +} {123} +do_execsql_test subtype1-120 { + SELECT typeof(test_setsubtype('hello',123)); +} {text} +do_execsql_test subtype1-130 { + SELECT test_setsubtype('hello',123); +} {hello} + +finish_test From cf94f179647fd7573f58ad40bbf5603a77035a73 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 10 Sep 2015 20:40:21 +0000 Subject: [PATCH 080/100] Make the sqlite3_value_subtype() and sqlite3_result_subtype() interfaces available to loadable extensions. FossilOrigin-Name: c6fca0be11f7414292279e2ea1b004260e6f4bb6 --- manifest | 17 +++++++---------- manifest.uuid | 2 +- src/loadext.c | 5 ++++- src/sqlite3ext.h | 6 ++++++ 4 files changed, 18 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 1d742f0ff3..285e65a2cd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Experimental\simplementation\sof\ssqlite3_result_subtype()\sand\nsqlite3_value_subtype()\sinterfaces. -D 2015-09-10T20:34:56.434 +C Make\sthe\ssqlite3_value_subtype()\sand\ssqlite3_result_subtype()\sinterfaces\navailable\sto\sloadable\sextensions. +D 2015-09-10T20:40:21.038 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -304,7 +304,7 @@ F src/insert.c 076dc5876e261a9908603d54cfc5344cd680166c F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c d344a95d60c24e2f490ee59db9784b1b17439012 -F src/loadext.c dfcee8c7c032cd0fd55af3e0fc1fcfb01e426df2 +F src/loadext.c f0b66d28e377fd6c6d36cc9d92df1ff251ebee44 F src/main.c e17fcffae4306a9b8334faf3bac80d7396850b54 F src/malloc.c 3a37ce6979a40f499d8cea9e9ab4e8517854d35d F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 @@ -344,7 +344,7 @@ F src/select.c c17613385bc6b095c421b1f30548814f5fd8a9b2 F src/shell.c 6332ef06db1390ef812cfdff1fc97b4fd76cdd42 F src/sqlite.h.in 65ff1449e8c181da9ec752e8304e0a1efffaf47f F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad -F src/sqlite3ext.h 5088f0b63491677da848c0d07d5711781302d362 +F src/sqlite3ext.h 64350bf36833a56ad675e27392a913f417c5c308 F src/sqliteInt.h 788dc0ea7ba32ec9fec06c628c1792d7b4753d86 F src/sqliteLimit.h 216557999cb45f2e3578ed53ebefe228d779cb46 F src/status.c f266ad8a2892d659b74f0f50cb6a88b6e7c12179 @@ -1386,10 +1386,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f5580f08538636ffb1367e717a33756288ccddde -R a16668bbab7e8bb266cf503c6d51d6a9 -T *branch * subtypes -T *sym-subtypes * -T -sym-trunk * +P 7b5be299c617a3d3ed327ed30ef0a66a62c85b1f +R 3b51d146c21b59a26b4bd6730fa2e17a U drh -Z 5917071f863dd81d33499ad8df67eb85 +Z 3c2729e75edd6da9e7ec99089fe2f222 diff --git a/manifest.uuid b/manifest.uuid index 4537efc0d6..d8c4b3eba0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7b5be299c617a3d3ed327ed30ef0a66a62c85b1f \ No newline at end of file +c6fca0be11f7414292279e2ea1b004260e6f4bb6 \ No newline at end of file diff --git a/src/loadext.c b/src/loadext.c index 1d398c54ce..b4b981e548 100644 --- a/src/loadext.c +++ b/src/loadext.c @@ -407,7 +407,10 @@ static const sqlite3_api_routines sqlite3Apis = { (sqlite3_value*(*)(const sqlite3_value*))sqlite3_value_dup, sqlite3_value_free, sqlite3_result_zeroblob64, - sqlite3_bind_zeroblob64 + sqlite3_bind_zeroblob64, + /* Version 3.8.12 and later */ + sqlite3_value_subtype, + sqlite3_result_subtype }; /* diff --git a/src/sqlite3ext.h b/src/sqlite3ext.h index 9c01241a12..9b9f610e93 100644 --- a/src/sqlite3ext.h +++ b/src/sqlite3ext.h @@ -272,6 +272,9 @@ struct sqlite3_api_routines { void (*value_free)(sqlite3_value*); int (*result_zeroblob64)(sqlite3_context*,sqlite3_uint64); int (*bind_zeroblob64)(sqlite3_stmt*, int, sqlite3_uint64); + /* Version 3.8.12 and later */ + unsigned int (*value_subtype)(sqlite3_value*); + void (*result_subtype)(sqlite3_context*,unsigned int); }; /* @@ -508,6 +511,9 @@ struct sqlite3_api_routines { #define sqlite3_value_free sqlite3_api->value_free #define sqlite3_result_zeroblob64 sqlite3_api->result_zeroblob64 #define sqlite3_bind_zeroblob64 sqlite3_api->bind_zeroblob64 +/* Version 3.8.12 and later */ +#define sqlite3_value_subtype sqlite3_api->value_subtype +#define sqlite3_result_subtype sqlite3_api->result_subtype #endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */ #if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) From f5ddb9c214b5e6237e9bfb878aa21cfd2ef129fc Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 11 Sep 2015 00:06:41 +0000 Subject: [PATCH 081/100] Take out the goofy '$$' path syntax. Instead, use subtypes to communicate when a string is JSON. Add the json() function that validates and minifies the JSON and sets the appropriate subtype. FossilOrigin-Name: 8a80d6459e246ec1b38325e1cbd1e862157138b3 --- ext/misc/json1.c | 89 +++++++++++++++-------------------------------- manifest | 14 ++++---- manifest.uuid | 2 +- test/json101.test | 18 ++++++++-- 4 files changed, 51 insertions(+), 72 deletions(-) diff --git a/ext/misc/json1.c b/ext/misc/json1.c index 755d035c37..a217ce80b8 100644 --- a/ext/misc/json1.c +++ b/ext/misc/json1.c @@ -68,6 +68,9 @@ struct JsonString { #define JSON_ARRAY 6 #define JSON_OBJECT 7 +/* The "subtype" set for JSON values */ +#define JSON_SUBTYPE 74 /* Ascii for "J" */ + /* ** Names of the various JSON types: */ @@ -82,8 +85,7 @@ static const char * const jsonType[] = { #define JNODE_REMOVE 0x04 /* Do not output */ #define JNODE_REPLACE 0x08 /* Replace with JsonNode.iVal */ #define JNODE_APPEND 0x10 /* More ARRAY/OBJECT entries at u.iAppend */ -#define JNODE_JSON 0x20 /* Treat REPLACE as JSON text */ -#define JNODE_LABEL 0x40 /* Is a label of an object */ +#define JNODE_LABEL 0x20 /* Is a label of an object */ /* A single node of parsed JSON @@ -243,8 +245,7 @@ static void jsonAppendString(JsonString *p, const char *zIn, u32 N){ */ static void jsonAppendValue( JsonString *p, /* Append to this JSON string */ - sqlite3_value *pValue, /* Value to append */ - u8 textIsJson /* Try to treat text values as JSON */ + sqlite3_value *pValue /* Value to append */ ){ switch( sqlite3_value_type(pValue) ){ case SQLITE_NULL: { @@ -261,7 +262,7 @@ static void jsonAppendValue( case SQLITE_TEXT: { const char *z = (const char*)sqlite3_value_text(pValue); u32 n = (u32)sqlite3_value_bytes(pValue); - if( textIsJson ){ + if( sqlite3_value_subtype(pValue)==JSON_SUBTYPE ){ jsonAppendRaw(p, z, n); }else{ jsonAppendString(p, z, n); @@ -365,8 +366,7 @@ static void jsonRenderNode( if( pNode[j].jnFlags & (JNODE_REMOVE|JNODE_REPLACE) ){ if( pNode[j].jnFlags & JNODE_REPLACE ){ jsonAppendSeparator(pOut); - jsonAppendValue(pOut, aReplace[pNode[j].iVal], - (pNode[j].jnFlags & JNODE_JSON)!=0); + jsonAppendValue(pOut, aReplace[pNode[j].iVal]); } }else{ jsonAppendSeparator(pOut); @@ -391,8 +391,7 @@ static void jsonRenderNode( jsonRenderNode(&pNode[j], pOut, aReplace); jsonAppendChar(pOut, ':'); if( pNode[j+1].jnFlags & JNODE_REPLACE ){ - jsonAppendValue(pOut, aReplace[pNode[j+1].iVal], - (pNode[j+1].jnFlags & JNODE_JSON)!=0); + jsonAppendValue(pOut, aReplace[pNode[j+1].iVal]); }else{ jsonRenderNode(&pNode[j+1], pOut, aReplace); } @@ -421,6 +420,7 @@ static void jsonReturnJson( jsonInit(&s, pCtx); jsonRenderNode(pNode, &s, aReplace); jsonResult(&s); + sqlite3_result_subtype(pCtx, JSON_SUBTYPE); } /* @@ -951,21 +951,15 @@ static char *jsonPathSyntaxError(const char *zErr){ ** ** If pApnd!=NULL then try to append missing nodes and set *pApnd = 1 if ** nodes are appended. -** -** If the path starts with $$ then set *pFlags to JNODE_REPLACE|JNODE_JSON -** as a single to the caller that the input text to be inserted should be -** interpreted as JSON rather than as ordinary text. */ static JsonNode *jsonLookup( JsonParse *pParse, /* The JSON to search */ const char *zPath, /* The path to search */ int *pApnd, /* Append nodes to complete path if not NULL */ - sqlite3_context *pCtx, /* Report errors here, if not NULL */ - u8 *pFlags /* Write JNODE_REPLACE or _REPLACE|_JSON here */ + sqlite3_context *pCtx /* Report errors here, if not NULL */ ){ const char *zErr = 0; JsonNode *pNode = 0; - u8 fg = JNODE_REPLACE; if( zPath==0 ) return 0; if( zPath[0]!='$' ){ @@ -973,15 +967,6 @@ static JsonNode *jsonLookup( goto lookup_err; } zPath++; - if( zPath[0]=='$' ){ - if( pFlags==0 ){ - zErr = zPath; - goto lookup_err; - } - zPath++; - fg = JNODE_REPLACE|JNODE_JSON; - } - if( pFlags ) *pFlags = fg; pNode = jsonLookupStep(pParse, 0, zPath, pApnd, &zErr); return pNode; @@ -996,7 +981,6 @@ lookup_err: sqlite3_result_error_nomem(pCtx); } } - if( pFlags ) *pFlags = fg; return 0; } @@ -1060,32 +1044,16 @@ static void jsonParseFunc( } /* -** The json_test1(JSON) function parses and rebuilds the JSON string. +** The json_test1(JSON) function return true (1) if the input is JSON +** text generated by another json function. It returns (0) if the input +** is not known to be JSON. */ static void jsonTest1Func( sqlite3_context *ctx, int argc, sqlite3_value **argv ){ - JsonParse x; /* The parse */ - if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; - jsonReturnJson(x.aNode, ctx, 0); - jsonParseReset(&x); -} - -/* -** The json_nodecount(JSON) function returns the number of nodes in the -** input JSON string. -*/ -static void jsonNodeCountFunc( - sqlite3_context *ctx, - int argc, - sqlite3_value **argv -){ - JsonParse x; /* The parse */ - if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; - sqlite3_result_int64(ctx, (sqlite3_int64)x.nNode); - jsonParseReset(&x); + sqlite3_result_int(ctx, sqlite3_value_subtype(argv[0])==JSON_SUBTYPE); } #endif /* SQLITE_DEBUG */ @@ -1110,10 +1078,11 @@ static void jsonArrayFunc( jsonAppendChar(&jx, '['); for(i=0; i2 ){ jsonAppendSeparator(&jx); @@ -1196,6 +1165,7 @@ static void jsonExtractFunc( if( argc>2 && i==argc ){ jsonAppendChar(&jx, ']'); jsonResult(&jx); + sqlite3_result_subtype(ctx, JSON_SUBTYPE); } jsonReset(&jx); jsonParseReset(&x); @@ -1234,10 +1204,11 @@ static void jsonObjectFunc( n = (u32)sqlite3_value_bytes(argv[i]); jsonAppendString(&jx, z, n); jsonAppendChar(&jx, ':'); - jsonAppendValue(&jx, argv[i+1], 0); + jsonAppendValue(&jx, argv[i+1]); } jsonAppendChar(&jx, '}'); jsonResult(&jx); + sqlite3_result_subtype(ctx, JSON_SUBTYPE); } @@ -1263,7 +1234,7 @@ static void jsonRemoveFunc( for(i=1; i<(u32)argc; i++){ zPath = (const char*)sqlite3_value_text(argv[i]); if( zPath==0 ) goto remove_done; - pNode = jsonLookup(&x, zPath, 0, ctx, 0); + pNode = jsonLookup(&x, zPath, 0, ctx); if( x.nErr ) goto remove_done; if( pNode ) pNode->jnFlags |= JNODE_REMOVE; } @@ -1299,13 +1270,11 @@ static void jsonReplaceFunc( if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; if( x.nNode ){ for(i=1; i<(u32)argc; i+=2){ - u8 jnFlags = JNODE_REPLACE; zPath = (const char*)sqlite3_value_text(argv[i]); - pNode = jsonLookup(&x, zPath, 0, ctx, &jnFlags); + pNode = jsonLookup(&x, zPath, 0, ctx); if( x.nErr ) goto replace_err; if( pNode ){ - pNode->jnFlags &= ~JNODE_JSON; - pNode->jnFlags |= jnFlags; + pNode->jnFlags |= JNODE_REPLACE; pNode->iVal = i+1; } } @@ -1351,18 +1320,16 @@ static void jsonSetFunc( if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; if( x.nNode ){ for(i=1; i<(u32)argc; i+=2){ - u8 jnFlags = JNODE_REPLACE; zPath = (const char*)sqlite3_value_text(argv[i]); bApnd = 0; - pNode = jsonLookup(&x, zPath, &bApnd, ctx, &jnFlags); + pNode = jsonLookup(&x, zPath, &bApnd, ctx); if( x.oom ){ sqlite3_result_error_nomem(ctx); goto jsonSetDone; }else if( x.nErr ){ goto jsonSetDone; }else if( pNode && (bApnd || bIsSet) ){ - pNode->jnFlags &= ~JNODE_JSON; - pNode->jnFlags |= jnFlags; + pNode->jnFlags |= JNODE_REPLACE; pNode->iVal = i+1; } } @@ -1396,7 +1363,7 @@ static void jsonTypeFunc( JsonNode *pNode; if( argc==2 ){ zPath = (const char*)sqlite3_value_text(argv[1]); - pNode = jsonLookup(&x, zPath, 0, ctx, 0); + pNode = jsonLookup(&x, zPath, 0, ctx); }else{ pNode = x.aNode; } @@ -1917,6 +1884,7 @@ int sqlite3_json_init( int flag; void (*xFunc)(sqlite3_context*,int,sqlite3_value**); } aFunc[] = { + { "json", 1, 0, jsonRemoveFunc }, { "json_array", -1, 0, jsonArrayFunc }, { "json_array_length", 1, 0, jsonArrayLengthFunc }, { "json_array_length", 2, 0, jsonArrayLengthFunc }, @@ -1934,7 +1902,6 @@ int sqlite3_json_init( /* DEBUG and TESTING functions */ { "json_parse", 1, 0, jsonParseFunc }, { "json_test1", 1, 0, jsonTest1Func }, - { "json_nodecount", 1, 0, jsonNodeCountFunc }, #endif }; #ifndef SQLITE_OMIT_VIRTUALTABLE diff --git a/manifest b/manifest index 285e65a2cd..525605fc0d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\sthe\ssqlite3_value_subtype()\sand\ssqlite3_result_subtype()\sinterfaces\navailable\sto\sloadable\sextensions. -D 2015-09-10T20:40:21.038 +C Take\sout\sthe\sgoofy\s'$$'\spath\ssyntax.\s\sInstead,\suse\ssubtypes\sto\scommunicate\swhen\na\sstring\sis\sJSON.\s\sAdd\sthe\sjson()\sfunction\sthat\svalidates\sand\sminifies\sthe\nJSON\sand\ssets\sthe\sappropriate\ssubtype. +D 2015-09-11T00:06:41.899 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -195,7 +195,7 @@ F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f F ext/misc/fuzzer.c 4c84635c71c26cfa7c2e5848cf49fe2d2cfcd767 F ext/misc/ieee754.c b0362167289170627659e84173f5d2e8fee8566e -F ext/misc/json1.c 4387d091c0ec0f4d9ed05560960f03d366db4fe0 +F ext/misc/json1.c 96490b8e34299a416ab221f827e0369344d95c53 F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c bcbee3c061b884eccb80e21651daaae8e1e43c63 F ext/misc/regexp.c af92cdaa5058fcec1451e49becc7ba44dba023dc @@ -814,7 +814,7 @@ F test/journal3.test ff8af941f9e06161d3db1b46bb9f965ff0e7f307 F test/jrnlmode.test 7864d59cf7f6e552b9b99ba0f38acd167edc10fa F test/jrnlmode2.test 81610545a4e6ed239ea8fa661891893385e23a1d F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa -F test/json101.test 11535d8986184500f4c30cc2f0b154b4ab05cc4e +F test/json101.test e20d2421c531db32fad59c5e06e80af0b1b002c8 F test/json102.test 12ef6d7d7c02c526fa3c2be1e933e7eb2a497cea F test/keyword1.test 37ef6bba5d2ed5b07ecdd6810571de2956599dff F test/lastinsert.test 42e948fd6442f07d60acbd15d33fb86473e0ef63 @@ -1386,7 +1386,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 7b5be299c617a3d3ed327ed30ef0a66a62c85b1f -R 3b51d146c21b59a26b4bd6730fa2e17a +P c6fca0be11f7414292279e2ea1b004260e6f4bb6 +R 6298ef57aadddbd507dac61cb4572ea0 U drh -Z 3c2729e75edd6da9e7ec99089fe2f222 +Z f00bae494bb0cd397ae4843df54988dc diff --git a/manifest.uuid b/manifest.uuid index d8c4b3eba0..0c45a9ed67 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c6fca0be11f7414292279e2ea1b004260e6f4bb6 \ No newline at end of file +8a80d6459e246ec1b38325e1cbd1e862157138b3 \ No newline at end of file diff --git a/test/json101.test b/test/json101.test index 1a84a5fc5e..9543ccfd24 100644 --- a/test/json101.test +++ b/test/json101.test @@ -16,9 +16,21 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl load_static_extension db json -do_execsql_test json1-1.1 { +do_execsql_test json1-1.1.00 { SELECT json_array(1,2.5,null,'hello'); } {[1,2.5,null,"hello"]} +do_execsql_test json1-1.1.01 { + SELECT json_array(1,'{"abc":2.5,"def":null,"ghi":hello}',99); + -- the second term goes in as a string: +} {[1,"{\\"abc\\":2.5,\\"def\\":null,\\"ghi\\":hello}",99]} +do_execsql_test json1-1.1.02 { + SELECT json_array(1,json('{"abc":2.5,"def":null,"ghi":"hello"}'),99); + -- the second term goes in as JSON +} {[1,{"abc":2.5,"def":null,"ghi":"hello"},99]} +do_execsql_test json1-1.1.03 { + SELECT json_array(1,json_object('abc',2.5,'def',null,'ghi','hello'),99); + -- the second term goes in as JSON +} {[1,{"abc":2.5,"def":null,"ghi":"hello"},99]} do_execsql_test json1-1.2 { SELECT hex(json_array('String "\ Test')); } {5B22537472696E67205C225C5C2054657374225D} @@ -54,13 +66,13 @@ do_execsql_test json1-3.1 { SELECT json_replace('{"a":1,"b":2}','$.a','[3,4,5]'); } {{{"a":"[3,4,5]","b":2}}} do_execsql_test json1-3.2 { - SELECT json_replace('{"a":1,"b":2}','$$.a','[3,4,5]'); + SELECT json_replace('{"a":1,"b":2}','$.a',json('[3,4,5]')); } {{{"a":[3,4,5],"b":2}}} do_execsql_test json1-3.3 { SELECT json_type(json_set('{"a":1,"b":2}','$.b','{"x":3,"y":4}'),'$.b'); } {text} do_execsql_test json1-3.4 { - SELECT json_type(json_set('{"a":1,"b":2}','$$.b','{"x":3,"y":4}'),'$.b'); + SELECT json_type(json_set('{"a":1,"b":2}','$.b',json('{"x":3,"y":4}')),'$.b'); } {object} # Per rfc7159, any JSON value is allowed at the top level, and whitespace From 12b3b895420b2f97a270555fb9c0d477d80b2a78 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 11 Sep 2015 01:22:41 +0000 Subject: [PATCH 082/100] Updates to the sqlite3_value_subtype() and sqlite3_result_subtype() documentation and to test cases for json1 dealing with those interfaces. FossilOrigin-Name: d6cadbe9fefce9a7af6b2d0cb83362f967d7d89a --- manifest | 15 +++--- manifest.uuid | 2 +- src/sqlite.h.in | 13 ++--- test/json102.test | 120 +++++++++++++++++++++++++++------------------- 4 files changed, 86 insertions(+), 64 deletions(-) diff --git a/manifest b/manifest index 3fc4b6ab33..253acb782d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\snew\sinterfaces\ssqlite3_value_subtype()\sand\ssqlite3_result_subtype().\nUpdate\sthe\sjson1.c\sextension\sto\stake\sadvantages\sof\sthose\sinterfaces\sto\savoid\nthe\sgoofy\s'$$'\spath\ssyntax\sand\sto\sallow\snested\scalls\sto\sjson_array()\sand\njson_object()\sthat\swork\sas\sexpected. -D 2015-09-11T00:26:04.863 +C Updates\sto\sthe\ssqlite3_value_subtype()\sand\ssqlite3_result_subtype()\ndocumentation\sand\sto\stest\scases\sfor\sjson1\sdealing\swith\sthose\sinterfaces. +D 2015-09-11T01:22:41.498 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -342,7 +342,7 @@ F src/resolve.c 3126f7694b8ce0f97282d7dd3a5198b8fa18dce9 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c c17613385bc6b095c421b1f30548814f5fd8a9b2 F src/shell.c 6332ef06db1390ef812cfdff1fc97b4fd76cdd42 -F src/sqlite.h.in 65ff1449e8c181da9ec752e8304e0a1efffaf47f +F src/sqlite.h.in dbaf8c3796e80221de4395b5f4f872abddb5f89f F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 64350bf36833a56ad675e27392a913f417c5c308 F src/sqliteInt.h 788dc0ea7ba32ec9fec06c628c1792d7b4753d86 @@ -815,7 +815,7 @@ F test/jrnlmode.test 7864d59cf7f6e552b9b99ba0f38acd167edc10fa F test/jrnlmode2.test 81610545a4e6ed239ea8fa661891893385e23a1d F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa F test/json101.test e20d2421c531db32fad59c5e06e80af0b1b002c8 -F test/json102.test 12ef6d7d7c02c526fa3c2be1e933e7eb2a497cea +F test/json102.test b5da30aef253102ad63f2186c9319e4b5d505a1b F test/keyword1.test 37ef6bba5d2ed5b07ecdd6810571de2956599dff F test/lastinsert.test 42e948fd6442f07d60acbd15d33fb86473e0ef63 F test/laststmtchanges.test ae613f53819206b3222771828d024154d51db200 @@ -1386,8 +1386,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f5580f08538636ffb1367e717a33756288ccddde 8a80d6459e246ec1b38325e1cbd1e862157138b3 -R 6298ef57aadddbd507dac61cb4572ea0 -T +closed 8a80d6459e246ec1b38325e1cbd1e862157138b3 +P db4152aef2253ed2a33e3cad01e0c6758e03f900 +R 1bba7f6a951d63c0429f61e7df8f5c7d U drh -Z 2fd3a23a70bcc4f4e537cdb3e485fd63 +Z 186760bb5fa5f9976b74dfde02dde033 diff --git a/manifest.uuid b/manifest.uuid index 1e2577e115..11d07fd29a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -db4152aef2253ed2a33e3cad01e0c6758e03f900 \ No newline at end of file +d6cadbe9fefce9a7af6b2d0cb83362f967d7d89a \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 65a0bedb05..d5b4f4e92d 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -4362,14 +4362,14 @@ int sqlite3_value_numeric_type(sqlite3_value*); ** METHOD: sqlite3_value ** ** The sqlite3_value_subtype(V) function returns the subtype for -** an application-defined SQL function argument V. The subtype +** an [application-defined SQL function] argument V. The subtype ** information can be used to pass a limited amount of context from ** one SQL function to another. Use the [sqlite3_result_subtype()] ** routine to set the subtype for the return value of an SQL function. ** ** SQLite makes no use of subtype itself. It merely passes the subtype -** from the result of one application-defined function to the input of -** another. +** from the result of one [application-defined SQL function] into the +** input of another. */ unsigned int sqlite3_value_subtype(sqlite3_value*); @@ -4678,9 +4678,10 @@ int sqlite3_result_zeroblob64(sqlite3_context*, sqlite3_uint64 n); ** METHOD: sqlite3_context ** ** The sqlite3_result_subtype(C,T) function causes the subtype of -** the result from the application-defined function with context C -** to be T. Only the lower 8 bits of the subtype T are preserved -** in current versions of SQLite; higher order bits are discarded. +** the result from the [application-defined SQL function] with +** [sqlite3_context] C to be the value T. Only the lower 8 bits +** of the subtype T are preserved in current versions of SQLite; +** higher order bits are discarded. ** The number of subtype bytes preserved by SQLite might increase ** in future releases of SQLite. */ diff --git a/test/json102.test b/test/json102.test index 7961627bc6..2c8dc49c57 100644 --- a/test/json102.test +++ b/test/json102.test @@ -20,125 +20,147 @@ source $testdir/tester.tcl load_static_extension db json do_execsql_test json102-100 { + SELECT json(' { "this" : "is", "a": [ "test" ] } '); +} {{{"this":"is","a":["test"]}}} +do_execsql_test json102-110 { SELECT json_array(1,2,'3',4); } {{[1,2,"3",4]}} -do_execsql_test json102-110 { +do_execsql_test json102-120 { SELECT json_array('[1,2]'); } {{["[1,2]"]}} -do_execsql_test json102-120 { +do_execsql_test json102-130 { + SELECT json_array(json_array(1,2)); +} {{[[1,2]]}} +do_execsql_test json102-140 { SELECT json_array(1,null,'3','[4,5]','{"six":7.7}'); } {{[1,null,"3","[4,5]","{\"six\":7.7}"]}} -do_execsql_test json102-130 { +do_execsql_test json102-150 { + SELECT json_array(1,null,'3',json('[4,5]'),json('{"six":7.7}')); +} {{[1,null,"3",[4,5],{"six":7.7}]}} +do_execsql_test json102-160 { SELECT json_array_length('[1,2,3,4]'); } {{4}} -do_execsql_test json102-140 { +do_execsql_test json102-170 { SELECT json_array_length('{"one":[1,2,3]}'); } {{0}} -do_execsql_test json102-150 { +do_execsql_test json102-180 { SELECT json_array_length('{"one":[1,2,3]}', '$.one'); } {{3}} -do_execsql_test json102-160 { +do_execsql_test json102-190 { SELECT json_extract('{"a":2,"c":[4,5,{"f":7}]}', '$'); } {{{"a":2,"c":[4,5,{"f":7}]}}} -do_execsql_test json102-170 { +do_execsql_test json102-200 { SELECT json_extract('{"a":2,"c":[4,5,{"f":7}]}', '$.c'); } {{[4,5,{"f":7}]}} -do_execsql_test json102-180 { +do_execsql_test json102-210 { SELECT json_extract('{"a":2,"c":[4,5,{"f":7}]}', '$.c[2]'); } {{{"f":7}}} -do_execsql_test json102-190 { +do_execsql_test json102-220 { SELECT json_extract('{"a":2,"c":[4,5,{"f":7}]}', '$.c[2].f'); } {{7}} -do_execsql_test json102-200 { +do_execsql_test json102-230 { SELECT json_extract('{"a":2,"c":[4,5],"f":7}','$.c','$.a'); } {{[[4,5],2]}} -do_execsql_test json102-210 { +do_execsql_test json102-240 { SELECT json_extract('{"a":2,"c":[4,5,{"f":7}]}', '$.x'); } {{}} -do_execsql_test json102-220 { +do_execsql_test json102-250 { SELECT json_insert('{"a":2,"c":4}', '$.a', 99); } {{{"a":2,"c":4}}} -do_execsql_test json102-230 { +do_execsql_test json102-260 { SELECT json_insert('{"a":2,"c":4}', '$.e', 99); } {{{"a":2,"c":4,"e":99}}} -do_execsql_test json102-240 { +do_execsql_test json102-270 { SELECT json_replace('{"a":2,"c":4}', '$.a', 99); } {{{"a":99,"c":4}}} -do_execsql_test json102-250 { +do_execsql_test json102-280 { SELECT json_replace('{"a":2,"c":4}', '$.e', 99); } {{{"a":2,"c":4}}} -do_execsql_test json102-260 { +do_execsql_test json102-290 { SELECT json_set('{"a":2,"c":4}', '$.a', 99); } {{{"a":99,"c":4}}} -do_execsql_test json102-270 { +do_execsql_test json102-300 { SELECT json_set('{"a":2,"c":4}', '$.e', 99); } {{{"a":2,"c":4,"e":99}}} -do_execsql_test json102-280 { +do_execsql_test json102-310 { + SELECT json_set('{"a":2,"c":4}', '$.c', '[97,96]'); +} {{{"a":2,"c":"[97,96]"}}} +do_execsql_test json102-320 { + SELECT json_set('{"a":2,"c":4}', '$.c', json('[97,96]')); +} {{{"a":2,"c":[97,96]}}} +do_execsql_test json102-330 { + SELECT json_set('{"a":2,"c":4}', '$.c', json_array(97,96)); +} {{{"a":2,"c":[97,96]}}} +do_execsql_test json102-340 { SELECT json_object('a',2,'c',4); } {{{"a":2,"c":4}}} -do_execsql_test json102-290 { +do_execsql_test json102-350 { SELECT json_object('a',2,'c','{e:5}'); } {{{"a":2,"c":"{e:5}"}}} -do_execsql_test json102-300 { +do_execsql_test json102-360 { + SELECT json_object('a',2,'c',json_object('e',5)); +} {{{"a":2,"c":{"e":5}}}} +do_execsql_test json102-370 { SELECT json_remove('[0,1,2,3,4]','$[2]'); } {{[0,1,3,4]}} -do_execsql_test json102-310 { +do_execsql_test json102-380 { SELECT json_remove('[0,1,2,3,4]','$[2]','$[0]'); } {{[1,3,4]}} -do_execsql_test json102-320 { +do_execsql_test json102-390 { SELECT json_remove('[0,1,2,3,4]','$[0]','$[2]'); } {{[1,2,4]}} -do_execsql_test json102-330 { +do_execsql_test json102-400 { SELECT json_remove('{"x":25,"y":42}'); } {{{"x":25,"y":42}}} -do_execsql_test json102-340 { +do_execsql_test json102-410 { SELECT json_remove('{"x":25,"y":42}','$.z'); } {{{"x":25,"y":42}}} -do_execsql_test json102-350 { +do_execsql_test json102-420 { SELECT json_remove('{"x":25,"y":42}','$.y'); } {{{"x":25}}} -do_execsql_test json102-360 { +do_execsql_test json102-430 { SELECT json_remove('{"x":25,"y":42}','$'); } {{}} -do_execsql_test json102-370 { +do_execsql_test json102-440 { SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}'); } {{object}} -do_execsql_test json102-380 { +do_execsql_test json102-450 { SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$'); } {{object}} -do_execsql_test json102-390 { +do_execsql_test json102-460 { SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a'); } {{array}} -do_execsql_test json102-400 { +do_execsql_test json102-470 { SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a[0]'); } {{integer}} -do_execsql_test json102-410 { +do_execsql_test json102-480 { SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a[1]'); } {{real}} -do_execsql_test json102-420 { +do_execsql_test json102-490 { SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a[2]'); } {{true}} -do_execsql_test json102-430 { +do_execsql_test json102-500 { SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a[3]'); } {{false}} -do_execsql_test json102-440 { +do_execsql_test json102-510 { SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a[4]'); } {{null}} -do_execsql_test json102-450 { +do_execsql_test json102-520 { SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a[5]'); } {{text}} -do_execsql_test json102-460 { +do_execsql_test json102-530 { SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a[6]'); } {{}} -do_execsql_test json102-470 { - SELECT json_valid('{"x":35}'); +do_execsql_test json102-540 { + SELECT json_valid(char(123)||'"x":35'||char(125)); } {{1}} -do_execsql_test json102-480 { - SELECT json_valid('{"x":35'); -- } +do_execsql_test json102-550 { + SELECT json_valid(char(123)||'"x":35'); } {{0}} + ifcapable vtab { -do_execsql_test json102-500 { +do_execsql_test json102-1000 { CREATE TABLE user(name,phone); INSERT INTO user(name,phone) VALUES ('Alice','["919-555-2345","804-555-3621"]'), @@ -151,13 +173,13 @@ do_execsql_test json102-500 { ORDER BY 1; } {Cindy Dave} -do_execsql_test json102-510 { +do_execsql_test json102-1010 { UPDATE user SET phone=json_extract(phone,'$[0]') WHERE json_array_length(phone)<2; SELECT name, substr(phone,1,5) FROM user ORDER BY name; } {Alice {["919} Bob 201-5 Cindy 704-5 Dave {["336}} -do_execsql_test json102-511 { +do_execsql_test json102-1011 { SELECT name FROM user WHERE phone LIKE '704-%' UNION SELECT user.name @@ -166,7 +188,7 @@ do_execsql_test json102-511 { AND json_each.value LIKE '704-%'; } {Cindy Dave} -do_execsql_test json102-600 { +do_execsql_test json102-1100 { CREATE TABLE big(json JSON); INSERT INTO big(json) VALUES('{ "id":123, @@ -203,32 +225,32 @@ set correct_answer [list \ 2 {$.stuff[2]} xyzzy \ 2 {$.partlist[0].uuid} 0 \ 2 {$.partlist[1].uuid} c690dc14-572e-11e5-95f9-dfc8861fd535] -do_execsql_test json102-610 { +do_execsql_test json102-1110 { SELECT big.rowid, fullkey, value FROM big, json_tree(big.json) WHERE json_tree.type NOT IN ('object','array') ORDER BY +big.rowid, +json_tree.id } $correct_answer -do_execsql_test json102-620 { +do_execsql_test json102-1120 { SELECT big.rowid, fullkey, atom FROM big, json_tree(big.json) WHERE atom IS NOT NULL ORDER BY +big.rowid, +json_tree.id } $correct_answer -do_execsql_test json102-630 { +do_execsql_test json102-1130 { SELECT DISTINCT json_extract(big.json,'$.id') FROM big, json_tree(big.json,'$.partlist') WHERE json_tree.key='uuid' AND json_tree.value='6fa5181e-5721-11e5-a04e-57f3d7b32808'; } {123} -do_execsql_test json102-631 { +do_execsql_test json102-1131 { SELECT DISTINCT json_extract(big.json,'$.id') FROM big, json_tree(big.json,'$') WHERE json_tree.key='uuid' AND json_tree.value='6fa5181e-5721-11e5-a04e-57f3d7b32808'; } {123} -do_execsql_test json102-632 { +do_execsql_test json102-1132 { SELECT DISTINCT json_extract(big.json,'$.id') FROM big, json_tree(big.json) WHERE json_tree.key='uuid' From bcb04b6128ac4ae577d69c70bb5e139dc22db716 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 11 Sep 2015 14:15:46 +0000 Subject: [PATCH 083/100] Fix some compiler warnings in fts5 code. FossilOrigin-Name: 0dc436116e55e3fd55eb6085ada71e099069b32d --- ext/fts5/fts5Int.h | 17 ----------------- ext/fts5/fts5_expr.c | 1 - ext/fts5/fts5_index.c | 35 +++++------------------------------ ext/fts5/fts5_main.c | 8 ++++---- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- 6 files changed, 20 insertions(+), 63 deletions(-) diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index c7f724eab3..6f6f4ed784 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -387,12 +387,6 @@ int sqlite3Fts5IndexSync(Fts5Index *p, int bCommit); */ int sqlite3Fts5IndexRollback(Fts5Index *p); -/* -** Retrieve and clear the current error code, respectively. -*/ -int sqlite3Fts5IndexErrcode(Fts5Index*); -void sqlite3Fts5IndexReset(Fts5Index*); - /* ** Get or set the "averages" values. */ @@ -678,17 +672,6 @@ int sqlite3Fts5TokenizerInit(fts5_api*); ** End of interface to code in fts5_tokenizer.c. **************************************************************************/ -/************************************************************************** -** Interface to code in fts5_sorter.c. -*/ -typedef struct Fts5Sorter Fts5Sorter; - -int sqlite3Fts5SorterNew(Fts5Expr *pExpr, Fts5Sorter **pp); - -/* -** End of interface to code in fts5_sorter.c. -**************************************************************************/ - /************************************************************************** ** Interface to code in fts5_vocab.c. */ diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index 4cfc1b155c..559f0db82b 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -947,7 +947,6 @@ static int fts5ExprNearNextMatch( for(j=0; jnTerm; j++){ Fts5ExprTerm *pTerm = &pPhrase->aTerm[j]; if( pTerm->pSynonym ){ - int bEof = 1; i64 iRowid = fts5ExprSynonymRowid(pTerm, bDesc, 0); if( iRowid==iLast ) continue; bMatch = 0; diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index cd3402418c..f499d0c389 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -737,7 +737,6 @@ static void fts5DataWrite(Fts5Index *p, i64 iRowid, const u8 *pData, int nData){ if( p->rc!=SQLITE_OK ) return; if( p->pWriter==0 ){ - int rc = SQLITE_OK; Fts5Config *pConfig = p->pConfig; fts5IndexPrepareStmt(p, &p->pWriter, sqlite3_mprintf( "REPLACE INTO '%q'.'%q_data'(id, block) VALUES(?,?)", @@ -973,7 +972,6 @@ static Fts5Structure *fts5StructureRead(Fts5Index *p){ Fts5Structure *pRet = 0; /* Object to return */ int iCookie; /* Configuration cookie */ Fts5Data *pData; - Fts5Buffer buf = {0, 0, 0}; pData = fts5DataRead(p, FTS5_STRUCTURE_ROWID); if( p->rc ) return 0; @@ -1417,11 +1415,6 @@ static int fts5DlidxIterPgno(Fts5DlidxIter *pIter){ return pIter->aLvl[0].iLeafPgno; } -static void fts5LeafHeader(Fts5Data *pLeaf, int *piRowid, int *piTerm){ - *piRowid = (int)fts5GetU16(&pLeaf->p[0]); - *piTerm = (int)fts5GetU16(&pLeaf->p[2]); -} - /* ** Load the next leaf page into the segment iterator. */ @@ -1827,7 +1820,6 @@ static void fts5SegIterNext( fts5DataRelease(pIter->pLeaf); pIter->pLeaf = 0; }else{ - int nExtra; fts5SegIterLoadTerm(p, pIter, nKeep); fts5SegIterLoadNPos(p, pIter); if( pbNewTerm ) *pbNewTerm = 1; @@ -1857,12 +1849,11 @@ static void fts5SegIterReverse(Fts5Index *p, Fts5SegIter *pIter){ pgnoLast = fts5DlidxIterPgno(pDlidx); pLast = fts5DataRead(p, FTS5_SEGMENT_ROWID(iSegid, pgnoLast)); }else{ - int iOff; /* Byte offset within pLeaf */ Fts5Data *pLeaf = pIter->pLeaf; /* Current leaf data */ - /* Currently, Fts5SegIter.iLeafOffset (and iOff) points to the first - ** byte of position-list content for the current rowid. Back it up - ** so that it points to the start of the position-list size field. */ + /* Currently, Fts5SegIter.iLeafOffset points to the first byte of + ** position-list content for the current rowid. Back it up so that it + ** points to the start of the position-list size field. */ pIter->iLeafOffset -= sqlite3Fts5GetVarintLen(pIter->nPos*2+pIter->bDel); /* If this condition is true then the largest rowid for the current @@ -1988,7 +1979,6 @@ static void fts5LeafSeek( int nMatch = 0; int nKeep = 0; int nNew = 0; - int iTerm = 0; int iTermOff; int iPgidx; /* Current offset in pgidx */ int bEndOfPage = 0; @@ -2048,7 +2038,6 @@ static void fts5LeafSeek( return; }else if( bEndOfPage ){ do { - iTerm = 0; fts5SegIterNextPage(p, pIter); if( pIter->pLeaf==0 ) return; a = pIter->pLeaf->p; @@ -3314,10 +3303,6 @@ static void fts5WriteAppendPoslistData( } } -static void fts5WriteAppendZerobyte(Fts5Index *p, Fts5SegWriter *pWriter){ - fts5BufferAppendVarint(&p->rc, &pWriter->writer.buf, 0); -} - /* ** Flush any data cached by the writer object to the database. Free any ** allocations associated with the writer. @@ -3480,7 +3465,6 @@ static void fts5IndexMergeLevel( Fts5SegWriter writer; /* Writer object */ Fts5StructureSegment *pSeg; /* Output segment */ Fts5Buffer term; - int bRequireDoclistTerm = 0; /* Doclist terminator (0x00) required */ int bOldest; /* True if the output segment is the oldest */ assert( iLvlnLevel ); @@ -3545,13 +3529,8 @@ static void fts5IndexMergeLevel( } /* This is a new term. Append a term to the output segment. */ - /* TODO2: Doclist 0x00 term */ - if( bRequireDoclistTerm ){ - /* fts5WriteAppendZerobyte(p, &writer); */ - } fts5WriteAppendTerm(p, &writer, nTerm, pTerm); fts5BufferSet(&p->rc, &term, nTerm, pTerm); - bRequireDoclistTerm = 1; } /* Append the rowid to the output */ @@ -3758,7 +3737,6 @@ static void fts5FlushOneHash(Fts5Index *p){ Fts5StructureSegment *pSeg; /* New segment within pStruct */ Fts5Buffer *pBuf; /* Buffer in which to assemble leaf page */ Fts5Buffer *pPgidx; /* Buffer in which to assemble pgidx */ - const u8 *zPrev = 0; Fts5SegWriter writer; fts5WriteInit(p, &writer, iSegid); @@ -3780,11 +3758,10 @@ static void fts5FlushOneHash(Fts5Index *p){ const char *zTerm; /* Buffer containing term */ const u8 *pDoclist; /* Pointer to doclist for this term */ int nDoclist; /* Size of doclist in bytes */ - int nSuffix; /* Size of term suffix */ /* Write the term for this entry to disk. */ sqlite3Fts5HashScanEntry(pHash, &zTerm, &pDoclist, &nDoclist); - fts5WriteAppendTerm(p, &writer, strlen(zTerm), zTerm); + fts5WriteAppendTerm(p, &writer, strlen(zTerm), (const u8*)zTerm); if( writer.bFirstRowidInPage==0 && pgsz>=(pBuf->n + pPgidx->n + nDoclist + 1) @@ -3854,7 +3831,6 @@ static void fts5FlushOneHash(Fts5Index *p){ /* TODO2: Doclist terminator written here. */ /* pBuf->p[pBuf->n++] = '\0'; */ assert( pBuf->n<=pBuf->nSpace ); - zPrev = (const u8*)zTerm; sqlite3Fts5HashScanNext(pHash); } sqlite3Fts5HashClear(pHash); @@ -4888,7 +4864,6 @@ static void fts5IndexIntegrityCheckEmpty( } static void fts5IntegrityCheckPgidx(Fts5Index *p, Fts5Data *pLeaf){ - int nPg = (pLeaf->nn - pLeaf->szLeaf) / 2; int iTermOff = 0; int ii; @@ -5476,7 +5451,7 @@ static void fts5RowidFunction( zArg = (const char*)sqlite3_value_text(apVal[0]); if( 0==sqlite3_stricmp(zArg, "segment") ){ i64 iRowid; - int segid, height, pgno; + int segid, pgno; if( nArg!=3 ){ sqlite3_result_error(pCtx, "should be: fts5_rowid('segment', segid, pgno))", -1 diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index 2fd633bd28..5590816bc2 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -25,11 +25,11 @@ int sqlite3_fts5_may_be_corrupt = 1; -typedef struct Fts5Table Fts5Table; -typedef struct Fts5Cursor Fts5Cursor; -typedef struct Fts5Auxiliary Fts5Auxiliary; typedef struct Fts5Auxdata Fts5Auxdata; - +typedef struct Fts5Auxiliary Fts5Auxiliary; +typedef struct Fts5Cursor Fts5Cursor; +typedef struct Fts5Sorter Fts5Sorter; +typedef struct Fts5Table Fts5Table; typedef struct Fts5TokenizerModule Fts5TokenizerModule; /* diff --git a/manifest b/manifest index 253acb782d..5d0cbe90ef 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Updates\sto\sthe\ssqlite3_value_subtype()\sand\ssqlite3_result_subtype()\ndocumentation\sand\sto\stest\scases\sfor\sjson1\sdealing\swith\sthose\sinterfaces. -D 2015-09-11T01:22:41.498 +C Fix\ssome\scompiler\swarnings\sin\sfts5\scode. +D 2015-09-11T14:15:46.470 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -106,14 +106,14 @@ F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0 F ext/fts5/fts5.h f04659e0df5af83731b102189a32280f74f4a6bc -F ext/fts5/fts5Int.h 81ba5e474979b166a52a8be306aa3b09d43a10e9 +F ext/fts5/fts5Int.h 666aba8432940a8449a3bd4636e898fe906ed95d F ext/fts5/fts5_aux.c 7a307760a9c57c750d043188ec0bad59f5b5ec7e F ext/fts5/fts5_buffer.c 64dcaf36a3ebda9e84b7c3b8788887ec325e12a4 F ext/fts5/fts5_config.c 57ee5fe71578cb494574fc0e6e51acb9a22a8695 -F ext/fts5/fts5_expr.c a7726fe7045eec7caca8a074af747c8ea3545b83 +F ext/fts5/fts5_expr.c 667faaf14a69a5683ac383acdc8d942cf32c3f93 F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246 -F ext/fts5/fts5_index.c 093e2e5936dab536cbe3e321bf4b53dda2b40547 -F ext/fts5/fts5_main.c 4b04c934084ea24a858438a04b5be8af3a9e0311 +F ext/fts5/fts5_index.c c07522cc5632d0d211402c0e6273ecb7493886d4 +F ext/fts5/fts5_main.c 3fa906f6c0177caf8f82862bc70f37b28bb3305c F ext/fts5/fts5_storage.c 120f7b143688b5b7710dacbd48cff211609b8059 F ext/fts5/fts5_tcl.c 6da58d6e8f42a93c4486b5ba9b187a7f995dee37 F ext/fts5/fts5_test_mi.c e96be827aa8f571031e65e481251dc1981d608bf @@ -1386,7 +1386,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P db4152aef2253ed2a33e3cad01e0c6758e03f900 -R 1bba7f6a951d63c0429f61e7df8f5c7d -U drh -Z 186760bb5fa5f9976b74dfde02dde033 +P d6cadbe9fefce9a7af6b2d0cb83362f967d7d89a +R 142f8055af436e9c39db41b2b1d79675 +U dan +Z 56f93ac2844adc1c45e33937963b4c3d diff --git a/manifest.uuid b/manifest.uuid index 11d07fd29a..ecea4d71e9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d6cadbe9fefce9a7af6b2d0cb83362f967d7d89a \ No newline at end of file +0dc436116e55e3fd55eb6085ada71e099069b32d \ No newline at end of file From e98b6fab47a954c822126a0354e6f824c24446f7 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 11 Sep 2015 15:32:33 +0000 Subject: [PATCH 084/100] More test cases in test/json102.test corresponding to new examples in the json1 documentation. FossilOrigin-Name: f599a42e190b4b89d74055402143c5487985cd90 --- manifest | 14 ++--- manifest.uuid | 2 +- test/json102.test | 130 ++++++++++++++++++++++++++-------------------- 3 files changed, 83 insertions(+), 63 deletions(-) diff --git a/manifest b/manifest index 5d0cbe90ef..8b1f777811 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\ssome\scompiler\swarnings\sin\sfts5\scode. -D 2015-09-11T14:15:46.470 +C More\stest\scases\sin\stest/json102.test\scorresponding\sto\snew\sexamples\sin\sthe\njson1\sdocumentation. +D 2015-09-11T15:32:33.147 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -815,7 +815,7 @@ F test/jrnlmode.test 7864d59cf7f6e552b9b99ba0f38acd167edc10fa F test/jrnlmode2.test 81610545a4e6ed239ea8fa661891893385e23a1d F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa F test/json101.test e20d2421c531db32fad59c5e06e80af0b1b002c8 -F test/json102.test b5da30aef253102ad63f2186c9319e4b5d505a1b +F test/json102.test 4e1403cb06481ab160cf471c3c139820498e0563 F test/keyword1.test 37ef6bba5d2ed5b07ecdd6810571de2956599dff F test/lastinsert.test 42e948fd6442f07d60acbd15d33fb86473e0ef63 F test/laststmtchanges.test ae613f53819206b3222771828d024154d51db200 @@ -1386,7 +1386,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P d6cadbe9fefce9a7af6b2d0cb83362f967d7d89a -R 142f8055af436e9c39db41b2b1d79675 -U dan -Z 56f93ac2844adc1c45e33937963b4c3d +P 0dc436116e55e3fd55eb6085ada71e099069b32d +R 6a1dee4bd5e0067c0115e5b47549c01a +U drh +Z 3b3b3aa659d24a2dd31ae86f119af47f diff --git a/manifest.uuid b/manifest.uuid index ecea4d71e9..a402b58010 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0dc436116e55e3fd55eb6085ada71e099069b32d \ No newline at end of file +f599a42e190b4b89d74055402143c5487985cd90 \ No newline at end of file diff --git a/test/json102.test b/test/json102.test index 2c8dc49c57..da9fbd1b76 100644 --- a/test/json102.test +++ b/test/json102.test @@ -20,145 +20,165 @@ source $testdir/tester.tcl load_static_extension db json do_execsql_test json102-100 { + SELECT json_object('ex','[52,3.14159]'); +} {{{"ex":"[52,3.14159]"}}} +do_execsql_test json102-110 { + SELECT json_object('ex',json('[52,3.14159]')); +} {{{"ex":[52,3.14159]}}} +do_execsql_test json102-120 { + SELECT json_object('ex',json_array(52,3.14159)); +} {{{"ex":[52,3.14159]}}} +do_execsql_test json102-130 { SELECT json(' { "this" : "is", "a": [ "test" ] } '); } {{{"this":"is","a":["test"]}}} -do_execsql_test json102-110 { +do_execsql_test json102-140 { SELECT json_array(1,2,'3',4); } {{[1,2,"3",4]}} -do_execsql_test json102-120 { +do_execsql_test json102-150 { SELECT json_array('[1,2]'); } {{["[1,2]"]}} -do_execsql_test json102-130 { +do_execsql_test json102-160 { SELECT json_array(json_array(1,2)); } {{[[1,2]]}} -do_execsql_test json102-140 { +do_execsql_test json102-170 { SELECT json_array(1,null,'3','[4,5]','{"six":7.7}'); } {{[1,null,"3","[4,5]","{\"six\":7.7}"]}} -do_execsql_test json102-150 { +do_execsql_test json102-180 { SELECT json_array(1,null,'3',json('[4,5]'),json('{"six":7.7}')); } {{[1,null,"3",[4,5],{"six":7.7}]}} -do_execsql_test json102-160 { +do_execsql_test json102-190 { SELECT json_array_length('[1,2,3,4]'); } {{4}} -do_execsql_test json102-170 { +do_execsql_test json102-200 { + SELECT json_array_length('[1,2,3,4]', '$'); +} {{4}} +do_execsql_test json102-210 { + SELECT json_array_length('[1,2,3,4]', '$[2]'); +} {{0}} +do_execsql_test json102-220 { SELECT json_array_length('{"one":[1,2,3]}'); } {{0}} -do_execsql_test json102-180 { +do_execsql_test json102-230 { SELECT json_array_length('{"one":[1,2,3]}', '$.one'); } {{3}} -do_execsql_test json102-190 { - SELECT json_extract('{"a":2,"c":[4,5,{"f":7}]}', '$'); -} {{{"a":2,"c":[4,5,{"f":7}]}}} -do_execsql_test json102-200 { - SELECT json_extract('{"a":2,"c":[4,5,{"f":7}]}', '$.c'); -} {{[4,5,{"f":7}]}} -do_execsql_test json102-210 { - SELECT json_extract('{"a":2,"c":[4,5,{"f":7}]}', '$.c[2]'); -} {{{"f":7}}} -do_execsql_test json102-220 { - SELECT json_extract('{"a":2,"c":[4,5,{"f":7}]}', '$.c[2].f'); -} {{7}} -do_execsql_test json102-230 { - SELECT json_extract('{"a":2,"c":[4,5],"f":7}','$.c','$.a'); -} {{[[4,5],2]}} do_execsql_test json102-240 { - SELECT json_extract('{"a":2,"c":[4,5,{"f":7}]}', '$.x'); + SELECT json_array_length('{"one":[1,2,3]}', '$.two'); } {{}} do_execsql_test json102-250 { + SELECT json_extract('{"a":2,"c":[4,5,{"f":7}]}', '$'); +} {{{"a":2,"c":[4,5,{"f":7}]}}} +do_execsql_test json102-260 { + SELECT json_extract('{"a":2,"c":[4,5,{"f":7}]}', '$.c'); +} {{[4,5,{"f":7}]}} +do_execsql_test json102-270 { + SELECT json_extract('{"a":2,"c":[4,5,{"f":7}]}', '$.c[2]'); +} {{{"f":7}}} +do_execsql_test json102-280 { + SELECT json_extract('{"a":2,"c":[4,5,{"f":7}]}', '$.c[2].f'); +} {{7}} +do_execsql_test json102-290 { + SELECT json_extract('{"a":2,"c":[4,5],"f":7}','$.c','$.a'); +} {{[[4,5],2]}} +do_execsql_test json102-300 { + SELECT json_extract('{"a":2,"c":[4,5,{"f":7}]}', '$.x'); +} {{}} +do_execsql_test json102-310 { + SELECT json_extract('{"a":2,"c":[4,5,{"f":7}]}', '$.x', '$.a'); +} {{[null,2]}} +do_execsql_test json102-320 { SELECT json_insert('{"a":2,"c":4}', '$.a', 99); } {{{"a":2,"c":4}}} -do_execsql_test json102-260 { +do_execsql_test json102-330 { SELECT json_insert('{"a":2,"c":4}', '$.e', 99); } {{{"a":2,"c":4,"e":99}}} -do_execsql_test json102-270 { +do_execsql_test json102-340 { SELECT json_replace('{"a":2,"c":4}', '$.a', 99); } {{{"a":99,"c":4}}} -do_execsql_test json102-280 { +do_execsql_test json102-350 { SELECT json_replace('{"a":2,"c":4}', '$.e', 99); } {{{"a":2,"c":4}}} -do_execsql_test json102-290 { +do_execsql_test json102-360 { SELECT json_set('{"a":2,"c":4}', '$.a', 99); } {{{"a":99,"c":4}}} -do_execsql_test json102-300 { +do_execsql_test json102-370 { SELECT json_set('{"a":2,"c":4}', '$.e', 99); } {{{"a":2,"c":4,"e":99}}} -do_execsql_test json102-310 { +do_execsql_test json102-380 { SELECT json_set('{"a":2,"c":4}', '$.c', '[97,96]'); } {{{"a":2,"c":"[97,96]"}}} -do_execsql_test json102-320 { +do_execsql_test json102-390 { SELECT json_set('{"a":2,"c":4}', '$.c', json('[97,96]')); } {{{"a":2,"c":[97,96]}}} -do_execsql_test json102-330 { +do_execsql_test json102-400 { SELECT json_set('{"a":2,"c":4}', '$.c', json_array(97,96)); } {{{"a":2,"c":[97,96]}}} -do_execsql_test json102-340 { +do_execsql_test json102-410 { SELECT json_object('a',2,'c',4); } {{{"a":2,"c":4}}} -do_execsql_test json102-350 { +do_execsql_test json102-420 { SELECT json_object('a',2,'c','{e:5}'); } {{{"a":2,"c":"{e:5}"}}} -do_execsql_test json102-360 { +do_execsql_test json102-430 { SELECT json_object('a',2,'c',json_object('e',5)); } {{{"a":2,"c":{"e":5}}}} -do_execsql_test json102-370 { +do_execsql_test json102-440 { SELECT json_remove('[0,1,2,3,4]','$[2]'); } {{[0,1,3,4]}} -do_execsql_test json102-380 { +do_execsql_test json102-450 { SELECT json_remove('[0,1,2,3,4]','$[2]','$[0]'); } {{[1,3,4]}} -do_execsql_test json102-390 { +do_execsql_test json102-460 { SELECT json_remove('[0,1,2,3,4]','$[0]','$[2]'); } {{[1,2,4]}} -do_execsql_test json102-400 { +do_execsql_test json102-470 { SELECT json_remove('{"x":25,"y":42}'); } {{{"x":25,"y":42}}} -do_execsql_test json102-410 { +do_execsql_test json102-480 { SELECT json_remove('{"x":25,"y":42}','$.z'); } {{{"x":25,"y":42}}} -do_execsql_test json102-420 { +do_execsql_test json102-490 { SELECT json_remove('{"x":25,"y":42}','$.y'); } {{{"x":25}}} -do_execsql_test json102-430 { +do_execsql_test json102-500 { SELECT json_remove('{"x":25,"y":42}','$'); } {{}} -do_execsql_test json102-440 { +do_execsql_test json102-510 { SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}'); } {{object}} -do_execsql_test json102-450 { +do_execsql_test json102-520 { SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$'); } {{object}} -do_execsql_test json102-460 { +do_execsql_test json102-530 { SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a'); } {{array}} -do_execsql_test json102-470 { +do_execsql_test json102-540 { SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a[0]'); } {{integer}} -do_execsql_test json102-480 { +do_execsql_test json102-550 { SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a[1]'); } {{real}} -do_execsql_test json102-490 { +do_execsql_test json102-560 { SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a[2]'); } {{true}} -do_execsql_test json102-500 { +do_execsql_test json102-570 { SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a[3]'); } {{false}} -do_execsql_test json102-510 { +do_execsql_test json102-580 { SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a[4]'); } {{null}} -do_execsql_test json102-520 { +do_execsql_test json102-590 { SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a[5]'); } {{text}} -do_execsql_test json102-530 { +do_execsql_test json102-600 { SELECT json_type('{"a":[2,3.5,true,false,null,"x"]}','$.a[6]'); } {{}} -do_execsql_test json102-540 { +do_execsql_test json102-610 { SELECT json_valid(char(123)||'"x":35'||char(125)); } {{1}} -do_execsql_test json102-550 { +do_execsql_test json102-620 { SELECT json_valid(char(123)||'"x":35'); } {{0}} - ifcapable vtab { do_execsql_test json102-1000 { CREATE TABLE user(name,phone); From 16a93128f8a9d806bd08f60dfc35ef6292c62f79 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Fri, 11 Sep 2015 18:05:01 +0000 Subject: [PATCH 085/100] Fix harmless compiler warnings. FossilOrigin-Name: bfc7b84b766860d2e410702ba7c1166d7328309a --- ext/fts5/fts5_index.c | 2 +- ext/misc/json1.c | 18 ++++++++++-------- manifest | 16 ++++++++-------- manifest.uuid | 2 +- 4 files changed, 20 insertions(+), 18 deletions(-) diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index f499d0c389..e36a5388fd 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -5275,7 +5275,7 @@ static int fts5DecodePoslist(int *pRc, Fts5Buffer *pBuf, const u8 *a, int n){ ** The return value is the number of bytes read from the input buffer. */ static int fts5DecodeDoclist(int *pRc, Fts5Buffer *pBuf, const u8 *a, int n){ - i64 iDocid; + i64 iDocid = 0; int iOff = 0; if( n>0 ){ diff --git a/ext/misc/json1.c b/ext/misc/json1.c index a217ce80b8..ca6021733c 100644 --- a/ext/misc/json1.c +++ b/ext/misc/json1.c @@ -495,12 +495,12 @@ static void jsonReturn( } if( v==0 ) break; if( v<=0x7f ){ - zOut[j++] = v; + zOut[j++] = (char)v; }else if( v<=0x7ff ){ - zOut[j++] = 0xc0 | (v>>6); + zOut[j++] = (char)(0xc0 | (v>>6)); zOut[j++] = 0x80 | (v&0x3f); }else{ - zOut[j++] = 0xe0 | (v>>12); + zOut[j++] = (char)(0xe0 | (v>>12)); zOut[j++] = 0x80 | ((v>>6)&0x3f); zOut[j++] = 0x80 | (v&0x3f); } @@ -1053,6 +1053,7 @@ static void jsonTest1Func( int argc, sqlite3_value **argv ){ + UNUSED_PARAM(argc); sqlite3_result_int(ctx, sqlite3_value_subtype(argv[0])==JSON_SUBTYPE); } #endif /* SQLITE_DEBUG */ @@ -1274,8 +1275,8 @@ static void jsonReplaceFunc( pNode = jsonLookup(&x, zPath, 0, ctx); if( x.nErr ) goto replace_err; if( pNode ){ - pNode->jnFlags |= JNODE_REPLACE; - pNode->iVal = i+1; + pNode->jnFlags |= (u8)JNODE_REPLACE; + pNode->iVal = (u8)(i+1); } } if( x.aNode[0].jnFlags & JNODE_REPLACE ){ @@ -1329,8 +1330,8 @@ static void jsonSetFunc( }else if( x.nErr ){ goto jsonSetDone; }else if( pNode && (bApnd || bIsSet) ){ - pNode->jnFlags |= JNODE_REPLACE; - pNode->iVal = i+1; + pNode->jnFlags |= (u8)JNODE_REPLACE; + pNode->iVal = (u8)(i+1); } } if( x.aNode[0].jnFlags & JNODE_REPLACE ){ @@ -1388,6 +1389,7 @@ static void jsonValidFunc( JsonParse x; /* The parse */ int rc = 0; + UNUSED_PARAM(argc); if( jsonParse(&x, 0, (const char*)sqlite3_value_text(argv[0]))==0 && x.nNode>0 ){ @@ -1734,7 +1736,7 @@ static int jsonEachFilter( ){ JsonEachCursor *p = (JsonEachCursor*)cur; const char *z; - const char *zRoot; + const char *zRoot = 0; sqlite3_int64 n; UNUSED_PARAM(idxStr); diff --git a/manifest b/manifest index 8b1f777811..829385f366 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C More\stest\scases\sin\stest/json102.test\scorresponding\sto\snew\sexamples\sin\sthe\njson1\sdocumentation. -D 2015-09-11T15:32:33.147 +C Fix\sharmless\scompiler\swarnings. +D 2015-09-11T18:05:01.352 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -112,7 +112,7 @@ F ext/fts5/fts5_buffer.c 64dcaf36a3ebda9e84b7c3b8788887ec325e12a4 F ext/fts5/fts5_config.c 57ee5fe71578cb494574fc0e6e51acb9a22a8695 F ext/fts5/fts5_expr.c 667faaf14a69a5683ac383acdc8d942cf32c3f93 F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246 -F ext/fts5/fts5_index.c c07522cc5632d0d211402c0e6273ecb7493886d4 +F ext/fts5/fts5_index.c 62a682a70ea2e84fa67c7786ead892b201116ad1 F ext/fts5/fts5_main.c 3fa906f6c0177caf8f82862bc70f37b28bb3305c F ext/fts5/fts5_storage.c 120f7b143688b5b7710dacbd48cff211609b8059 F ext/fts5/fts5_tcl.c 6da58d6e8f42a93c4486b5ba9b187a7f995dee37 @@ -195,7 +195,7 @@ F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f F ext/misc/fuzzer.c 4c84635c71c26cfa7c2e5848cf49fe2d2cfcd767 F ext/misc/ieee754.c b0362167289170627659e84173f5d2e8fee8566e -F ext/misc/json1.c 96490b8e34299a416ab221f827e0369344d95c53 +F ext/misc/json1.c f35d00fbd79a7e23af18d7630a2fcf22dce3692b F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c bcbee3c061b884eccb80e21651daaae8e1e43c63 F ext/misc/regexp.c af92cdaa5058fcec1451e49becc7ba44dba023dc @@ -1386,7 +1386,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 0dc436116e55e3fd55eb6085ada71e099069b32d -R 6a1dee4bd5e0067c0115e5b47549c01a -U drh -Z 3b3b3aa659d24a2dd31ae86f119af47f +P f599a42e190b4b89d74055402143c5487985cd90 +R 0604cd6b1b67b982add6999f3d35855f +U mistachkin +Z 360c245eb92b0ddf2395cd1ae9027341 diff --git a/manifest.uuid b/manifest.uuid index a402b58010..cb73a0c534 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f599a42e190b4b89d74055402143c5487985cd90 \ No newline at end of file +bfc7b84b766860d2e410702ba7c1166d7328309a \ No newline at end of file From b3366b99c8c0077b1bf59081d9c2b3111520dc56 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 11 Sep 2015 20:54:44 +0000 Subject: [PATCH 086/100] Update evidence marks due to wording changes in requirements text. No changes to code. FossilOrigin-Name: 86781093bdb4c4fdedd228cb1c8961db48a483bb --- manifest | 34 +++++++++++++++++----------------- manifest.uuid | 2 +- test/e_createtable.test | 15 +++++++-------- test/e_delete.test | 13 ++++++++----- test/e_expr.test | 8 ++++---- test/e_insert.test | 16 ++++++++-------- test/e_reindex.test | 7 +++---- test/e_resolve.test | 12 ++++++------ test/e_update.test | 16 ++++++++-------- test/fkey5.test | 4 ++-- test/indexedby.test | 2 +- test/pragma.test | 27 +++++++++++++-------------- test/pragma2.test | 2 +- 13 files changed, 79 insertions(+), 79 deletions(-) diff --git a/manifest b/manifest index 829385f366..2f7902f582 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarnings. -D 2015-09-11T18:05:01.352 +C Update\sevidence\smarks\sdue\sto\swording\schanges\sin\srequirements\stext.\s\sNo\nchanges\sto\scode. +D 2015-09-11T20:54:44.879 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -583,20 +583,20 @@ F test/e_blobclose.test df756753f571bc30e42e3a6cba2807576e49e716 F test/e_blobopen.test 234f960d90235a9b51ec3ca1e062e8541dd558d8 F test/e_blobwrite.test 615b405f29feb2cfb5a1f03dab7933258294fa26 F test/e_changes.test fd66105385153dbf21fdb35eb8ef6c3e1eade579 -F test/e_createtable.test c7e67b49e6cf92473c8fb30ab26143e9e2128cf7 -F test/e_delete.test d5186e2f5478b659f16a2c8b66c09892823e542a +F test/e_createtable.test d4c6059d44dcd4b636de9aae322766062b471844 +F test/e_delete.test ab39084f26ae1f033c940b70ebdbbd523dc4962e F test/e_droptrigger.test 3cd080807622c13e5bbb61fc9a57bd7754da2412 F test/e_dropview.test 0c9f7f60989164a70a67a9d9c26d1083bc808306 -F test/e_expr.test 8f5fdd7261e2d746813b0c6a1c0e34824ad3c5ad +F test/e_expr.test 03a84a6fa9bd3472112d6bd4599f5269f5f74803 F test/e_fkey.test a1783fe1f759e1990e6a11adfcf0702dac4d0707 F test/e_fts3.test 5c02288842e4f941896fd44afdef564dd5fc1459 -F test/e_insert.test 0e63edc037afe738bb81a626a676811ed7862c90 -F test/e_reindex.test 57d439f6c644befc8274ac93cf2f5449cf2736c1 -F test/e_resolve.test dcce9308fb13b934ce29591105d031d3e14fbba6 +F test/e_insert.test 3de217e95094d3d165992a6de1164bbc4bd92dc7 +F test/e_reindex.test 2bebf7b393e519198b7c654407221cf171a439b8 +F test/e_resolve.test a61751c368b109db73df0f20fc75fb47e166b1d8 F test/e_select.test 52692ff3849541e828ad4661fe3773a9b8711763 F test/e_select2.test aceb80ab927d46fba5ce7586ebabf23e2bb0604f F test/e_totalchanges.test b12ee5809d3e63aeb83238dd501a7bca7fd72c10 -F test/e_update.test 312cb8f5ccfe41515a6bb092f8ea562a9bd54d52 +F test/e_update.test f46c2554d915c9197548681e8d8c33a267e84528 F test/e_uri.test 5ae33760fb2039c61aa2d90886f1664664173585 F test/e_vacuum.test 5bfbdc21b65c0abf24398d0ba31dc88d93ca77a9 F test/e_wal.test ae9a593207a77d711443ee69ffe081fda9243625 @@ -624,7 +624,7 @@ F test/fkey1.test de5b287f6a480b36bd51e8debcf48168e26e4ed2 F test/fkey2.test f3d27ecba480a348c328965d154214719bb158a9 F test/fkey3.test 76d475c80b84ee7a5d062e56ccb6ea68882e2b49 F test/fkey4.test 86446017011273aad8f9a99c1a65019e7bd9ca9d -F test/fkey5.test 56bcb5a6e8b725b17febc267fb041a6695e86853 +F test/fkey5.test 5a373303f201ac03c22ba1ef17a733d3f56e611a F test/fkey6.test abb59f866c1b44926fd02d1fdd217d831fe04f48 F test/fkey7.test 72e915890ee4a005daaf3002cb208e8fe973ac13 F test/fkey8.test 8f08203458321e6c19a263829de4cfc936274ab0 @@ -782,7 +782,7 @@ F test/index4.test ab92e736d5946840236cd61ac3191f91a7856bf6 F test/index5.test 8621491915800ec274609e42e02a97d67e9b13e7 F test/index6.test 7102ec371414c42dfb1d5ca37eb4519aa9edc23a F test/index7.test 9c6765a74fc3fcde7aebc5b3bd40d98df14a527c -F test/indexedby.test 5f527a78bae74c61b8046ae3037f9dfb0bf0c353 +F test/indexedby.test 69d2292dfdabe85aa7c5df577c71bb4325607ec2 F test/indexexpr1.test 4feec154aadacb033b41acc1760a18edc4c60470 F test/indexfault.test 31d4ab9a7d2f6e9616933eb079722362a883eb1d F test/init.test 15c823093fdabbf7b531fe22cf037134d09587a7 @@ -921,8 +921,8 @@ F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff F test/permutations.test ac3b00c299250cc087d4a527b5c75a0f8aef4e54 -F test/pragma.test be7195f0aa72bdb8a512133e9640ac40f15b57a2 -F test/pragma2.test 8e72df3a16c0fda748ad52abf79cb8256b04a6fe +F test/pragma.test a44253f911e7d50127d4a08f927f47c861a4c772 +F test/pragma2.test b5e2ce4c892afceb308c6ae6163a9099b2a0d8d7 F test/pragma3.test 6f849ccffeee7e496d2f2b5e74152306c0b8757c F test/printf.test b3ff34e73d59124140eaf89f7672e21bc2ca5fcc F test/printf2.test 0b61566dd1c0f0b802f59dffa228c5dc5aa6b054 @@ -1386,7 +1386,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f599a42e190b4b89d74055402143c5487985cd90 -R 0604cd6b1b67b982add6999f3d35855f -U mistachkin -Z 360c245eb92b0ddf2395cd1ae9027341 +P bfc7b84b766860d2e410702ba7c1166d7328309a +R ed47f4fa6e24d2d49fdba867f1c9ec47 +U drh +Z fe2e843011e778f2686c50d38eded399 diff --git a/manifest.uuid b/manifest.uuid index cb73a0c534..2f4be5f0e9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bfc7b84b766860d2e410702ba7c1166d7328309a \ No newline at end of file +86781093bdb4c4fdedd228cb1c8961db48a483bb \ No newline at end of file diff --git a/test/e_createtable.test b/test/e_createtable.test index 2921d86c6f..f07fbb9c74 100644 --- a/test/e_createtable.test +++ b/test/e_createtable.test @@ -373,8 +373,8 @@ do_createtable_tests 1.1.2 { } -# EVIDENCE-OF: R-10195-31023 If a is specified, it -# must be either "main", "temp", or the name of an attached database. +# EVIDENCE-OF: R-18448-33677 If a schema-name is specified, it must be +# either "main", "temp", or the name of an attached database. # # EVIDENCE-OF: R-39822-07822 In this case the new table is created in # the named database. @@ -422,9 +422,9 @@ do_createtable_tests 1.4 -tclquery { 2 "CREATE TEMPORARY TABLE t2(a, b)" {{} {t1 t2} {} {}} } -# EVIDENCE-OF: R-49439-47561 It is an error to specify both a -# and the TEMP or TEMPORARY keyword, unless the -# is "temp". +# EVIDENCE-OF: R-23976-43329 It is an error to specify both a +# schema-name and the TEMP or TEMPORARY keyword, unless the schema-name +# is "temp". # drop_all_tables do_createtable_tests 1.5.1 -error { @@ -447,9 +447,8 @@ do_createtable_tests 1.5.2 -tclquery { 4 "CREATE TEMPORARY TABLE TEMP.xxx(x)" {{} {t1 t2 t3 xxx} {} {}} } -# EVIDENCE-OF: R-00917-09393 If no database name is specified and the -# TEMP keyword is not present then the table is created in the main -# database. +# EVIDENCE-OF: R-31997-24564 If no schema name is specified and the TEMP +# keyword is not present then the table is created in the main database. # drop_all_tables do_createtable_tests 1.6 -tclquery { diff --git a/test/e_delete.test b/test/e_delete.test index b857cf147f..9bd93229e2 100644 --- a/test/e_delete.test +++ b/test/e_delete.test @@ -70,9 +70,12 @@ do_delete_tests e_delete-1.1 { 2 "DELETE FROM main.t2 ; SELECT * FROM t2" {} } -# EVIDENCE-OF: R-30203-16177 If a WHERE clause is supplied, then only -# those rows for which the result of evaluating the WHERE clause as a -# boolean expression is true are deleted. +# EVIDENCE-OF: R-26300-50198 If a WHERE clause is supplied, then only +# those rows for which the WHERE clause boolean expression is true are +# deleted. +# +# EVIDENCE-OF: R-23360-48280 Rows for which the expression is false or +# NULL are retained. # do_delete_tests e_delete-1.2 { 1 "DELETE FROM t3 WHERE 1 ; SELECT x FROM t3" {} @@ -117,8 +120,8 @@ do_execsql_test e_delete-2.0 { # EVIDENCE-OF: R-09681-58560 The table-name specified as part of a # DELETE statement within a trigger body must be unqualified. # -# EVIDENCE-OF: R-36771-43788 In other words, the database-name. prefix -# on the table name is not allowed within triggers. +# EVIDENCE-OF: R-12275-20298 In other words, the schema-name. prefix on +# the table name is not allowed within triggers. # do_delete_tests e_delete-2.1 -error { qualified table names are not allowed on INSERT, UPDATE, and DELETE statements within triggers diff --git a/test/e_expr.test b/test/e_expr.test index 271635f944..8c0957f8d3 100644 --- a/test/e_expr.test +++ b/test/e_expr.test @@ -1426,16 +1426,16 @@ do_execsql_test e_expr-27.1.2 { typeof(CAST(4.5 as INTEGER)), CAST(4.5 as INTEGER) } {text UVU real 1.23 integer 4} -# EVIDENCE-OF: R-27225-65050 If the value of is NULL, then -# the result of the CAST expression is also NULL. +# EVIDENCE-OF: R-32434-09092 If the value of expr is NULL, then the +# result of the CAST expression is also NULL. # do_expr_test e_expr-27.2.1 { CAST(NULL AS integer) } null {} do_expr_test e_expr-27.2.2 { CAST(NULL AS text) } null {} do_expr_test e_expr-27.2.3 { CAST(NULL AS blob) } null {} do_expr_test e_expr-27.2.4 { CAST(NULL AS number) } null {} -# EVIDENCE-OF: R-31076-23575 Casting a value to a with -# no affinity causes the value to be converted into a BLOB. +# EVIDENCE-OF: R-43522-35548 Casting a value to a type-name with no +# affinity causes the value to be converted into a BLOB. # do_expr_test e_expr-27.3.1 { CAST('abc' AS blob) } blob abc do_expr_test e_expr-27.3.2 { CAST('def' AS shobblob_x) } blob def diff --git a/test/e_insert.test b/test/e_insert.test index ffafa0ea52..32d75cbbd7 100644 --- a/test/e_insert.test +++ b/test/e_insert.test @@ -157,9 +157,9 @@ do_insert_tests e_insert-1.1 { 3b "SELECT count(*) FROM a2" {4} } -# EVIDENCE-OF: R-53616-44976 If no column-list is specified then the -# number of values inserted into each row must be the same as the number -# of columns in the table. +# EVIDENCE-OF: R-19218-01018 If the column-name list after table-name is +# omitted then the number of values inserted into each row must be the +# same as the number of columns in the table. # # A test in the block above verifies that if the VALUES list has the # correct number of columns (for table a2, 3 columns) works. So these @@ -191,9 +191,9 @@ do_insert_tests e_insert-1.3 { 3b "SELECT * FROM a2 WHERE oid=last_insert_rowid()" {2 x y} } -# EVIDENCE-OF: R-09234-17933 If a column-list is specified, then the -# number of values in each term of the VALUE list must match the number -# of specified columns. +# EVIDENCE-OF: R-21115-58321 If a column-name list is specified, then +# the number of values in each term of the VALUE list must match the +# number of specified columns. # do_insert_tests e_insert-1.4 -error { %d values for %d columns @@ -394,8 +394,8 @@ foreach {tn sql error ac data } { do_test e_insert-4.1.$tn.3 {sqlite3_get_autocommit db} $ac } -# EVIDENCE-OF: R-64196-02418 The optional "database-name." prefix on the -# table-name is support for top-level INSERT statements only. +# EVIDENCE-OF: R-59829-49719 The optional "schema-name." prefix on the +# table-name is supported for top-level INSERT statements only. # # EVIDENCE-OF: R-05731-00924 The table name must be unqualified for # INSERT statements that occur within CREATE TRIGGER statements. diff --git a/test/e_reindex.test b/test/e_reindex.test index fa66aa7a1f..c6a9e0352f 100644 --- a/test/e_reindex.test +++ b/test/e_reindex.test @@ -265,10 +265,9 @@ test_index 5.40 t2 collB value test_index 5.41 aux.t1 collA length test_index 5.42 aux.t1 collB value -# EVIDENCE-OF: R-15639-02023 If no database-name is specified and there -# exists both a table or index and a collation sequence of the specified -# name, SQLite interprets this as a request to rebuild the indices that -# use the named collation sequence. +# EVIDENCE-OF: R-35892-30289 For a command of the form "REINDEX name", a +# match against collation-name takes precedence over a match against +# index-name or table-name. # set_collations value length do_execsql_test e_reindex-2.6.0 { diff --git a/test/e_resolve.test b/test/e_resolve.test index 512fcf2748..f4bb1a4317 100644 --- a/test/e_resolve.test +++ b/test/e_resolve.test @@ -65,9 +65,9 @@ do_execsql_test 1.2 { SELECT * FROM n2 } {main n2} do_execsql_test 1.3 { SELECT * FROM n3 } {at1 n3} do_execsql_test 1.4 { SELECT * FROM n4 } {at2 n4} -# EVIDENCE-OF: R-54577-28142 If a database name is specified as part of -# an object reference, it must be either "main", or "temp" or the name -# of an attached database. +# EVIDENCE-OF: R-00634-08585 If a schema name is specified as part of an +# object reference, it must be either "main", or "temp" or the +# schema-name of an attached database. # # Or else it is a "no such table: xxx" error. # @@ -79,7 +79,7 @@ do_execsql_test 2.1.4 { SELECT * FROM at2.n1 } {at2 n1} do_catchsql_test 2.2 { SELECT * FROM xxx.n1 } {1 {no such table: xxx.n1}} -# EVIDENCE-OF: R-26223-47623 Like other SQL identifiers, database names +# EVIDENCE-OF: R-17446-42210 Like other SQL identifiers, schema names # are case-insensitive. # resolve_reopen_db @@ -88,8 +88,8 @@ do_execsql_test 3.2 { SELECT * FROM tEmP.n1 } {temp n1} do_execsql_test 3.3 { SELECT * FROM aT1.n1 } {at1 n1} do_execsql_test 3.4 { SELECT * FROM At2.n1 } {at2 n1} -# EVIDENCE-OF: R-15639-28392 If a database name is specified, then only -# the named database is searched for the named object. +# EVIDENCE-OF: R-14755-58619 If a schema name is specified, then only +# that one schema is searched for the named object. # do_catchsql_test 4.1 { SELECT * FROM temp.n2 } {1 {no such table: temp.n2}} do_catchsql_test 4.2 { SELECT * FROM main.n2 } {0 {main n2}} diff --git a/test/e_update.test b/test/e_update.test index e9c6b26809..a13b059b32 100644 --- a/test/e_update.test +++ b/test/e_update.test @@ -146,9 +146,8 @@ do_update_tests e_update-1.2 { {greek roman greek roman greek roman} } -# EVIDENCE-OF: R-42117-40023 Otherwise, the UPDATE affects only those -# rows for which the result of evaluating the WHERE clause expression as -# a boolean expression is true. +# EVIDENCE-OF: R-58095-46013 Otherwise, the UPDATE affects only those +# rows for which the WHERE clause boolean expression is true. # do_execsql_test e_update-1.3.0 { DELETE FROM main.t1; @@ -265,9 +264,10 @@ do_update_tests e_update-1.7 -query { 3 "UPDATE t2 SET a=c||c, c=NULL" {44 5 {} 99 14 {} 55 11 {}} } -# EVIDENCE-OF: R-12619-24112 The optional conflict-clause allows the -# user to nominate a specific constraint conflict resolution algorithm -# to use during this one UPDATE command. +# EVIDENCE-OF: R-28518-13457 The optional "OR action" conflict clause +# that follows the UPDATE keyword allows the user to nominate a specific +# constraint conflict resolution algorithm to use during this one UPDATE +# command. # do_execsql_test e_update-1.8.0 { DELETE FROM t3; @@ -322,8 +322,8 @@ foreach {tn sql error ac data } { # EVIDENCE-OF: R-12123-54095 The table-name specified as part of an # UPDATE statement within a trigger body must be unqualified. # -# EVIDENCE-OF: R-09690-36749 In other words, the database-name. prefix -# on the table name of the UPDATE is not allowed within triggers. +# EVIDENCE-OF: R-43190-62442 In other words, the schema-name. prefix on +# the table name of the UPDATE is not allowed within triggers. # do_update_tests e_update-2.1 -error { qualified table names are not allowed on INSERT, UPDATE, and DELETE statements within triggers diff --git a/test/fkey5.test b/test/fkey5.test index 21feb2bb8b..b9e1fc2eec 100644 --- a/test/fkey5.test +++ b/test/fkey5.test @@ -12,8 +12,8 @@ # # This file tests the PRAGMA foreign_key_check command. # -# EVIDENCE-OF: R-01427-50262 PRAGMA database.foreign_key_check; PRAGMA -# database.foreign_key_check(table-name); +# EVIDENCE-OF: R-15402-03103 PRAGMA schema.foreign_key_check; PRAGMA +# schema.foreign_key_check(table-name); # # EVIDENCE-OF: R-23918-17301 The foreign_key_check pragma checks the # database, or the table called "table-name", for foreign key diff --git a/test/indexedby.test b/test/indexedby.test index 413bf07422..975b2be605 100644 --- a/test/indexedby.test +++ b/test/indexedby.test @@ -58,7 +58,7 @@ do_execsql_test indexedby-1.4 { # SQL view. Also test that specifying an index that does not exist or # is attached to a different table is detected as an error. # -# EVIDENCE-OF: R-63761-48810 -- syntax diagram qualified-table-name +# EVIDENCE-OF: R-07004-11522 -- syntax diagram qualified-table-name # # EVIDENCE-OF: R-58230-57098 The "INDEXED BY index-name" phrase # specifies that the named index must be used in order to look up values diff --git a/test/pragma.test b/test/pragma.test index 587a03c8a6..246a7cd854 100644 --- a/test/pragma.test +++ b/test/pragma.test @@ -83,8 +83,8 @@ delete_file test.db test.db-journal delete_file test3.db test3.db-journal sqlite3 db test.db; set DB [sqlite3_connection_pointer db] -# EVIDENCE-OF: R-24197-42751 PRAGMA database.cache_size; PRAGMA -# database.cache_size = pages; PRAGMA database.cache_size = -kibibytes; +# EVIDENCE-OF: R-13861-56665 PRAGMA schema.cache_size; PRAGMA +# schema.cache_size = pages; PRAGMA schema.cache_size = -kibibytes; # Query or change the suggested maximum number of database disk pages # that SQLite will hold in memory at once per open database file. # @@ -697,8 +697,8 @@ do_test pragma-6.5.1b { } {0 0 a 1 1 b 2 -1 {}} -# EVIDENCE-OF: R-62725-03366 PRAGMA database.index_info(index-name); -# This pragma returns one row for each key column in the named index. +# EVIDENCE-OF: R-29448-60346 PRAGMA schema.index_info(index-name); This +# pragma returns one row for each key column in the named index. # # (The first column of output from PRAGMA index_info is...) # EVIDENCE-OF: R-34186-52914 The rank of the column within the index. (0 @@ -784,9 +784,9 @@ do_test pragma-6.8 { # Miscellaneous tests # ifcapable schema_pragmas { -# EVIDENCE-OF: R-63500-32024 PRAGMA database.index_list(table-name); -# This pragma returns one row for each index associated with the given -# table. +# EVIDENCE-OF: R-64103-17776 PRAGMA schema.index_list(table-name); This +# pragma returns one row for each index associated with the given table. +# do_test pragma-7.1.1 { # Make sure a pragma knows to read the schema if it needs to db close @@ -1379,8 +1379,8 @@ ifcapable pager_pragmas { forcedelete test.db sqlite3 db test.db - # EVIDENCE-OF: R-13905-26312 PRAGMA database.page_count; Return the - # total number of pages in the database file. + # EVIDENCE-OF: R-15672-33611 PRAGMA schema.page_count; Return the total + # number of pages in the database file. # do_test pragma-14.1 { execsql { pragma auto_vacuum = 0 } @@ -1817,8 +1817,8 @@ do_test 23.2a { db2 eval {SELECT cid, name, '|' FROM out ORDER BY seqno} } {2 c | 3 d | 1 b |} -# EVIDENCE-OF: R-44874-46325 PRAGMA database.index_xinfo(index-name); -# This pragma returns information about every column in an index. +# EVIDENCE-OF: R-56143-29319 PRAGMA schema.index_xinfo(index-name); This +# pragma returns information about every column in an index. # # EVIDENCE-OF: R-45970-35618 Unlike this index_info pragma, this pragma # returns information about every column in the index, not just the key @@ -1861,9 +1861,8 @@ do_test 23.2d { db2 eval {PRAGMA index_xinfo(i2x)} } {0 3 d 0 nocase 1 1 2 c 1 BINARY 1 2 -1 {} 0 BINARY 0} -# EVIDENCE-OF: R-63500-32024 PRAGMA database.index_list(table-name); -# This pragma returns one row for each index associated with the given -# table. +# EVIDENCE-OF: R-64103-17776 PRAGMA schema.index_list(table-name); This +# pragma returns one row for each index associated with the given table. # # (The first column of output from PRAGMA index_list is...) # EVIDENCE-OF: R-02753-24748 A sequence number assigned to each index diff --git a/test/pragma2.test b/test/pragma2.test index 12d5cce5f7..e2c87019eb 100644 --- a/test/pragma2.test +++ b/test/pragma2.test @@ -42,7 +42,7 @@ sqlite3 db test.db; set DB [sqlite3_connection_pointer db] db eval {PRAGMA auto_vacuum=0} -# EVIDENCE-OF: R-17887-14874 PRAGMA database.freelist_count; Return the +# EVIDENCE-OF: R-11211-21323 PRAGMA schema.freelist_count; Return the # number of unused pages in the database file. # do_test pragma2-1.1 { From 04abf0878f82de4b14796cd71af275b68c73af35 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Sat, 12 Sep 2015 18:57:45 +0000 Subject: [PATCH 087/100] Import common changes from the mutex initialization branch. FossilOrigin-Name: 334720c01722478af0d3dfd6fe8bafd88ba09f49 --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/mutex.c | 11 ++++++++++- src/mutex_unix.c | 2 +- src/sqliteInt.h | 2 +- 5 files changed, 22 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 2f7902f582..eb8d534263 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sevidence\smarks\sdue\sto\swording\schanges\sin\srequirements\stext.\s\sNo\nchanges\sto\scode. -D 2015-09-11T20:54:44.879 +C Import\scommon\schanges\sfrom\sthe\smutex\sinitialization\sbranch. +D 2015-09-12T18:57:45.818 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -314,10 +314,10 @@ F src/mem3.c 61c9d47b792908c532ca3a62b999cf21795c6534 F src/mem5.c 61eeb90134f9a5be6c2e68d8daae7628b25953fb F src/memjournal.c 3eb2c0b51adbd869cb6a44780323f05fa904dc85 F src/msvc.h d9ba56c6851227ab44b3f228a35f3f5772296495 -F src/mutex.c a39809c6c33f1ebc9cc781186c338ad90433e1e7 +F src/mutex.c 8e45800ee78e0cd1f1f3fe8e398853307f4a085c F src/mutex.h 779d588e3b7756ec3ecf7d78cde1d84aba414f85 F src/mutex_noop.c 9d4309c075ba9cc7249e19412d3d62f7f94839c4 -F src/mutex_unix.c 7762c8ec907379204f2ed751a0e33663ab1c14d7 +F src/mutex_unix.c 8cfa6e83c618d2fcae0fe63f4d2b5bb16b11a97a F src/mutex_w32.c 2e025e6642eaf27597403690980f560d1a91f62c F src/notify.c 9711a7575036f0d3040ba61bc6e217f13a9888e7 F src/os.c 8fd25588eeba74068d41102d26810e216999b6c8 @@ -345,7 +345,7 @@ F src/shell.c 6332ef06db1390ef812cfdff1fc97b4fd76cdd42 F src/sqlite.h.in dbaf8c3796e80221de4395b5f4f872abddb5f89f F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 64350bf36833a56ad675e27392a913f417c5c308 -F src/sqliteInt.h 788dc0ea7ba32ec9fec06c628c1792d7b4753d86 +F src/sqliteInt.h b3e590f374b376a793b93e2387b8d5aca0fc92c4 F src/sqliteLimit.h 216557999cb45f2e3578ed53ebefe228d779cb46 F src/status.c f266ad8a2892d659b74f0f50cb6a88b6e7c12179 F src/table.c 51b46b2a62d1b3a959633d593b89bab5e2c9155e @@ -1386,7 +1386,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P bfc7b84b766860d2e410702ba7c1166d7328309a -R ed47f4fa6e24d2d49fdba867f1c9ec47 -U drh -Z fe2e843011e778f2686c50d38eded399 +P 86781093bdb4c4fdedd228cb1c8961db48a483bb +R 65cef7a212785e2c4ed01bbc55ee07c1 +U mistachkin +Z 122b4ea017ab904cde327a85d32df791 diff --git a/manifest.uuid b/manifest.uuid index 2f4be5f0e9..62601ecd30 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -86781093bdb4c4fdedd228cb1c8961db48a483bb \ No newline at end of file +334720c01722478af0d3dfd6fe8bafd88ba09f49 \ No newline at end of file diff --git a/src/mutex.c b/src/mutex.c index a2e4e6387a..6f1bc9767d 100644 --- a/src/mutex.c +++ b/src/mutex.c @@ -22,7 +22,7 @@ ** allocate a mutex while the system is uninitialized. */ static SQLITE_WSD int mutexIsInit = 0; -#endif /* SQLITE_DEBUG */ +#endif /* SQLITE_DEBUG && !defined(SQLITE_MUTEX_OMIT) */ #ifndef SQLITE_MUTEX_OMIT @@ -56,6 +56,7 @@ int sqlite3MutexInit(void){ sqlite3MemoryBarrier(); pTo->xMutexAlloc = pFrom->xMutexAlloc; } + assert( sqlite3GlobalConfig.mutex.xMutexInit ); rc = sqlite3GlobalConfig.mutex.xMutexInit(); #ifdef SQLITE_DEBUG @@ -90,6 +91,7 @@ sqlite3_mutex *sqlite3_mutex_alloc(int id){ if( id<=SQLITE_MUTEX_RECURSIVE && sqlite3_initialize() ) return 0; if( id>SQLITE_MUTEX_RECURSIVE && sqlite3MutexInit() ) return 0; #endif + assert( sqlite3GlobalConfig.mutex.xMutexAlloc ); return sqlite3GlobalConfig.mutex.xMutexAlloc(id); } @@ -98,6 +100,7 @@ sqlite3_mutex *sqlite3MutexAlloc(int id){ return 0; } assert( GLOBAL(int, mutexIsInit) ); + assert( sqlite3GlobalConfig.mutex.xMutexAlloc ); return sqlite3GlobalConfig.mutex.xMutexAlloc(id); } @@ -106,6 +109,7 @@ sqlite3_mutex *sqlite3MutexAlloc(int id){ */ void sqlite3_mutex_free(sqlite3_mutex *p){ if( p ){ + assert( sqlite3GlobalConfig.mutex.xMutexFree ); sqlite3GlobalConfig.mutex.xMutexFree(p); } } @@ -116,6 +120,7 @@ void sqlite3_mutex_free(sqlite3_mutex *p){ */ void sqlite3_mutex_enter(sqlite3_mutex *p){ if( p ){ + assert( sqlite3GlobalConfig.mutex.xMutexEnter ); sqlite3GlobalConfig.mutex.xMutexEnter(p); } } @@ -127,6 +132,7 @@ void sqlite3_mutex_enter(sqlite3_mutex *p){ int sqlite3_mutex_try(sqlite3_mutex *p){ int rc = SQLITE_OK; if( p ){ + assert( sqlite3GlobalConfig.mutex.xMutexTry ); return sqlite3GlobalConfig.mutex.xMutexTry(p); } return rc; @@ -140,6 +146,7 @@ int sqlite3_mutex_try(sqlite3_mutex *p){ */ void sqlite3_mutex_leave(sqlite3_mutex *p){ if( p ){ + assert( sqlite3GlobalConfig.mutex.xMutexLeave ); sqlite3GlobalConfig.mutex.xMutexLeave(p); } } @@ -150,9 +157,11 @@ void sqlite3_mutex_leave(sqlite3_mutex *p){ ** intended for use inside assert() statements. */ int sqlite3_mutex_held(sqlite3_mutex *p){ + assert( p==0 || sqlite3GlobalConfig.mutex.xMutexHeld ); return p==0 || sqlite3GlobalConfig.mutex.xMutexHeld(p); } int sqlite3_mutex_notheld(sqlite3_mutex *p){ + assert( p==0 || sqlite3GlobalConfig.mutex.xMutexNotheld ); return p==0 || sqlite3GlobalConfig.mutex.xMutexNotheld(p); } #endif diff --git a/src/mutex_unix.c b/src/mutex_unix.c index 78fba1d81f..cebb96c90e 100644 --- a/src/mutex_unix.c +++ b/src/mutex_unix.c @@ -86,7 +86,7 @@ static int pthreadMutexNotheld(sqlite3_mutex *p){ void sqlite3MemoryBarrier(void){ #if defined(SQLITE_MEMORY_BARRIER) SQLITE_MEMORY_BARRIER; -#elif defined(__GNUC__) +#elif defined(__GNUC__) && GCC_VERSION>=4001000 __sync_synchronize(); #endif } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 98cbca5193..c9452b1d55 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3195,7 +3195,7 @@ const sqlite3_mem_methods *sqlite3MemGetMemsys5(void); #if !defined(SQLITE_MUTEX_OMIT) && !defined(SQLITE_MUTEX_NOOP) void sqlite3MemoryBarrier(void); #else -# define sqlite3MemoryBarrier(); +# define sqlite3MemoryBarrier() #endif sqlite3_int64 sqlite3StatusValue(int); From f0ee1d3c127973f9e643786cfc5cb4b733224399 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 12 Sep 2015 19:26:11 +0000 Subject: [PATCH 088/100] Experimental change to use a single-pass approach for DELETE statements on non-virtual tables that do not fire triggers or require foriegn-key processing. FossilOrigin-Name: eaeb2b80f6f8f83679c8323a81bb39570ec946fe --- manifest | 36 +++++++------ manifest.uuid | 2 +- src/btree.c | 126 +++++++++++++++++++++++++++++++++----------- src/btree.h | 2 +- src/delete.c | 120 +++++++++++++++++++++++++++-------------- src/insert.c | 6 +-- src/sqliteInt.h | 6 ++- src/update.c | 2 +- src/vdbe.c | 25 +++++---- src/where.c | 19 +++++-- test/delete.test | 1 - test/delete4.test | 102 +++++++++++++++++++++++++++++++++++ test/indexedby.test | 4 +- 13 files changed, 338 insertions(+), 113 deletions(-) create mode 100644 test/delete4.test diff --git a/manifest b/manifest index eb8d534263..e4c2c2f086 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Import\scommon\schanges\sfrom\sthe\smutex\sinitialization\sbranch. -D 2015-09-12T18:57:45.818 +C Experimental\schange\sto\suse\sa\ssingle-pass\sapproach\sfor\sDELETE\sstatements\son\snon-virtual\stables\sthat\sdo\snot\sfire\striggers\sor\srequire\sforiegn-key\sprocessing. +D 2015-09-12T19:26:11.066 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -282,8 +282,8 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 4d9134dc988a87838c06056c89c0e8c4700a0452 F src/bitvec.c d1f21d7d91690747881f03940584f4cc548c9d3d F src/btmutex.c 45a968cc85afed9b5e6cf55bf1f42f8d18107f79 -F src/btree.c 4084d9eed2817331f6e6a82230ba30e448cad497 -F src/btree.h 969adc948e89e449220ff0ff724c94bb2a52e9f1 +F src/btree.c 38ed0262d1c66d21bb084f086650e6106ae43d98 +F src/btree.h 40189aefdc2b830d25c8b58fd7d56538481bfdd7 F src/btreeInt.h 8177c9ab90d772d6d2c6c517e05bed774b7c92c0 F src/build.c f81380bc4d5d239c18b42982a9866a94489fd444 F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 @@ -291,7 +291,7 @@ F src/complete.c addcd8160b081131005d5bc2d34adf20c1c5c92f F src/ctime.c 5a0b735dc95604766f5dac73973658eef782ee8b F src/date.c fb1c99172017dcc8e237339132c91a21a0788584 F src/dbstat.c e637e7a7ff40ef32132a418c6fdf1cfb63aa27c7 -F src/delete.c 6792c80d7fb54c4df9f7680413952600e7439492 +F src/delete.c d5a2dc4a4663225abbcab042478dc37a5749b7d7 F src/expr.c 3a76afcdac925294c39903b7002ddb9e5fd29863 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 83e1baba999bed3144ea5a2143fc922edf51135f @@ -300,7 +300,7 @@ F src/global.c 508e4087f7b41d688e4762dcf4d4fe28cfbc87f9 F src/hash.c 4263fbc955f26c2e8cdc0cf214bc42435aa4e4f5 F src/hash.h c8f3c31722cf3277d03713909761e152a5b81094 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 -F src/insert.c 076dc5876e261a9908603d54cfc5344cd680166c +F src/insert.c db8a34cf8ba600ac1cebb3c03e93c92154d0fc4c F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c d344a95d60c24e2f490ee59db9784b1b17439012 @@ -345,7 +345,7 @@ F src/shell.c 6332ef06db1390ef812cfdff1fc97b4fd76cdd42 F src/sqlite.h.in dbaf8c3796e80221de4395b5f4f872abddb5f89f F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 64350bf36833a56ad675e27392a913f417c5c308 -F src/sqliteInt.h b3e590f374b376a793b93e2387b8d5aca0fc92c4 +F src/sqliteInt.h 91bf09de55402157d1476a61df46ef6cfbc0bbc3 F src/sqliteLimit.h 216557999cb45f2e3578ed53ebefe228d779cb46 F src/status.c f266ad8a2892d659b74f0f50cb6a88b6e7c12179 F src/table.c 51b46b2a62d1b3a959633d593b89bab5e2c9155e @@ -399,11 +399,11 @@ F src/threads.c 6bbcc9fe50c917864d48287b4792d46d6e873481 F src/tokenize.c 83c6ed569423a3af83a83973b444cf7123be33a6 F src/treeview.c 154f0acc622fa3514de8777dcedf4c8a8802b4ce F src/trigger.c 322f23aad694e8f31d384dcfa386d52a48d3c52f -F src/update.c 3c5bc9570df3bfafa0db36828406a8a14e4c426e +F src/update.c eb7ab3ff2928628692a4f14be397c95f4a681d97 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c fc612367108b74573c5fd13a85d0a23027f438bd F src/vacuum.c 2ddd5cad2a7b9cef7f9e431b8c7771634c6b1701 -F src/vdbe.c 6d85be995bd2308a5aa2a68c7b564c5d4cc1a6fb +F src/vdbe.c 5587d76bd5a4fb4f7ca023f161a6c6adbb1de26c F src/vdbe.h 4bc88bd0e06f8046ee6ab7487c0015e85ad949ad F src/vdbeInt.h 8b867eac234e28627ffcace3cd4b4b79bbec664b F src/vdbeapi.c 0d890f57caf143b114a95ce699e59af51359c508 @@ -417,7 +417,7 @@ F src/vxworks.h c18586c8edc1bddbc15c004fa16aeb1e1342b4fb F src/wal.c 18b0ed49830cf04fe2d68224b41838a73ac6cd24 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 2e14d17f592d176b6dc879c33fbdec4fbccaa2ba -F src/where.c 1227687e7892d4009f3c3433e974eb9c9e3c4d6a +F src/where.c 98cbedead64380fc26a098350f43d92237c8fa17 F src/whereInt.h 292d3ac90da4eab1e03ac8452f1add746bcafaa1 F src/wherecode.c 6ac8599523f4840d9efac335329f627ebf3f79fd F src/whereexpr.c 2473e4350e30f9b55d1c6a8f66ca23c689f23f1d @@ -569,9 +569,10 @@ F test/date.test 42973251b9429f2c41b77eb98a7b0b0ba2d3b2c0 F test/dbstatus.test 8de104bb5606f19537d23cd553b41349b5ab1204 F test/dbstatus2.test 10418e62b3db5dca070f0c3eef3ea13946f339c2 F test/default.test 0cb49b1c315a0d81c81d775e407f66906a2a604d -F test/delete.test a065b05d2ebf60fd16639c579a4adfb7c381c701 +F test/delete.test e1bcdf8926234e27aac24b346ad83d3329ec8b6f F test/delete2.test 3a03f2cca1f9a67ec469915cb8babd6485db43fa F test/delete3.test 555e84a00a99230b7d049d477a324a631126a6ab +F test/delete4.test d9e7d553a939597b27d205b022d769469f361c1f F test/descidx1.test 6d03b44c8538fe0eb4924e19fba10cdd8f3c9240 F test/descidx2.test 9f1a0c83fd57f8667c82310ca21b30a350888b5d F test/descidx3.test 09ddbe3f5295f482d2f8b687cf6db8bad7acd9a2 @@ -782,7 +783,7 @@ F test/index4.test ab92e736d5946840236cd61ac3191f91a7856bf6 F test/index5.test 8621491915800ec274609e42e02a97d67e9b13e7 F test/index6.test 7102ec371414c42dfb1d5ca37eb4519aa9edc23a F test/index7.test 9c6765a74fc3fcde7aebc5b3bd40d98df14a527c -F test/indexedby.test 69d2292dfdabe85aa7c5df577c71bb4325607ec2 +F test/indexedby.test 9c4cd331224e57f79fbf411ae245e6272d415985 F test/indexexpr1.test 4feec154aadacb033b41acc1760a18edc4c60470 F test/indexfault.test 31d4ab9a7d2f6e9616933eb079722362a883eb1d F test/init.test 15c823093fdabbf7b531fe22cf037134d09587a7 @@ -1386,7 +1387,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 86781093bdb4c4fdedd228cb1c8961db48a483bb -R 65cef7a212785e2c4ed01bbc55ee07c1 -U mistachkin -Z 122b4ea017ab904cde327a85d32df791 +P 334720c01722478af0d3dfd6fe8bafd88ba09f49 +R 7f7d9f32d96ad1f6004a92150fc6799e +T *branch * onepass-delete +T *sym-onepass-delete * +T -sym-trunk * +U dan +Z 9a0d15a0b8fdf4a99fe6f2640c27db22 diff --git a/manifest.uuid b/manifest.uuid index 62601ecd30..5a82be83bf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -334720c01722478af0d3dfd6fe8bafd88ba09f49 \ No newline at end of file +eaeb2b80f6f8f83679c8323a81bb39570ec946fe \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index d5b9b5fd85..80aac20524 100644 --- a/src/btree.c +++ b/src/btree.c @@ -591,6 +591,49 @@ static void btreeReleaseAllCursorPages(BtCursor *pCur){ pCur->iPage = -1; } +/* +** The cursor passed as the only argument must point to a valid entry +** when this function is called (i.e. have eState==CURSOR_VALID). This +** function saves the current cursor key in variables pCur->nKey and +** pCur->pKey. SQLITE_OK is returned if successful or an SQLite error +** code otherwise. +** +** If the cursor is open on an intkey table, then the integer key +** (the rowid) is stored in pCur->nKey and pCur->pKey is left set to +** NULL. If the cursor is open on a non-intkey table, then pCur->pKey is +** set to point to a malloced buffer pCur->nKey bytes in size containing +** the key. +*/ +static int saveCursorKey(BtCursor *pCur){ + int rc; + assert( CURSOR_VALID==pCur->eState ); + assert( 0==pCur->pKey ); + assert( cursorHoldsMutex(pCur) ); + + rc = sqlite3BtreeKeySize(pCur, &pCur->nKey); + assert( rc==SQLITE_OK ); /* KeySize() cannot fail */ + + /* If this is an intKey table, then the above call to BtreeKeySize() + ** stores the integer key in pCur->nKey. In this case this value is + ** all that is required. Otherwise, if pCur is not open on an intKey + ** table, then malloc space for and store the pCur->nKey bytes of key + ** data. */ + if( 0==pCur->curIntKey ){ + void *pKey = sqlite3Malloc( pCur->nKey ); + if( pKey ){ + rc = sqlite3BtreeKey(pCur, 0, (int)pCur->nKey, pKey); + if( rc==SQLITE_OK ){ + pCur->pKey = pKey; + }else{ + sqlite3_free(pKey); + } + }else{ + rc = SQLITE_NOMEM; + } + } + assert( !pCur->curIntKey || !pCur->pKey ); + return rc; +} /* ** Save the current cursor position in the variables BtCursor.nKey @@ -611,30 +654,8 @@ static int saveCursorPosition(BtCursor *pCur){ }else{ pCur->skipNext = 0; } - rc = sqlite3BtreeKeySize(pCur, &pCur->nKey); - assert( rc==SQLITE_OK ); /* KeySize() cannot fail */ - - /* If this is an intKey table, then the above call to BtreeKeySize() - ** stores the integer key in pCur->nKey. In this case this value is - ** all that is required. Otherwise, if pCur is not open on an intKey - ** table, then malloc space for and store the pCur->nKey bytes of key - ** data. - */ - if( 0==pCur->curIntKey ){ - void *pKey = sqlite3Malloc( pCur->nKey ); - if( pKey ){ - rc = sqlite3BtreeKey(pCur, 0, (int)pCur->nKey, pKey); - if( rc==SQLITE_OK ){ - pCur->pKey = pKey; - }else{ - sqlite3_free(pKey); - } - }else{ - rc = SQLITE_NOMEM; - } - } - assert( !pCur->curIntKey || !pCur->pKey ); + rc = saveCursorKey(pCur); if( rc==SQLITE_OK ){ btreeReleaseAllCursorPages(pCur); pCur->eState = CURSOR_REQUIRESEEK; @@ -8026,10 +8047,15 @@ end_insert: } /* -** Delete the entry that the cursor is pointing to. The cursor -** is left pointing at an arbitrary location. +** Delete the entry that the cursor is pointing to. +** +** If the second parameter is zero, then the cursor is left pointing at an +** arbitrary location after the delete. If it is non-zero, then the cursor +** is left in a state such that the next call to BtreeNext() or BtreePrev() +** moves it to the same row as it would if the call to BtreeDelete() had +** been omitted. */ -int sqlite3BtreeDelete(BtCursor *pCur){ +int sqlite3BtreeDelete(BtCursor *pCur, int bPreserve){ Btree *p = pCur->pBtree; BtShared *pBt = p->pBt; int rc; /* Return code */ @@ -8038,6 +8064,7 @@ int sqlite3BtreeDelete(BtCursor *pCur){ int iCellIdx; /* Index of cell to delete */ int iCellDepth; /* Depth of node containing pCell */ u16 szCell; /* Size of the cell being deleted */ + int bSkipnext = 0; /* Leaf cursor in SKIPNEXT state */ assert( cursorHoldsMutex(pCur) ); assert( pBt->inTransaction==TRANS_WRITE ); @@ -8067,10 +8094,7 @@ int sqlite3BtreeDelete(BtCursor *pCur){ } /* Save the positions of any other cursors open on this table before - ** making any modifications. Make the page containing the entry to be - ** deleted writable. Then free any overflow pages associated with the - ** entry and finally remove the cell itself from within the page. - */ + ** making any modifications. */ if( pCur->curFlags & BTCF_Multiple ){ rc = saveAllCursors(pBt, pCur->pgnoRoot, pCur); if( rc ) return rc; @@ -8082,6 +8106,31 @@ int sqlite3BtreeDelete(BtCursor *pCur){ invalidateIncrblobCursors(p, pCur->info.nKey, 0); } + /* If the bPreserve flag is set to true, then the cursor position must + ** be preserved following this delete operation. If the current delete + ** will cause a b-tree rebalance, then this is done by saving the cursor + ** key and leaving the cursor in CURSOR_REQUIRESEEK state before + ** returning. + ** + ** Or, if the current delete will not cause a rebalance, then the cursor + ** will be left in CURSOR_SKIPNEXT state pointing to the entry immediately + ** before or after the deleted entry. In this case set bSkipnext to true. */ + if( bPreserve ){ + if( !pPage->leaf + || (pPage->nFree + cellSizePtr(pPage, pCell) + 2)>(pBt->usableSize*2/3) + ){ + /* A b-tree rebalance will be required after deleting this entry. + ** Save the cursor key. */ + rc = saveCursorKey(pCur); + if( rc ) return rc; + }else{ + bSkipnext = 1; + } + } + + /* Make the page containing the entry to be deleted writable. Then free any + ** overflow pages associated with the entry and finally remove the cell + ** itself from within the page. */ rc = sqlite3PagerWrite(pPage->pDbPage); if( rc ) return rc; rc = clearCell(pPage, pCell, &szCell); @@ -8135,7 +8184,22 @@ int sqlite3BtreeDelete(BtCursor *pCur){ } if( rc==SQLITE_OK ){ - moveToRoot(pCur); + if( bSkipnext ){ + assert( bPreserve && pCur->iPage==iCellDepth ); + assert( pPage->nCell>0 && iCellIdx<=pPage->nCell ); + pCur->eState = CURSOR_SKIPNEXT; + if( iCellIdx>=pPage->nCell ){ + pCur->skipNext = -1; + pCur->aiIdx[iCellDepth] = pPage->nCell-1; + }else{ + pCur->skipNext = 1; + } + }else{ + rc = moveToRoot(pCur); + if( bPreserve ){ + pCur->eState = CURSOR_REQUIRESEEK; + } + } } return rc; } diff --git a/src/btree.h b/src/btree.h index 3edc2b3b57..f7e92a2609 100644 --- a/src/btree.h +++ b/src/btree.h @@ -185,7 +185,7 @@ int sqlite3BtreeMovetoUnpacked( ); int sqlite3BtreeCursorHasMoved(BtCursor*); int sqlite3BtreeCursorRestore(BtCursor*, int*); -int sqlite3BtreeDelete(BtCursor*); +int sqlite3BtreeDelete(BtCursor*, int); int sqlite3BtreeInsert(BtCursor*, const void *pKey, i64 nKey, const void *pData, int nData, int nZero, int bias, int seekResult); diff --git a/src/delete.c b/src/delete.c index efbb7747f7..efec82aab9 100644 --- a/src/delete.c +++ b/src/delete.c @@ -235,7 +235,7 @@ void sqlite3DeleteFrom( int iDb; /* Database number */ int memCnt = -1; /* Memory cell used for change counting */ int rcauth; /* Value returned by authorization callback */ - int okOnePass; /* True for one-pass algorithm without the FIFO */ + int eOnePass; /* Non-zero for one-pass algorithm without the FIFO */ int aiCurOnePass[2]; /* The write cursors opened by WHERE_ONEPASS */ u8 *aToOpen = 0; /* Open cursor iTabCur+j if aToOpen[j] is true */ Index *pPk; /* The PRIMARY KEY index on the table */ @@ -253,6 +253,7 @@ void sqlite3DeleteFrom( #ifndef SQLITE_OMIT_TRIGGER int isView; /* True if attempting to delete from a view */ Trigger *pTrigger; /* List of table triggers, if required */ + int bComplex; /* True if there are either triggers or FKs */ #endif memset(&sContext, 0, sizeof(sContext)); @@ -276,9 +277,11 @@ void sqlite3DeleteFrom( #ifndef SQLITE_OMIT_TRIGGER pTrigger = sqlite3TriggersExist(pParse, pTab, TK_DELETE, 0, 0); isView = pTab->pSelect!=0; + bComplex = pTrigger || sqlite3FkRequired(pParse, pTab, 0, 0); #else # define pTrigger 0 # define isView 0 +# define bComplex 0 #endif #ifdef SQLITE_OMIT_VIEW # undef isView @@ -359,9 +362,7 @@ void sqlite3DeleteFrom( ** It is easier just to erase the whole table. Prior to version 3.6.5, ** this optimization caused the row change count (the value returned by ** API function sqlite3_count_changes) to be set incorrectly. */ - if( rcauth==SQLITE_OK && pWhere==0 && !pTrigger && !IsVirtual(pTab) - && 0==sqlite3FkRequired(pParse, pTab, 0, 0) - ){ + if( rcauth==SQLITE_OK && pWhere==0 && !bComplex && !IsVirtual(pTab) ){ assert( !isView ); sqlite3TableLock(pParse, iDb, pTab->tnum, 1, pTab->zName); if( HasRowid(pTab) ){ @@ -375,6 +376,8 @@ void sqlite3DeleteFrom( }else #endif /* SQLITE_OMIT_TRUNCATE_OPTIMIZATION */ { + u16 wcf = WHERE_ONEPASS_DESIRED|WHERE_DUPLICATES_OK; + wcf |= (bComplex ? 0 : WHERE_ONEPASS_MULTIROW); if( HasRowid(pTab) ){ /* For a rowid table, initialize the RowSet to an empty set */ pPk = 0; @@ -395,13 +398,17 @@ void sqlite3DeleteFrom( } /* Construct a query to find the rowid or primary key for every row - ** to be deleted, based on the WHERE clause. + ** to be deleted, based on the WHERE clause. Set variable eOnePass + ** to indicate the strategy used to implement this delete: + ** + ** 0: Two-pass approach - use a FIFO for rowids/PK values. + ** 1: One-pass approach - at most one row deleted. + ** 2: One-pass approach - any number of rows may be deleted. */ - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, 0, 0, - WHERE_ONEPASS_DESIRED|WHERE_DUPLICATES_OK, - iTabCur+1); + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, 0, 0, wcf, iTabCur+1); if( pWInfo==0 ) goto delete_from_cleanup; - okOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass); + eOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass); + assert( IsVirtual(pTab)==0 || eOnePass==0 ); /* Keep track of the number of rows to be deleted */ if( db->flags & SQLITE_CountRows ){ @@ -422,11 +429,10 @@ void sqlite3DeleteFrom( if( iKey>pParse->nMem ) pParse->nMem = iKey; } - if( okOnePass ){ - /* For ONEPASS, no need to store the rowid/primary-key. There is only + if( eOnePass ){ + /* For ONEPASS, no need to store the rowid/primary-key. There is only ** one, so just keep it in its register(s) and fall through to the - ** delete code. - */ + ** delete code. */ nKey = nPk; /* OP_Found will use an unpacked key */ aToOpen = sqlite3DbMallocRaw(db, nIdx+2); if( aToOpen==0 ){ @@ -438,27 +444,27 @@ void sqlite3DeleteFrom( if( aiCurOnePass[0]>=0 ) aToOpen[aiCurOnePass[0]-iTabCur] = 0; if( aiCurOnePass[1]>=0 ) aToOpen[aiCurOnePass[1]-iTabCur] = 0; if( addrEphOpen ) sqlite3VdbeChangeToNoop(v, addrEphOpen); - addrDelete = sqlite3VdbeAddOp0(v, OP_Goto); /* Jump to DELETE logic */ - }else if( pPk ){ - /* Construct a composite key for the row to be deleted and remember it */ - iKey = ++pParse->nMem; - nKey = 0; /* Zero tells OP_Found to use a composite key */ - sqlite3VdbeAddOp4(v, OP_MakeRecord, iPk, nPk, iKey, - sqlite3IndexAffinityStr(pParse->db, pPk), nPk); - sqlite3VdbeAddOp2(v, OP_IdxInsert, iEphCur, iKey); }else{ - /* Get the rowid of the row to be deleted and remember it in the RowSet */ - nKey = 1; /* OP_Seek always uses a single rowid */ - sqlite3VdbeAddOp2(v, OP_RowSetAdd, iRowSet, iKey); + if( pPk ){ + /* Add the PK key for this row to the temporary table */ + iKey = ++pParse->nMem; + nKey = 0; /* Zero tells OP_Found to use a composite key */ + sqlite3VdbeAddOp4(v, OP_MakeRecord, iPk, nPk, iKey, + sqlite3IndexAffinityStr(pParse->db, pPk), nPk); + sqlite3VdbeAddOp2(v, OP_IdxInsert, iEphCur, iKey); + }else{ + /* Add the rowid of the row to be deleted to the RowSet */ + nKey = 1; /* OP_Seek always uses a single rowid */ + sqlite3VdbeAddOp2(v, OP_RowSetAdd, iRowSet, iKey); + } } - /* End of the WHERE loop */ - sqlite3WhereEnd(pWInfo); - if( okOnePass ){ - /* Bypass the delete logic below if the WHERE loop found zero rows */ + /* If this DELETE cannot use the ONEPASS strategy, this is the + ** end of the WHERE loop */ + if( eOnePass ){ addrBypass = sqlite3VdbeMakeLabel(v); - sqlite3VdbeGoto(v, addrBypass); - sqlite3VdbeJumpHere(v, addrDelete); + }else{ + sqlite3WhereEnd(pWInfo); } /* Unless this is a view, open cursors for the table we are @@ -467,20 +473,21 @@ void sqlite3DeleteFrom( ** triggers. */ if( !isView ){ + int iAddrOnce; + if( eOnePass==2 ) iAddrOnce = sqlite3CodeOnce(pParse); testcase( IsVirtual(pTab) ); sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, iTabCur, aToOpen, &iDataCur, &iIdxCur); assert( pPk || IsVirtual(pTab) || iDataCur==iTabCur ); assert( pPk || IsVirtual(pTab) || iIdxCur==iDataCur+1 ); + if( eOnePass==2 ) sqlite3VdbeJumpHere(v, iAddrOnce); } /* Set up a loop over the rowids/primary-keys that were found in the ** where-clause loop above. */ - if( okOnePass ){ - /* Just one row. Hence the top-of-loop is a no-op */ + if( eOnePass ){ assert( nKey==nPk ); /* OP_Found will use an unpacked key */ - assert( !IsVirtual(pTab) ); if( aToOpen[iDataCur-iTabCur] ){ assert( pPk!=0 || pTab->pSelect!=0 ); sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, addrBypass, iKey, nKey); @@ -508,13 +515,18 @@ void sqlite3DeleteFrom( #endif { int count = (pParse->nested==0); /* True to count changes */ + int iIdxNoSeek = -1; + if( bComplex==0 && aiCurOnePass[1]!=iDataCur ){ + iIdxNoSeek = aiCurOnePass[1]; + } sqlite3GenerateRowDelete(pParse, pTab, pTrigger, iDataCur, iIdxCur, - iKey, nKey, count, OE_Default, okOnePass); + iKey, nKey, count, OE_Default, eOnePass, iIdxNoSeek); } /* End of the loop over all rowids/primary-keys. */ - if( okOnePass ){ + if( eOnePass ){ sqlite3VdbeResolveLabel(v, addrBypass); + sqlite3WhereEnd(pWInfo); }else if( pPk ){ sqlite3VdbeAddOp2(v, OP_Next, iEphCur, addrLoop+1); VdbeCoverage(v); sqlite3VdbeJumpHere(v, addrLoop); @@ -586,6 +598,25 @@ delete_from_cleanup: ** sequence of nPk memory cells starting at iPk. If nPk==0 that means ** that a search record formed from OP_MakeRecord is contained in the ** single memory location iPk. +** +** eMode: +** Parameter eMode may be passed either 0, 1 or 2. If it is passed a +** non-zero value, then it is guaranteed that cursor iDataCur already +** points to the row to delete. If it is passed 0, then this function +** must seek iDataCur to the entry identified by iPk and nPk before +** reading from it. +** +** If eMode is passed the value 2, then this call is being made as part +** of a ONEPASS delete that affects multiple rows. In this case, if +** iIdxNoSeek is a valid cursor number (>=0), then its position should +** be preserved following the delete operation. Or, if iIdxNoSeek is not +** a valid cursor number, the position of iDataCur should be preserved +** instead. +** +** iIdxNoSeek: +** If iIdxNoSeek is a valid cursor number (>=0), then it identifies an +** index cursor (from within array of cursors starting at iIdxCur) that +** already points to the index entry to be deleted. */ void sqlite3GenerateRowDelete( Parse *pParse, /* Parsing context */ @@ -597,7 +628,8 @@ void sqlite3GenerateRowDelete( i16 nPk, /* Number of PRIMARY KEY memory cells */ u8 count, /* If non-zero, increment the row change counter */ u8 onconf, /* Default ON CONFLICT policy for triggers */ - u8 bNoSeek /* iDataCur is already pointing to the row to delete */ + u8 eMode, /* See explanation above */ + int iIdxNoSeek /* Cursor number of cursor that does not need seeking */ ){ Vdbe *v = pParse->pVdbe; /* Vdbe */ int iOld = 0; /* First register in OLD.* array */ @@ -614,7 +646,7 @@ void sqlite3GenerateRowDelete( ** not attempt to delete it or fire any DELETE triggers. */ iLabel = sqlite3VdbeMakeLabel(v); opSeek = HasRowid(pTab) ? OP_NotExists : OP_NotFound; - if( !bNoSeek ){ + if( eMode==0 ){ sqlite3VdbeAddOp4Int(v, opSeek, iDataCur, iLabel, iPk, nPk); VdbeCoverageIf(v, opSeek==OP_NotExists); VdbeCoverageIf(v, opSeek==OP_NotFound); @@ -674,11 +706,15 @@ void sqlite3GenerateRowDelete( ** a view (in which case the only effect of the DELETE statement is to ** fire the INSTEAD OF triggers). */ if( pTab->pSelect==0 ){ - sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur, 0); + sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur,0,iIdxNoSeek); sqlite3VdbeAddOp2(v, OP_Delete, iDataCur, (count?OPFLAG_NCHANGE:0)); if( count ){ sqlite3VdbeChangeP4(v, -1, pTab->zName, P4_TRANSIENT); } + if( iIdxNoSeek>=0 ){ + sqlite3VdbeAddOp1(v, OP_Delete, iIdxNoSeek); + } + sqlite3VdbeChangeP5(v, eMode==2); } /* Do any ON CASCADE, SET NULL or SET DEFAULT operations required to @@ -721,7 +757,8 @@ void sqlite3GenerateRowIndexDelete( Table *pTab, /* Table containing the row to be deleted */ int iDataCur, /* Cursor of table holding data. */ int iIdxCur, /* First index cursor */ - int *aRegIdx /* Only delete if aRegIdx!=0 && aRegIdx[i]>0 */ + int *aRegIdx, /* Only delete if aRegIdx!=0 && aRegIdx[i]>0 */ + int iIdxNoSeek /* Do not delete from this cursor */ ){ int i; /* Index loop counter */ int r1 = -1; /* Register holding an index key */ @@ -737,11 +774,12 @@ void sqlite3GenerateRowIndexDelete( assert( iIdxCur+i!=iDataCur || pPk==pIdx ); if( aRegIdx!=0 && aRegIdx[i]==0 ) continue; if( pIdx==pPk ) continue; + if( iIdxCur+i==iIdxNoSeek ) continue; VdbeModuleComment((v, "GenRowIdxDel for %s", pIdx->zName)); r1 = sqlite3GenerateIndexKey(pParse, pIdx, iDataCur, 0, 1, - &iPartIdxLabel, pPrior, r1); + &iPartIdxLabel, pPrior, r1); sqlite3VdbeAddOp3(v, OP_IdxDelete, iIdxCur+i, r1, - pIdx->uniqNotNull ? pIdx->nKeyCol : pIdx->nColumn); + pIdx->uniqNotNull ? pIdx->nKeyCol : pIdx->nColumn); sqlite3ResolvePartIdxLabel(pParse, iPartIdxLabel); pPrior = pIdx; } diff --git a/src/insert.c b/src/insert.c index dc1214b337..21e1a2577d 100644 --- a/src/insert.c +++ b/src/insert.c @@ -1347,10 +1347,10 @@ void sqlite3GenerateConstraintChecks( if( pTrigger || sqlite3FkRequired(pParse, pTab, 0, 0) ){ sqlite3MultiWrite(pParse); sqlite3GenerateRowDelete(pParse, pTab, pTrigger, iDataCur, iIdxCur, - regNewData, 1, 0, OE_Replace, 1); + regNewData, 1, 0, OE_Replace, 1, -1); }else if( pTab->pIndex ){ sqlite3MultiWrite(pParse); - sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur, 0); + sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur, 0, -1); } seenReplace = 1; break; @@ -1528,7 +1528,7 @@ void sqlite3GenerateConstraintChecks( pTrigger = sqlite3TriggersExist(pParse, pTab, TK_DELETE, 0, 0); } sqlite3GenerateRowDelete(pParse, pTab, pTrigger, iDataCur, iIdxCur, - regR, nPkField, 0, OE_Replace, pIdx==pPk); + regR, nPkField, 0, OE_Replace, pIdx==pPk, -1); seenReplace = 1; break; } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index c9452b1d55..7fc3464d18 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2352,6 +2352,7 @@ struct SrcList { #define WHERE_WANT_DISTINCT 0x0400 /* All output needs to be distinct */ #define WHERE_SORTBYGROUP 0x0800 /* Support sqlite3WhereIsSorted() */ #define WHERE_REOPEN_IDX 0x1000 /* Try to use OP_ReopenIdx */ +#define WHERE_ONEPASS_MULTIROW 0x2000 /* ONEPASS is ok with multiple rows */ /* Allowed return values from sqlite3WhereIsDistinct() */ @@ -3440,8 +3441,9 @@ int sqlite3ExprIsInteger(Expr*, int*); int sqlite3ExprCanBeNull(const Expr*); int sqlite3ExprNeedsNoAffinityChange(const Expr*, char); int sqlite3IsRowid(const char*); -void sqlite3GenerateRowDelete(Parse*,Table*,Trigger*,int,int,int,i16,u8,u8,u8); -void sqlite3GenerateRowIndexDelete(Parse*, Table*, int, int, int*); +void sqlite3GenerateRowDelete( + Parse*,Table*,Trigger*,int,int,int,i16,u8,u8,u8,int); +void sqlite3GenerateRowIndexDelete(Parse*, Table*, int, int, int*, int); int sqlite3GenerateIndexKey(Parse*, Index*, int, int, int, int*,Index*,int); void sqlite3ResolvePartIdxLabel(Parse*,int); void sqlite3GenerateConstraintChecks(Parse*,Table*,int*,int,int,int,int, diff --git a/src/update.c b/src/update.c index fefb008f40..94f7a4dd99 100644 --- a/src/update.c +++ b/src/update.c @@ -587,7 +587,7 @@ void sqlite3Update( } VdbeCoverageNeverTaken(v); } - sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur, aRegIdx); + sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur, aRegIdx, -1); /* If changing the record number, delete the old record. */ if( hasFK || chngKey || pPk!=0 ){ diff --git a/src/vdbe.c b/src/vdbe.c index 2c0034cf64..fc0884b5be 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -4274,14 +4274,15 @@ case OP_InsertInt: { break; } -/* Opcode: Delete P1 P2 * P4 * +/* Opcode: Delete P1 P2 * P4 P5 ** ** Delete the record at which the P1 cursor is currently pointing. ** -** The cursor will be left pointing at either the next or the previous -** record in the table. If it is left pointing at the next record, then -** the next Next instruction will be a no-op. Hence it is OK to delete -** a record from within a Next loop. +** If the P5 parameter is non-zero, the cursor will be left pointing at +** either the next or the previous record in the table. If it is left +** pointing at the next record, then the next Next instruction will be a +** no-op. As a result, in this case it is OK to delete a record from within a +** Next loop. If P5 is zero, then the cursor is left in an undefined state. ** ** If the OPFLAG_NCHANGE flag of P2 is set, then the row change count is ** incremented (otherwise not). @@ -4301,20 +4302,26 @@ case OP_Delete: { pC = p->apCsr[pOp->p1]; assert( pC!=0 ); assert( pC->pCursor!=0 ); /* Only valid for real tables, no pseudotables */ - assert( pC->deferredMoveto==0 ); + + if( pC->deferredMoveto ){ + rc = sqlite3VdbeCursorMoveto(pC); + if( rc!=SQLITE_OK ) goto abort_due_to_error; + }else if( pOp->p5 && db->xUpdateCallback && pOp->p4.z && pC->isTable ){ + sqlite3BtreeKeySize(pC->pCursor, &pC->movetoTarget); + } #ifdef SQLITE_DEBUG /* The seek operation that positioned the cursor prior to OP_Delete will ** have also set the pC->movetoTarget field to the rowid of the row that ** is being deleted */ - if( pOp->p4.z && pC->isTable ){ + if( pOp->p4.z && pC->isTable && pOp->p5==0 ){ i64 iKey = 0; sqlite3BtreeKeySize(pC->pCursor, &iKey); assert( pC->movetoTarget==iKey ); } #endif - rc = sqlite3BtreeDelete(pC->pCursor); + rc = sqlite3BtreeDelete(pC->pCursor, pOp->p5); pC->cacheStatus = CACHE_STALE; /* Invoke the update-hook if required. */ @@ -4857,7 +4864,7 @@ case OP_IdxDelete: { #endif rc = sqlite3BtreeMovetoUnpacked(pCrsr, &r, 0, 0, &res); if( rc==SQLITE_OK && res==0 ){ - rc = sqlite3BtreeDelete(pCrsr); + rc = sqlite3BtreeDelete(pCrsr, 0); } assert( pC->deferredMoveto==0 ); pC->cacheStatus = CACHE_STALE; diff --git a/src/where.c b/src/where.c index b76b9f2e51..526e06afe5 100644 --- a/src/where.c +++ b/src/where.c @@ -3958,6 +3958,10 @@ WhereInfo *sqlite3WhereBegin( sqlite3 *db; /* Database connection */ int rc; /* Return code */ + assert( (wctrlFlags & WHERE_ONEPASS_MULTIROW)==0 || ( + (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0 + && (wctrlFlags & WHERE_OMIT_OPEN_CLOSE)==0 + )); /* Variable initialization */ db = pParse->db; @@ -4199,11 +4203,16 @@ WhereInfo *sqlite3WhereBegin( ** the statement to update or delete a single row. */ assert( (wctrlFlags & WHERE_ONEPASS_DESIRED)==0 || pWInfo->nLevel==1 ); - if( (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0 - && (pWInfo->a[0].pWLoop->wsFlags & WHERE_ONEROW)!=0 ){ - pWInfo->okOnePass = 1; - if( HasRowid(pTabList->a[0].pTab) ){ - pWInfo->a[0].pWLoop->wsFlags &= ~WHERE_IDX_ONLY; + if( (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0 ){ + int wsFlags = pWInfo->a[0].pWLoop->wsFlags; + int bOnerow = (wsFlags & WHERE_ONEROW)!=0; + if( bOnerow || ( (wctrlFlags & WHERE_ONEPASS_MULTIROW) + && 0==(wsFlags & WHERE_VIRTUALTABLE) + )){ + pWInfo->okOnePass = bOnerow ? 1 : 2; + if( HasRowid(pTabList->a[0].pTab) ){ + pWInfo->a[0].pWLoop->wsFlags &= ~WHERE_IDX_ONLY; + } } } diff --git a/test/delete.test b/test/delete.test index 47d357811b..d2dc106495 100644 --- a/test/delete.test +++ b/test/delete.test @@ -68,7 +68,6 @@ do_test delete-3.1.7 { } {1 2 4 16} integrity_check delete-3.2 - # Semantic errors in the WHERE clause # do_test delete-4.1 { diff --git a/test/delete4.test b/test/delete4.test new file mode 100644 index 0000000000..7334bf02ed --- /dev/null +++ b/test/delete4.test @@ -0,0 +1,102 @@ +# 2005 August 24 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this script is a test of the DELETE command. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix delete4 + +do_execsql_test 1.1 { + CREATE TABLE t1(x INTEGER PRIMARY KEY, y); + INSERT INTO t1 VALUES(1, 0); + INSERT INTO t1 VALUES(2, 1); + INSERT INTO t1 VALUES(3, 0); + INSERT INTO t1 VALUES(4, 1); + INSERT INTO t1 VALUES(5, 0); + INSERT INTO t1 VALUES(6, 1); + INSERT INTO t1 VALUES(7, 0); + INSERT INTO t1 VALUES(8, 1); +} +do_execsql_test 1.2 { + DELETE FROM t1 WHERE y=1; +} +do_execsql_test 1.3 { + SELECT x FROM t1; +} {1 3 5 7} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 2.1 { + CREATE TABLE t1(x INTEGER PRIMARY KEY, y, z); + INSERT INTO t1 VALUES(1, 0, randomblob(200)); + INSERT INTO t1 VALUES(2, 1, randomblob(200)); + INSERT INTO t1 VALUES(3, 0, randomblob(200)); + INSERT INTO t1 VALUES(4, 1, randomblob(200)); + INSERT INTO t1 VALUES(5, 0, randomblob(200)); + INSERT INTO t1 VALUES(6, 1, randomblob(200)); + INSERT INTO t1 VALUES(7, 0, randomblob(200)); + INSERT INTO t1 VALUES(8, 1, randomblob(200)); +} +do_execsql_test 2.2 { + DELETE FROM t1 WHERE y=1; +} +do_execsql_test 2.3 { + SELECT x FROM t1; +} {1 3 5 7} + + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 3.1 { + CREATE TABLE t1(a, b, PRIMARY KEY(a, b)) WITHOUT ROWID; + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(2, 4); + INSERT INTO t1 VALUES(1, 5); + DELETE FROM t1 WHERE a=1; + SELECT * FROM t1; +} {2 4} + +#------------------------------------------------------------------------- +# DELETE statement that uses the OR optimization +# +reset_db +do_execsql_test 3.1 { + CREATE TABLE t1(i INTEGER PRIMARY KEY, a, b); + CREATE INDEX i1a ON t1(a); + CREATE INDEX i1b ON t1(b); + INSERT INTO t1 VALUES(1, 'one', 'i'); + INSERT INTO t1 VALUES(2, 'two', 'ii'); + INSERT INTO t1 VALUES(3, 'three', 'iii'); + INSERT INTO t1 VALUES(4, 'four', 'iv'); + INSERT INTO t1 VALUES(5, 'one', 'i'); + INSERT INTO t1 VALUES(6, 'two', 'ii'); + INSERT INTO t1 VALUES(7, 'three', 'iii'); + INSERT INTO t1 VALUES(8, 'four', 'iv'); +} {} + +do_execsql_test 3.2 { + DELETE FROM t1 WHERE a='two' OR b='iv'; +} + +do_execsql_test 3.3 { + SELECT i FROM t1 ORDER BY i; +} {1 3 5 7} + +do_execsql_test 3.4 { + PRAGMA integrity_check; +} {ok} + + +finish_test diff --git a/test/indexedby.test b/test/indexedby.test index 975b2be605..83c7a5cccc 100644 --- a/test/indexedby.test +++ b/test/indexedby.test @@ -231,13 +231,13 @@ do_execsql_test indexedby-6.2 { # do_execsql_test indexedby-7.1 { EXPLAIN QUERY PLAN DELETE FROM t1 WHERE a = 5 -} {0 0 0 {SEARCH TABLE t1 USING COVERING INDEX i1 (a=?)}} +} {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (a=?)}} do_execsql_test indexedby-7.2 { EXPLAIN QUERY PLAN DELETE FROM t1 NOT INDEXED WHERE a = 5 } {0 0 0 {SCAN TABLE t1}} do_execsql_test indexedby-7.3 { EXPLAIN QUERY PLAN DELETE FROM t1 INDEXED BY i1 WHERE a = 5 -} {0 0 0 {SEARCH TABLE t1 USING COVERING INDEX i1 (a=?)}} +} {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (a=?)}} do_execsql_test indexedby-7.4 { EXPLAIN QUERY PLAN DELETE FROM t1 INDEXED BY i1 WHERE a = 5 AND b = 10 } {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (a=?)}} From 2e25a000196aff155f3bbcac1a361cd0b92f5ec5 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 12 Sep 2015 19:27:41 +0000 Subject: [PATCH 089/100] Fix API documentation typos. Emphasize that the sqlite3_config() routine is not threadsafe. FossilOrigin-Name: 786333e05a75406bbd0b5c9c7beb8f16022eabec --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/sqlite.h.in | 12 +++++++----- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index eb8d534263..2ada10a8c9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Import\scommon\schanges\sfrom\sthe\smutex\sinitialization\sbranch. -D 2015-09-12T18:57:45.818 +C Fix\sAPI\sdocumentation\stypos.\s\sEmphasize\sthat\sthe\ssqlite3_config()\sroutine\sis\nnot\sthreadsafe. +D 2015-09-12T19:27:41.193 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -342,7 +342,7 @@ F src/resolve.c 3126f7694b8ce0f97282d7dd3a5198b8fa18dce9 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c c17613385bc6b095c421b1f30548814f5fd8a9b2 F src/shell.c 6332ef06db1390ef812cfdff1fc97b4fd76cdd42 -F src/sqlite.h.in dbaf8c3796e80221de4395b5f4f872abddb5f89f +F src/sqlite.h.in b7386d2bae6a43ba35be56e0d495b8725998101e F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 64350bf36833a56ad675e27392a913f417c5c308 F src/sqliteInt.h b3e590f374b376a793b93e2387b8d5aca0fc92c4 @@ -1386,7 +1386,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 86781093bdb4c4fdedd228cb1c8961db48a483bb -R 65cef7a212785e2c4ed01bbc55ee07c1 -U mistachkin -Z 122b4ea017ab904cde327a85d32df791 +P 334720c01722478af0d3dfd6fe8bafd88ba09f49 +R 584c0e6cff093a89c297ea7dca4357da +U drh +Z 4de500fe7cfddcd5498eea834d793093 diff --git a/manifest.uuid b/manifest.uuid index 62601ecd30..1ca4768f53 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -334720c01722478af0d3dfd6fe8bafd88ba09f49 \ No newline at end of file +786333e05a75406bbd0b5c9c7beb8f16022eabec \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index d5b4f4e92d..93a624c93a 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -124,7 +124,7 @@ extern "C" { ** but are associated with the library instead of the header file. ^(Cautious ** programmers might include assert() statements in their application to ** verify that values returned by these interfaces match the macros in -** the header, and thus insure that the application is +** the header, and thus ensure that the application is ** compiled with matching library and header files. ** ** 
@@ -374,7 +374,7 @@ typedef int (*sqlite3_callback)(void*,int,char**, char**);
 ** Restrictions:
 **
 ** 
    
-** 
  •  The application must insure that the 1st parameter to sqlite3_exec()
+** 
  •  The application must ensure that the 1st parameter to sqlite3_exec()
 **      is a valid and open [database connection].
 ** 
  •  The application must not close the [database connection] specified by
 **      the 1st parameter to sqlite3_exec() while sqlite3_exec() is running.
@@ -1367,9 +1367,11 @@ int sqlite3_os_end(void);
 ** applications and so this routine is usually not necessary.  It is
 ** provided to support rare applications with unusual needs.
 **
-** The sqlite3_config() interface is not threadsafe.  The application
-** must insure that no other SQLite interfaces are invoked by other
-** threads while sqlite3_config() is running.  Furthermore, sqlite3_config()
+** The sqlite3_config() interface is not threadsafe. The application
+** must ensure that no other SQLite interfaces are invoked by other
+** threads while sqlite3_config() is running.
+**
+** The sqlite3_config() interface
 ** may only be invoked prior to library initialization using
 ** [sqlite3_initialize()] or after shutdown by [sqlite3_shutdown()].
 ** ^If sqlite3_config() is called after [sqlite3_initialize()] and before

From 7210b3d1e82c09cc32245abbe28e6db3714aa405 Mon Sep 17 00:00:00 2001
From: dan 
Date: Sat, 12 Sep 2015 19:50:58 +0000
Subject: [PATCH 090/100] Fix compiler warnings in delete.c.

FossilOrigin-Name: 0a4d285e18b78cd529483ba5c8c607ab62a443d4
---
 manifest      | 15 ++++++---------
 manifest.uuid |  2 +-
 src/delete.c  |  3 +--
 3 files changed, 8 insertions(+), 12 deletions(-)

diff --git a/manifest b/manifest
index e4c2c2f086..782e7b6568 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Experimental\schange\sto\suse\sa\ssingle-pass\sapproach\sfor\sDELETE\sstatements\son\snon-virtual\stables\sthat\sdo\snot\sfire\striggers\sor\srequire\sforiegn-key\sprocessing.
-D 2015-09-12T19:26:11.066
+C Fix\scompiler\swarnings\sin\sdelete.c.
+D 2015-09-12T19:50:58.177
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -291,7 +291,7 @@ F src/complete.c addcd8160b081131005d5bc2d34adf20c1c5c92f
 F src/ctime.c 5a0b735dc95604766f5dac73973658eef782ee8b
 F src/date.c fb1c99172017dcc8e237339132c91a21a0788584
 F src/dbstat.c e637e7a7ff40ef32132a418c6fdf1cfb63aa27c7
-F src/delete.c d5a2dc4a4663225abbcab042478dc37a5749b7d7
+F src/delete.c 18b9c93ef552adc12b7127707b10deefc0c1d451
 F src/expr.c 3a76afcdac925294c39903b7002ddb9e5fd29863
 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb
 F src/fkey.c 83e1baba999bed3144ea5a2143fc922edf51135f
@@ -1387,10 +1387,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 334720c01722478af0d3dfd6fe8bafd88ba09f49
-R 7f7d9f32d96ad1f6004a92150fc6799e
-T *branch * onepass-delete
-T *sym-onepass-delete *
-T -sym-trunk *
+P eaeb2b80f6f8f83679c8323a81bb39570ec946fe
+R eb15fa201d3086c36456da6232a12264
 U dan
-Z 9a0d15a0b8fdf4a99fe6f2640c27db22
+Z 7f9a5cfb9accd4d1fdad39fe13d7d9ec
diff --git a/manifest.uuid b/manifest.uuid
index 5a82be83bf..717bd41b6c 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-eaeb2b80f6f8f83679c8323a81bb39570ec946fe
\ No newline at end of file
+0a4d285e18b78cd529483ba5c8c607ab62a443d4
\ No newline at end of file
diff --git a/src/delete.c b/src/delete.c
index efec82aab9..472b6bb4bc 100644
--- a/src/delete.c
+++ b/src/delete.c
@@ -247,7 +247,6 @@ void sqlite3DeleteFrom(
   int iRowSet = 0;       /* Register for rowset of rows to delete */
   int addrBypass = 0;    /* Address of jump over the delete logic */
   int addrLoop = 0;      /* Top of the delete loop */
-  int addrDelete = 0;    /* Jump directly to the delete logic */
   int addrEphOpen = 0;   /* Instruction to open the Ephemeral table */
  
 #ifndef SQLITE_OMIT_TRIGGER
@@ -473,7 +472,7 @@ void sqlite3DeleteFrom(
     ** triggers.
     */
     if( !isView ){
-      int iAddrOnce;
+      int iAddrOnce = 0;
       if( eOnePass==2 ) iAddrOnce = sqlite3CodeOnce(pParse);
       testcase( IsVirtual(pTab) );
       sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, iTabCur, aToOpen,

From c215322bd7a5e0114e5dbe6772c06c6f82751b8f Mon Sep 17 00:00:00 2001
From: mistachkin 
Date: Sun, 13 Sep 2015 20:15:01 +0000
Subject: [PATCH 091/100] Add static VFS mutexes to the primary header file
 comments.

FossilOrigin-Name: 9a867d9fbe74fe2b33d55e32737a66e9a77b7466
---
 manifest        | 14 +++++++-------
 manifest.uuid   |  2 +-
 src/sqlite.h.in |  3 +++
 3 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 2ada10a8c9..7d3bf391d2 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sAPI\sdocumentation\stypos.\s\sEmphasize\sthat\sthe\ssqlite3_config()\sroutine\sis\nnot\sthreadsafe.
-D 2015-09-12T19:27:41.193
+C Add\sstatic\sVFS\smutexes\sto\sthe\sprimary\sheader\sfile\scomments.
+D 2015-09-13T20:15:01.158
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -342,7 +342,7 @@ F src/resolve.c 3126f7694b8ce0f97282d7dd3a5198b8fa18dce9
 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e
 F src/select.c c17613385bc6b095c421b1f30548814f5fd8a9b2
 F src/shell.c 6332ef06db1390ef812cfdff1fc97b4fd76cdd42
-F src/sqlite.h.in b7386d2bae6a43ba35be56e0d495b8725998101e
+F src/sqlite.h.in 50f83145c6543000b7d27525ecaec59a23d8280b
 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad
 F src/sqlite3ext.h 64350bf36833a56ad675e27392a913f417c5c308
 F src/sqliteInt.h b3e590f374b376a793b93e2387b8d5aca0fc92c4
@@ -1386,7 +1386,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 334720c01722478af0d3dfd6fe8bafd88ba09f49
-R 584c0e6cff093a89c297ea7dca4357da
-U drh
-Z 4de500fe7cfddcd5498eea834d793093
+P 786333e05a75406bbd0b5c9c7beb8f16022eabec
+R 1a3f3f98d74800359cb14e46ec6e0053
+U mistachkin
+Z 7262f2bc6346d75912550091a71aa522
diff --git a/manifest.uuid b/manifest.uuid
index 1ca4768f53..671ed1e6ca 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-786333e05a75406bbd0b5c9c7beb8f16022eabec
\ No newline at end of file
+9a867d9fbe74fe2b33d55e32737a66e9a77b7466
\ No newline at end of file
diff --git a/src/sqlite.h.in b/src/sqlite.h.in
index 93a624c93a..7a51ec109f 100644
--- a/src/sqlite.h.in
+++ b/src/sqlite.h.in
@@ -6127,6 +6127,9 @@ int sqlite3_vfs_unregister(sqlite3_vfs*);
 ** 
  •   SQLITE_MUTEX_STATIC_APP1
 ** 
  •   SQLITE_MUTEX_STATIC_APP2
 ** 
  •   SQLITE_MUTEX_STATIC_APP3
+** 
  •   SQLITE_MUTEX_STATIC_VFS1
+** 
  •   SQLITE_MUTEX_STATIC_VFS2
+** 
  •   SQLITE_MUTEX_STATIC_VFS3
 ** 

 **
 ** ^The first two constants (SQLITE_MUTEX_FAST and SQLITE_MUTEX_RECURSIVE)

From c6157e197103ef741c488014730b741e876dfe02 Mon Sep 17 00:00:00 2001
From: dan 
Date: Mon, 14 Sep 2015 09:23:47 +0000
Subject: [PATCH 092/100] Avoid calling sqlite3VdbeCursorMoveto() from more
 than one point in vdbe.c. Doing so prevents it from being inlined as part of
 OP_Column.

FossilOrigin-Name: 166d5af8914c6954fb24a06e9686f194c0d0acde
---
 manifest        | 14 +++++++-------
 manifest.uuid   |  2 +-
 src/vdbe.c      | 18 ++++++++++--------
 src/wherecode.c |  6 +++++-
 4 files changed, 23 insertions(+), 17 deletions(-)

diff --git a/manifest b/manifest
index 782e7b6568..892c86e00d 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\scompiler\swarnings\sin\sdelete.c.
-D 2015-09-12T19:50:58.177
+C Avoid\scalling\ssqlite3VdbeCursorMoveto()\sfrom\smore\sthan\sone\spoint\sin\svdbe.c.\sDoing\sso\sprevents\sit\sfrom\sbeing\sinlined\sas\spart\sof\sOP_Column.
+D 2015-09-14T09:23:47.876
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -403,7 +403,7 @@ F src/update.c eb7ab3ff2928628692a4f14be397c95f4a681d97
 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c
 F src/util.c fc612367108b74573c5fd13a85d0a23027f438bd
 F src/vacuum.c 2ddd5cad2a7b9cef7f9e431b8c7771634c6b1701
-F src/vdbe.c 5587d76bd5a4fb4f7ca023f161a6c6adbb1de26c
+F src/vdbe.c a8a5cb1126bf79104e00326abd6a7d22ac3bc4c3
 F src/vdbe.h 4bc88bd0e06f8046ee6ab7487c0015e85ad949ad
 F src/vdbeInt.h 8b867eac234e28627ffcace3cd4b4b79bbec664b
 F src/vdbeapi.c 0d890f57caf143b114a95ce699e59af51359c508
@@ -419,7 +419,7 @@ F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4
 F src/walker.c 2e14d17f592d176b6dc879c33fbdec4fbccaa2ba
 F src/where.c 98cbedead64380fc26a098350f43d92237c8fa17
 F src/whereInt.h 292d3ac90da4eab1e03ac8452f1add746bcafaa1
-F src/wherecode.c 6ac8599523f4840d9efac335329f627ebf3f79fd
+F src/wherecode.c 780cccf12a07ddc1ea0c6f6eb95895a3d8f79a6e
 F src/whereexpr.c 2473e4350e30f9b55d1c6a8f66ca23c689f23f1d
 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd
@@ -1387,7 +1387,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P eaeb2b80f6f8f83679c8323a81bb39570ec946fe
-R eb15fa201d3086c36456da6232a12264
+P 0a4d285e18b78cd529483ba5c8c607ab62a443d4
+R 96a467fbd50feb4126a3810f2f5f97d3
 U dan
-Z 7f9a5cfb9accd4d1fdad39fe13d7d9ec
+Z 9af565931700230812c8ce5da8bff453
diff --git a/manifest.uuid b/manifest.uuid
index 717bd41b6c..40f469b0aa 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-0a4d285e18b78cd529483ba5c8c607ab62a443d4
\ No newline at end of file
+166d5af8914c6954fb24a06e9686f194c0d0acde
\ No newline at end of file
diff --git a/src/vdbe.c b/src/vdbe.c
index fc0884b5be..0d1035e5a0 100644
--- a/src/vdbe.c
+++ b/src/vdbe.c
@@ -3968,9 +3968,10 @@ case OP_Found: {        /* jump, in3 */
 **
 ** P1 is the index of a cursor open on an SQL table btree (with integer
 ** keys).  P3 is an integer rowid.  If P1 does not contain a record with
-** rowid P3 then jump immediately to P2.  If P1 does contain a record
-** with rowid P3 then leave the cursor pointing at that record and fall
-** through to the next instruction.
+** rowid P3 then jump immediately to P2.  Or, if P2 is 0, raise an
+** SQLITE_CORRUPT error. If P1 does contain a record with rowid P3 then 
+** leave the cursor pointing at that record and fall through to the next
+** instruction.
 **
 ** The OP_NotFound opcode performs the same operation on index btrees
 ** (with arbitrary multi-value keys).
@@ -4008,7 +4009,10 @@ case OP_NotExists: {        /* jump, in3 */
   pC->deferredMoveto = 0;
   VdbeBranchTaken(res!=0,2);
   pC->seekResult = res;
-  if( res!=0 ) goto jump_to_p2;
+  if( res!=0 ){
+    if( pOp->p2==0 && rc==SQLITE_OK ) rc = SQLITE_CORRUPT_BKPT;
+    goto jump_to_p2;
+  }
   break;
 }
 
@@ -4302,11 +4306,9 @@ case OP_Delete: {
   pC = p->apCsr[pOp->p1];
   assert( pC!=0 );
   assert( pC->pCursor!=0 );  /* Only valid for real tables, no pseudotables */
+  assert( pC->deferredMoveto==0 );
 
-  if( pC->deferredMoveto ){
-    rc = sqlite3VdbeCursorMoveto(pC);
-    if( rc!=SQLITE_OK ) goto abort_due_to_error;
-  }else if( pOp->p5 && db->xUpdateCallback && pOp->p4.z && pC->isTable ){
+  if( pOp->p5 && db->xUpdateCallback && pOp->p4.z && pC->isTable ){
     sqlite3BtreeKeySize(pC->pCursor, &pC->movetoTarget);
   }
 
diff --git a/src/wherecode.c b/src/wherecode.c
index ecf3d3e7c4..09a4a41368 100644
--- a/src/wherecode.c
+++ b/src/wherecode.c
@@ -1069,7 +1069,11 @@ Bitmask sqlite3WhereCodeOneLoopStart(
       iRowidReg = ++pParse->nMem;
       sqlite3VdbeAddOp2(v, OP_IdxRowid, iIdxCur, iRowidReg);
       sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg);
-      sqlite3VdbeAddOp2(v, OP_Seek, iCur, iRowidReg);  /* Deferred seek */
+      if( pWInfo->okOnePass ){
+        sqlite3VdbeAddOp3(v, OP_NotExists, iCur, 0, iRowidReg);
+      }else{
+        sqlite3VdbeAddOp2(v, OP_Seek, iCur, iRowidReg);  /* Deferred seek */
+      }
     }else if( iCur!=iIdxCur ){
       Index *pPk = sqlite3PrimaryKeyIndex(pIdx->pTable);
       iRowidReg = sqlite3GetTempRange(pParse, pPk->nKeyCol);

From 66336f3755fa6f37212b6cb3218c61d44945ea78 Mon Sep 17 00:00:00 2001
From: drh 
Date: Mon, 14 Sep 2015 14:08:25 +0000
Subject: [PATCH 093/100] Fix a compiler warning and providing missing
 VdbeCoverage() calls.

FossilOrigin-Name: 2edd2e5edd60210e18db58bce1e096dd211b6ece
---
 manifest        | 19 +++++++++----------
 manifest.uuid   |  2 +-
 src/btree.c     |  2 +-
 src/delete.c    |  4 +++-
 src/wherecode.c |  1 +
 5 files changed, 15 insertions(+), 13 deletions(-)

diff --git a/manifest b/manifest
index 180c89652a..aecf048bd3 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Use\sa\ssingle-pass\sapproach\sfor\sDELETE\sstatements\son\snon-virtual\stables\sthat\sdo\snot\srequire\strigger\sor\sforeign\skey\sprocessing.
-D 2015-09-14T11:09:58.319
+C Fix\sa\scompiler\swarning\sand\sproviding\smissing\sVdbeCoverage()\scalls.
+D 2015-09-14T14:08:25.548
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -282,7 +282,7 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240
 F src/backup.c 4d9134dc988a87838c06056c89c0e8c4700a0452
 F src/bitvec.c d1f21d7d91690747881f03940584f4cc548c9d3d
 F src/btmutex.c 45a968cc85afed9b5e6cf55bf1f42f8d18107f79
-F src/btree.c 38ed0262d1c66d21bb084f086650e6106ae43d98
+F src/btree.c d31008cfbf83e3ae5cb96bae3a00f4b57f244a16
 F src/btree.h 40189aefdc2b830d25c8b58fd7d56538481bfdd7
 F src/btreeInt.h 8177c9ab90d772d6d2c6c517e05bed774b7c92c0
 F src/build.c f81380bc4d5d239c18b42982a9866a94489fd444
@@ -291,7 +291,7 @@ F src/complete.c addcd8160b081131005d5bc2d34adf20c1c5c92f
 F src/ctime.c 5a0b735dc95604766f5dac73973658eef782ee8b
 F src/date.c fb1c99172017dcc8e237339132c91a21a0788584
 F src/dbstat.c e637e7a7ff40ef32132a418c6fdf1cfb63aa27c7
-F src/delete.c 18b9c93ef552adc12b7127707b10deefc0c1d451
+F src/delete.c 33dedc277b1b7267734b41580c280ebb7610270d
 F src/expr.c 3a76afcdac925294c39903b7002ddb9e5fd29863
 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb
 F src/fkey.c 83e1baba999bed3144ea5a2143fc922edf51135f
@@ -419,7 +419,7 @@ F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4
 F src/walker.c 2e14d17f592d176b6dc879c33fbdec4fbccaa2ba
 F src/where.c 98cbedead64380fc26a098350f43d92237c8fa17
 F src/whereInt.h 292d3ac90da4eab1e03ac8452f1add746bcafaa1
-F src/wherecode.c 780cccf12a07ddc1ea0c6f6eb95895a3d8f79a6e
+F src/wherecode.c 91c50036928edf53bdfe7a2db13f2a40342b4f46
 F src/whereexpr.c 2473e4350e30f9b55d1c6a8f66ca23c689f23f1d
 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd
@@ -1387,8 +1387,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 9a867d9fbe74fe2b33d55e32737a66e9a77b7466 166d5af8914c6954fb24a06e9686f194c0d0acde
-R c69d19f4812c097d21c7641aacc08fe2
-T +closed 166d5af8914c6954fb24a06e9686f194c0d0acde
-U dan
-Z 2b1ef5f5371b990363dc017742f1aae5
+P 8b93cc5937000535c35c763c9326507a19892a6e
+R 761047ba83700d06df34cef492a4fea0
+U drh
+Z 4b77c33ded83fae554d0e60cce550d5b
diff --git a/manifest.uuid b/manifest.uuid
index bb1a8ea99e..228c9af5bf 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-8b93cc5937000535c35c763c9326507a19892a6e
\ No newline at end of file
+2edd2e5edd60210e18db58bce1e096dd211b6ece
\ No newline at end of file
diff --git a/src/btree.c b/src/btree.c
index 80aac20524..6e019aebd8 100644
--- a/src/btree.c
+++ b/src/btree.c
@@ -8117,7 +8117,7 @@ int sqlite3BtreeDelete(BtCursor *pCur, int bPreserve){
   ** before or after the deleted entry. In this case set bSkipnext to true.  */
   if( bPreserve ){
     if( !pPage->leaf 
-     || (pPage->nFree + cellSizePtr(pPage, pCell) + 2)>(pBt->usableSize*2/3)
+     || (pPage->nFree+cellSizePtr(pPage,pCell)+2)>(int)(pBt->usableSize*2/3)
     ){
       /* A b-tree rebalance will be required after deleting this entry.
       ** Save the cursor key.  */
diff --git a/src/delete.c b/src/delete.c
index 472b6bb4bc..caf73ab2b2 100644
--- a/src/delete.c
+++ b/src/delete.c
@@ -473,7 +473,9 @@ void sqlite3DeleteFrom(
     */
     if( !isView ){
       int iAddrOnce = 0;
-      if( eOnePass==2 ) iAddrOnce = sqlite3CodeOnce(pParse);
+      if( eOnePass==2 ){
+        iAddrOnce = sqlite3CodeOnce(pParse); VdbeCoverage(v);
+      }
       testcase( IsVirtual(pTab) );
       sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, iTabCur, aToOpen,
                                  &iDataCur, &iIdxCur);
diff --git a/src/wherecode.c b/src/wherecode.c
index 09a4a41368..1f65a93e63 100644
--- a/src/wherecode.c
+++ b/src/wherecode.c
@@ -1071,6 +1071,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
       sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg);
       if( pWInfo->okOnePass ){
         sqlite3VdbeAddOp3(v, OP_NotExists, iCur, 0, iRowidReg);
+        VdbeCoverage(v);
       }else{
         sqlite3VdbeAddOp2(v, OP_Seek, iCur, iRowidReg);  /* Deferred seek */
       }

From b0264eeca104038cc8d99270be0d38ab159e21a0 Mon Sep 17 00:00:00 2001
From: drh 
Date: Mon, 14 Sep 2015 14:45:50 +0000
Subject: [PATCH 094/100] Use symbolic names ONEPASS_OFF, ONEPASS_SINGLE, and
 ONEPASS_MULTI for the various modes of the one-pass optimization.

FossilOrigin-Name: 16e56bdadef903c6439a487f2683388aeeb0c625
---
 manifest        | 22 +++++++++++-----------
 manifest.uuid   |  2 +-
 src/delete.c    | 40 ++++++++++++++++++++--------------------
 src/insert.c    |  6 ++++--
 src/sqliteInt.h |  3 +++
 src/where.c     | 25 ++++++++++++++-----------
 src/whereInt.h  |  2 +-
 src/wherecode.c |  2 +-
 8 files changed, 55 insertions(+), 47 deletions(-)

diff --git a/manifest b/manifest
index aecf048bd3..44058425de 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\scompiler\swarning\sand\sproviding\smissing\sVdbeCoverage()\scalls.
-D 2015-09-14T14:08:25.548
+C Use\ssymbolic\snames\sONEPASS_OFF,\sONEPASS_SINGLE,\sand\sONEPASS_MULTI\sfor\sthe\nvarious\smodes\sof\sthe\sone-pass\soptimization.
+D 2015-09-14T14:45:50.973
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -291,7 +291,7 @@ F src/complete.c addcd8160b081131005d5bc2d34adf20c1c5c92f
 F src/ctime.c 5a0b735dc95604766f5dac73973658eef782ee8b
 F src/date.c fb1c99172017dcc8e237339132c91a21a0788584
 F src/dbstat.c e637e7a7ff40ef32132a418c6fdf1cfb63aa27c7
-F src/delete.c 33dedc277b1b7267734b41580c280ebb7610270d
+F src/delete.c 249a287682dd6e7ab362dd81e4d65abe2d5c3ce4
 F src/expr.c 3a76afcdac925294c39903b7002ddb9e5fd29863
 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb
 F src/fkey.c 83e1baba999bed3144ea5a2143fc922edf51135f
@@ -300,7 +300,7 @@ F src/global.c 508e4087f7b41d688e4762dcf4d4fe28cfbc87f9
 F src/hash.c 4263fbc955f26c2e8cdc0cf214bc42435aa4e4f5
 F src/hash.h c8f3c31722cf3277d03713909761e152a5b81094
 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08
-F src/insert.c db8a34cf8ba600ac1cebb3c03e93c92154d0fc4c
+F src/insert.c ace93aa254f86b1bddb20eb9bf382f6b851bbda5
 F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d
 F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e
 F src/lempar.c d344a95d60c24e2f490ee59db9784b1b17439012
@@ -345,7 +345,7 @@ F src/shell.c 6332ef06db1390ef812cfdff1fc97b4fd76cdd42
 F src/sqlite.h.in 50f83145c6543000b7d27525ecaec59a23d8280b
 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad
 F src/sqlite3ext.h 64350bf36833a56ad675e27392a913f417c5c308
-F src/sqliteInt.h 91bf09de55402157d1476a61df46ef6cfbc0bbc3
+F src/sqliteInt.h 5afc6e50402be1e0a870f28e1cd8b32eb9db590f
 F src/sqliteLimit.h 216557999cb45f2e3578ed53ebefe228d779cb46
 F src/status.c f266ad8a2892d659b74f0f50cb6a88b6e7c12179
 F src/table.c 51b46b2a62d1b3a959633d593b89bab5e2c9155e
@@ -417,9 +417,9 @@ F src/vxworks.h c18586c8edc1bddbc15c004fa16aeb1e1342b4fb
 F src/wal.c 18b0ed49830cf04fe2d68224b41838a73ac6cd24
 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4
 F src/walker.c 2e14d17f592d176b6dc879c33fbdec4fbccaa2ba
-F src/where.c 98cbedead64380fc26a098350f43d92237c8fa17
-F src/whereInt.h 292d3ac90da4eab1e03ac8452f1add746bcafaa1
-F src/wherecode.c 91c50036928edf53bdfe7a2db13f2a40342b4f46
+F src/where.c 882fb44b36201fafc32dd7d59366f852806b7e70
+F src/whereInt.h 7892bb54cf9ca0ae5c7e6094491b94c9286dc647
+F src/wherecode.c 186b493599000e640203be0a441223b395dabd24
 F src/whereexpr.c 2473e4350e30f9b55d1c6a8f66ca23c689f23f1d
 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd
@@ -1387,7 +1387,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 8b93cc5937000535c35c763c9326507a19892a6e
-R 761047ba83700d06df34cef492a4fea0
+P 2edd2e5edd60210e18db58bce1e096dd211b6ece
+R 664fd3c2c4c1207a1dc899fd67c056b8
 U drh
-Z 4b77c33ded83fae554d0e60cce550d5b
+Z ed742f4af7607da542ef263fbb0dcf2e
diff --git a/manifest.uuid b/manifest.uuid
index 228c9af5bf..9156fc088c 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-2edd2e5edd60210e18db58bce1e096dd211b6ece
\ No newline at end of file
+16e56bdadef903c6439a487f2683388aeeb0c625
\ No newline at end of file
diff --git a/src/delete.c b/src/delete.c
index caf73ab2b2..7c2f3fcd00 100644
--- a/src/delete.c
+++ b/src/delete.c
@@ -235,7 +235,7 @@ void sqlite3DeleteFrom(
   int iDb;               /* Database number */
   int memCnt = -1;       /* Memory cell used for change counting */
   int rcauth;            /* Value returned by authorization callback */
-  int eOnePass;          /* Non-zero for one-pass algorithm without the FIFO */
+  int eOnePass;          /* ONEPASS_OFF or _SINGLE or _MULTI */
   int aiCurOnePass[2];   /* The write cursors opened by WHERE_ONEPASS */
   u8 *aToOpen = 0;       /* Open cursor iTabCur+j if aToOpen[j] is true */
   Index *pPk;            /* The PRIMARY KEY index on the table */
@@ -400,14 +400,14 @@ void sqlite3DeleteFrom(
     ** to be deleted, based on the WHERE clause. Set variable eOnePass
     ** to indicate the strategy used to implement this delete:
     **
-    **   0: Two-pass approach - use a FIFO for rowids/PK values.
-    **   1: One-pass approach - at most one row deleted.
-    **   2: One-pass approach - any number of rows may be deleted.
+    **  ONEPASS_OFF:    Two-pass approach - use a FIFO for rowids/PK values.
+    **  ONEPASS_SINGLE: One-pass approach - at most one row deleted.
+    **  ONEPASS_MULTI:  One-pass approach - any number of rows may be deleted.
     */
     pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, 0, 0, wcf, iTabCur+1);
     if( pWInfo==0 ) goto delete_from_cleanup;
     eOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass);
-    assert( IsVirtual(pTab)==0 || eOnePass==0 );
+    assert( IsVirtual(pTab)==0 || eOnePass==ONEPASS_OFF );
   
     /* Keep track of the number of rows to be deleted */
     if( db->flags & SQLITE_CountRows ){
@@ -428,7 +428,7 @@ void sqlite3DeleteFrom(
       if( iKey>pParse->nMem ) pParse->nMem = iKey;
     }
   
-    if( eOnePass ){
+    if( eOnePass!=ONEPASS_OFF ){
       /* For ONEPASS, no need to store the rowid/primary-key. There is only
       ** one, so just keep it in its register(s) and fall through to the
       ** delete code.  */
@@ -460,7 +460,7 @@ void sqlite3DeleteFrom(
   
     /* If this DELETE cannot use the ONEPASS strategy, this is the 
     ** end of the WHERE loop */
-    if( eOnePass ){
+    if( eOnePass!=ONEPASS_OFF ){
       addrBypass = sqlite3VdbeMakeLabel(v);
     }else{
       sqlite3WhereEnd(pWInfo);
@@ -473,7 +473,7 @@ void sqlite3DeleteFrom(
     */
     if( !isView ){
       int iAddrOnce = 0;
-      if( eOnePass==2 ){
+      if( eOnePass==ONEPASS_MULTI ){
         iAddrOnce = sqlite3CodeOnce(pParse); VdbeCoverage(v);
       }
       testcase( IsVirtual(pTab) );
@@ -481,13 +481,13 @@ void sqlite3DeleteFrom(
                                  &iDataCur, &iIdxCur);
       assert( pPk || IsVirtual(pTab) || iDataCur==iTabCur );
       assert( pPk || IsVirtual(pTab) || iIdxCur==iDataCur+1 );
-      if( eOnePass==2 ) sqlite3VdbeJumpHere(v, iAddrOnce);
+      if( eOnePass==ONEPASS_MULTI ) sqlite3VdbeJumpHere(v, iAddrOnce);
     }
   
     /* Set up a loop over the rowids/primary-keys that were found in the
     ** where-clause loop above.
     */
-    if( eOnePass ){
+    if( eOnePass!=ONEPASS_OFF ){
       assert( nKey==nPk );  /* OP_Found will use an unpacked key */
       if( aToOpen[iDataCur-iTabCur] ){
         assert( pPk!=0 || pTab->pSelect!=0 );
@@ -525,7 +525,7 @@ void sqlite3DeleteFrom(
     }
   
     /* End of the loop over all rowids/primary-keys. */
-    if( eOnePass ){
+    if( eOnePass!=ONEPASS_OFF ){
       sqlite3VdbeResolveLabel(v, addrBypass);
       sqlite3WhereEnd(pWInfo);
     }else if( pPk ){
@@ -601,13 +601,13 @@ delete_from_cleanup:
 **       single memory location iPk.
 **
 ** eMode:
-**   Parameter eMode may be passed either 0, 1 or 2. If it is passed a 
-**   non-zero value, then it is guaranteed that cursor iDataCur already 
-**   points to the row to delete. If it is passed 0, then this function 
-**   must seek iDataCur to the entry identified by iPk and nPk before
-**   reading from it.
+**   Parameter eMode may be passed either ONEPASS_OFF (0), ONEPASS_SINGLE, or
+**   ONEPASS_MULTI.  If eMode is not ONEPASS_OFF, then the cursor
+**   iDataCur already points to the row to delete. If eMode is ONEPASS_OFF
+**   then this function must seek iDataCur to the entry identified by iPk
+**   and nPk before reading from it.
 **
-**   If eMode is passed the value 2, then this call is being made as part
+**   If eMode is ONEPASS_MULTI, then this call is being made as part
 **   of a ONEPASS delete that affects multiple rows. In this case, if 
 **   iIdxNoSeek is a valid cursor number (>=0), then its position should
 **   be preserved following the delete operation. Or, if iIdxNoSeek is not
@@ -629,7 +629,7 @@ void sqlite3GenerateRowDelete(
   i16 nPk,           /* Number of PRIMARY KEY memory cells */
   u8 count,          /* If non-zero, increment the row change counter */
   u8 onconf,         /* Default ON CONFLICT policy for triggers */
-  u8 eMode,          /* See explanation above */
+  u8 eMode,          /* ONEPASS_OFF, _SINGLE, or _MULTI.  See above */
   int iIdxNoSeek     /* Cursor number of cursor that does not need seeking */
 ){
   Vdbe *v = pParse->pVdbe;        /* Vdbe */
@@ -647,7 +647,7 @@ void sqlite3GenerateRowDelete(
   ** not attempt to delete it or fire any DELETE triggers.  */
   iLabel = sqlite3VdbeMakeLabel(v);
   opSeek = HasRowid(pTab) ? OP_NotExists : OP_NotFound;
-  if( eMode==0 ){
+  if( eMode==ONEPASS_OFF ){
     sqlite3VdbeAddOp4Int(v, opSeek, iDataCur, iLabel, iPk, nPk);
     VdbeCoverageIf(v, opSeek==OP_NotExists);
     VdbeCoverageIf(v, opSeek==OP_NotFound);
@@ -715,7 +715,7 @@ void sqlite3GenerateRowDelete(
     if( iIdxNoSeek>=0 ){
       sqlite3VdbeAddOp1(v, OP_Delete, iIdxNoSeek);
     }
-    sqlite3VdbeChangeP5(v, eMode==2);
+    sqlite3VdbeChangeP5(v, eMode==ONEPASS_MULTI);
   }
 
   /* Do any ON CASCADE, SET NULL or SET DEFAULT operations required to
diff --git a/src/insert.c b/src/insert.c
index 21e1a2577d..6132778355 100644
--- a/src/insert.c
+++ b/src/insert.c
@@ -1347,7 +1347,8 @@ void sqlite3GenerateConstraintChecks(
         if( pTrigger || sqlite3FkRequired(pParse, pTab, 0, 0) ){
           sqlite3MultiWrite(pParse);
           sqlite3GenerateRowDelete(pParse, pTab, pTrigger, iDataCur, iIdxCur,
-                                   regNewData, 1, 0, OE_Replace, 1, -1);
+                                   regNewData, 1, 0, OE_Replace,
+                                   ONEPASS_SINGLE, -1);
         }else if( pTab->pIndex ){
           sqlite3MultiWrite(pParse);
           sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur, 0, -1);
@@ -1528,7 +1529,8 @@ void sqlite3GenerateConstraintChecks(
           pTrigger = sqlite3TriggersExist(pParse, pTab, TK_DELETE, 0, 0);
         }
         sqlite3GenerateRowDelete(pParse, pTab, pTrigger, iDataCur, iIdxCur,
-            regR, nPkField, 0, OE_Replace, pIdx==pPk, -1);
+            regR, nPkField, 0, OE_Replace,
+            (pIdx==pPk ? ONEPASS_SINGLE : ONEPASS_OFF), -1);
         seenReplace = 1;
         break;
       }
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index 7fc3464d18..747f19b39a 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -3381,6 +3381,9 @@ int sqlite3WhereIsSorted(WhereInfo*);
 int sqlite3WhereContinueLabel(WhereInfo*);
 int sqlite3WhereBreakLabel(WhereInfo*);
 int sqlite3WhereOkOnePass(WhereInfo*, int*);
+#define ONEPASS_OFF      0        /* Use of ONEPASS not allowed */
+#define ONEPASS_SINGLE   1        /* ONEPASS valid for a single row update */
+#define ONEPASS_MULTI    2        /* ONEPASS is valid for multiple rows */
 void sqlite3ExprCodeLoadIndexColumn(Parse*, Index*, int, int, int);
 int sqlite3ExprCodeGetColumn(Parse*, Table*, int, int, int, u8);
 void sqlite3ExprCodeGetColumnOfTable(Vdbe*, Table*, int, int, int);
diff --git a/src/where.c b/src/where.c
index 526e06afe5..69b7c064e8 100644
--- a/src/where.c
+++ b/src/where.c
@@ -69,9 +69,11 @@ int sqlite3WhereBreakLabel(WhereInfo *pWInfo){
 }
 
 /*
-** Return TRUE if an UPDATE or DELETE statement can operate directly on
-** the rowids returned by a WHERE clause.  Return FALSE if doing an
-** UPDATE or DELETE might change subsequent WHERE clause results.
+** Return ONEPASS_OFF (0) if an UPDATE or DELETE statement is unable to
+** operate directly on the rowis returned by a WHERE clause.  Return
+** ONEPASS_SINGLE (1) if the statement can operation directly because only
+** a single row is to be changed.  Return ONEPASS_MULTI (2) if the one-pass
+** optimization can be used on multiple 
 **
 ** If the ONEPASS optimization is used (if this routine returns true)
 ** then also write the indices of open cursors used by ONEPASS
@@ -85,7 +87,7 @@ int sqlite3WhereBreakLabel(WhereInfo *pWInfo){
 */
 int sqlite3WhereOkOnePass(WhereInfo *pWInfo, int *aiCur){
   memcpy(aiCur, pWInfo->aiCurOnePass, sizeof(int)*2);
-  return pWInfo->okOnePass;
+  return pWInfo->eOnePass;
 }
 
 /*
@@ -4017,6 +4019,7 @@ WhereInfo *sqlite3WhereBegin(
   pWInfo->iBreak = pWInfo->iContinue = sqlite3VdbeMakeLabel(v);
   pWInfo->wctrlFlags = wctrlFlags;
   pWInfo->savedNQueryLoop = pParse->nQueryLoop;
+  assert( pWInfo->eOnePass==ONEPASS_OFF );  /* ONEPASS defaults to OFF */
   pMaskSet = &pWInfo->sMaskSet;
   sWLB.pWInfo = pWInfo;
   sWLB.pWC = &pWInfo->sWC;
@@ -4209,7 +4212,7 @@ WhereInfo *sqlite3WhereBegin(
     if( bOnerow || ( (wctrlFlags & WHERE_ONEPASS_MULTIROW)
        && 0==(wsFlags & WHERE_VIRTUALTABLE)
     )){
-      pWInfo->okOnePass = bOnerow ? 1 : 2;
+      pWInfo->eOnePass = bOnerow ? ONEPASS_SINGLE : ONEPASS_MULTI;
       if( HasRowid(pTabList->a[0].pTab) ){
         pWInfo->a[0].pWLoop->wsFlags &= ~WHERE_IDX_ONLY;
       }
@@ -4243,15 +4246,15 @@ WhereInfo *sqlite3WhereBegin(
     if( (pLoop->wsFlags & WHERE_IDX_ONLY)==0
          && (wctrlFlags & WHERE_OMIT_OPEN_CLOSE)==0 ){
       int op = OP_OpenRead;
-      if( pWInfo->okOnePass ){
+      if( pWInfo->eOnePass!=ONEPASS_OFF ){
         op = OP_OpenWrite;
         pWInfo->aiCurOnePass[0] = pTabItem->iCursor;
       };
       sqlite3OpenTable(pParse, pTabItem->iCursor, iDb, pTab, op);
       assert( pTabItem->iCursor==pLevel->iTabCur );
-      testcase( !pWInfo->okOnePass && pTab->nCol==BMS-1 );
-      testcase( !pWInfo->okOnePass && pTab->nCol==BMS );
-      if( !pWInfo->okOnePass && pTab->nColeOnePass==ONEPASS_OFF && pTab->nCol==BMS-1 );
+      testcase( pWInfo->eOnePass==ONEPASS_OFF && pTab->nCol==BMS );
+      if( pWInfo->eOnePass==ONEPASS_OFF && pTab->nColcolUsed;
         int n = 0;
         for(; b; b=b>>1, n++){}
@@ -4279,7 +4282,7 @@ WhereInfo *sqlite3WhereBegin(
         ** WITHOUT ROWID table.  No need for a separate index */
         iIndexCur = pLevel->iTabCur;
         op = 0;
-      }else if( pWInfo->okOnePass ){
+      }else if( pWInfo->eOnePass!=ONEPASS_OFF ){
         Index *pJ = pTabItem->pTab->pIndex;
         iIndexCur = iIdxCur;
         assert( wctrlFlags & WHERE_ONEPASS_DESIRED );
@@ -4487,7 +4490,7 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){
      && (pWInfo->wctrlFlags & WHERE_OMIT_OPEN_CLOSE)==0
     ){
       int ws = pLoop->wsFlags;
-      if( !pWInfo->okOnePass && (ws & WHERE_IDX_ONLY)==0 ){
+      if( pWInfo->eOnePass==ONEPASS_OFF && (ws & WHERE_IDX_ONLY)==0 ){
         sqlite3VdbeAddOp1(v, OP_Close, pTabItem->iCursor);
       }
       if( (ws & WHERE_INDEXED)!=0
diff --git a/src/whereInt.h b/src/whereInt.h
index 65c1180048..cae09acc82 100644
--- a/src/whereInt.h
+++ b/src/whereInt.h
@@ -412,7 +412,7 @@ struct WhereInfo {
   u16 wctrlFlags;           /* Flags originally passed to sqlite3WhereBegin() */
   i8 nOBSat;                /* Number of ORDER BY terms satisfied by indices */
   u8 sorted;                /* True if really sorted (not just grouped) */
-  u8 okOnePass;             /* Ok to use one-pass algorithm for UPDATE/DELETE */
+  u8 eOnePass;              /* ONEPASS_OFF, or _SINGLE, or _MULTI */
   u8 untestedTerms;         /* Not all WHERE terms resolved by outer loop */
   u8 eDistinct;             /* One of the WHERE_DISTINCT_* values below */
   u8 nLevel;                /* Number of nested loop */
diff --git a/src/wherecode.c b/src/wherecode.c
index 1f65a93e63..4fb6084018 100644
--- a/src/wherecode.c
+++ b/src/wherecode.c
@@ -1069,7 +1069,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
       iRowidReg = ++pParse->nMem;
       sqlite3VdbeAddOp2(v, OP_IdxRowid, iIdxCur, iRowidReg);
       sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg);
-      if( pWInfo->okOnePass ){
+      if( pWInfo->eOnePass!=ONEPASS_OFF ){
         sqlite3VdbeAddOp3(v, OP_NotExists, iCur, 0, iRowidReg);
         VdbeCoverage(v);
       }else{

From b1ff90673c0641b21bd32e7b90c0ee81ec745ad1 Mon Sep 17 00:00:00 2001
From: drh 
Date: Mon, 14 Sep 2015 14:49:23 +0000
Subject: [PATCH 095/100] Fix a typo in an SQLITE_USER_AUTHENTICATION macro.

FossilOrigin-Name: 379455af9fdfb299a857d46f35f0a76ed6daa08a
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/build.c   |  2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 44058425de..d135901a66 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Use\ssymbolic\snames\sONEPASS_OFF,\sONEPASS_SINGLE,\sand\sONEPASS_MULTI\sfor\sthe\nvarious\smodes\sof\sthe\sone-pass\soptimization.
-D 2015-09-14T14:45:50.973
+C Fix\sa\stypo\sin\san\sSQLITE_USER_AUTHENTICATION\smacro.
+D 2015-09-14T14:49:23.201
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -285,7 +285,7 @@ F src/btmutex.c 45a968cc85afed9b5e6cf55bf1f42f8d18107f79
 F src/btree.c d31008cfbf83e3ae5cb96bae3a00f4b57f244a16
 F src/btree.h 40189aefdc2b830d25c8b58fd7d56538481bfdd7
 F src/btreeInt.h 8177c9ab90d772d6d2c6c517e05bed774b7c92c0
-F src/build.c f81380bc4d5d239c18b42982a9866a94489fd444
+F src/build.c 8a86f4203ac8a9ac0734f242a96f043edffb6018
 F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0
 F src/complete.c addcd8160b081131005d5bc2d34adf20c1c5c92f
 F src/ctime.c 5a0b735dc95604766f5dac73973658eef782ee8b
@@ -1387,7 +1387,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 2edd2e5edd60210e18db58bce1e096dd211b6ece
-R 664fd3c2c4c1207a1dc899fd67c056b8
+P 16e56bdadef903c6439a487f2683388aeeb0c625
+R f8fa410ffd0eb073e2e43d46c84640d3
 U drh
-Z ed742f4af7607da542ef263fbb0dcf2e
+Z 5de1b91d25cb0490451927525818647a
diff --git a/manifest.uuid b/manifest.uuid
index 9156fc088c..266475face 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-16e56bdadef903c6439a487f2683388aeeb0c625
\ No newline at end of file
+379455af9fdfb299a857d46f35f0a76ed6daa08a
\ No newline at end of file
diff --git a/src/build.c b/src/build.c
index 9635920ea1..eee62ee151 100644
--- a/src/build.c
+++ b/src/build.c
@@ -374,7 +374,7 @@ Table *sqlite3LocateTable(
     }
     pParse->checkSchema = 1;
   }
-#if SQLITE_USER_AUTHENICATION
+#if SQLITE_USER_AUTHENTICATION
   else if( pParse->db->auth.authLevel
Date: Mon, 14 Sep 2015 19:26:37 +0000
Subject: [PATCH 096/100] Testability improvements for the ONEPASS_MULTI
 enhancement.

FossilOrigin-Name: d2df93f26fc0cf6fe01acfeaf2946972e9d8cca2
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/vdbe.c    | 15 +++++++++++----
 3 files changed, 18 insertions(+), 11 deletions(-)

diff --git a/manifest b/manifest
index d135901a66..836476cb10 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\stypo\sin\san\sSQLITE_USER_AUTHENTICATION\smacro.
-D 2015-09-14T14:49:23.201
+C Testability\simprovements\sfor\sthe\sONEPASS_MULTI\senhancement.
+D 2015-09-14T19:26:37.988
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -403,7 +403,7 @@ F src/update.c eb7ab3ff2928628692a4f14be397c95f4a681d97
 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c
 F src/util.c fc612367108b74573c5fd13a85d0a23027f438bd
 F src/vacuum.c 2ddd5cad2a7b9cef7f9e431b8c7771634c6b1701
-F src/vdbe.c a8a5cb1126bf79104e00326abd6a7d22ac3bc4c3
+F src/vdbe.c 5048103aa6bc9c01bf1a4620729256d1e873bb6c
 F src/vdbe.h 4bc88bd0e06f8046ee6ab7487c0015e85ad949ad
 F src/vdbeInt.h 8b867eac234e28627ffcace3cd4b4b79bbec664b
 F src/vdbeapi.c 0d890f57caf143b114a95ce699e59af51359c508
@@ -1387,7 +1387,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 16e56bdadef903c6439a487f2683388aeeb0c625
-R f8fa410ffd0eb073e2e43d46c84640d3
+P 379455af9fdfb299a857d46f35f0a76ed6daa08a
+R 240d53905959924a39a1969f0c2c4fad
 U drh
-Z 5de1b91d25cb0490451927525818647a
+Z db76ee6590838fdbed88f2282d6173bf
diff --git a/manifest.uuid b/manifest.uuid
index 266475face..93b327f2ec 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-379455af9fdfb299a857d46f35f0a76ed6daa08a
\ No newline at end of file
+d2df93f26fc0cf6fe01acfeaf2946972e9d8cca2
\ No newline at end of file
diff --git a/src/vdbe.c b/src/vdbe.c
index 0d1035e5a0..23338879c5 100644
--- a/src/vdbe.c
+++ b/src/vdbe.c
@@ -4003,6 +4003,7 @@ case OP_NotExists: {        /* jump, in3 */
   res = 0;
   iKey = pIn3->u.i;
   rc = sqlite3BtreeMovetoUnpacked(pCrsr, 0, iKey, 0, &res);
+  assert( rc==SQLITE_OK || res==0 );
   pC->movetoTarget = iKey;  /* Used by OP_Delete */
   pC->nullRow = 0;
   pC->cacheStatus = CACHE_STALE;
@@ -4010,8 +4011,12 @@ case OP_NotExists: {        /* jump, in3 */
   VdbeBranchTaken(res!=0,2);
   pC->seekResult = res;
   if( res!=0 ){
-    if( pOp->p2==0 && rc==SQLITE_OK ) rc = SQLITE_CORRUPT_BKPT;
-    goto jump_to_p2;
+    assert( rc==SQLITE_OK );
+    if( pOp->p2==0 ){
+      rc = SQLITE_CORRUPT_BKPT;
+    }else{
+      goto jump_to_p2;
+    }
   }
   break;
 }
@@ -4301,6 +4306,7 @@ case OP_InsertInt: {
 */
 case OP_Delete: {
   VdbeCursor *pC;
+  u8 hasUpdateCallback;
 
   assert( pOp->p1>=0 && pOp->p1nCursor );
   pC = p->apCsr[pOp->p1];
@@ -4308,7 +4314,8 @@ case OP_Delete: {
   assert( pC->pCursor!=0 );  /* Only valid for real tables, no pseudotables */
   assert( pC->deferredMoveto==0 );
 
-  if( pOp->p5 && db->xUpdateCallback && pOp->p4.z && pC->isTable ){
+  hasUpdateCallback = db->xUpdateCallback && pOp->p4.z && pC->isTable;
+  if( pOp->p5 && hasUpdateCallback ){
     sqlite3BtreeKeySize(pC->pCursor, &pC->movetoTarget);
   }
 
@@ -4327,7 +4334,7 @@ case OP_Delete: {
   pC->cacheStatus = CACHE_STALE;
 
   /* Invoke the update-hook if required. */
-  if( rc==SQLITE_OK && db->xUpdateCallback && pOp->p4.z && pC->isTable ){
+  if( rc==SQLITE_OK && hasUpdateCallback ){
     db->xUpdateCallback(db->pUpdateArg, SQLITE_DELETE,
                         db->aDb[pC->iDb].zName, pOp->p4.z, pC->movetoTarget);
     assert( pC->iDb>=0 );

From dafeda754b98da699abbcee87728b67e9b009917 Mon Sep 17 00:00:00 2001
From: dan 
Date: Mon, 14 Sep 2015 19:51:05 +0000
Subject: [PATCH 097/100] Improve test coverage of fts5_index.c.

FossilOrigin-Name: c1f76686cee3918b1be785a4071d68cb3afda0ef
---
 ext/fts5/fts5_index.c           |  33 ++----
 ext/fts5/test/fts5corrupt3.test | 196 ++++++++++++++++++++++++++++++--
 ext/fts5/test/fts5dlidx.test    |  57 ++++++++++
 ext/fts5/test/fts5fault7.test   |  48 +++++++-
 manifest                        |  20 ++--
 manifest.uuid                   |   2 +-
 6 files changed, 313 insertions(+), 43 deletions(-)

diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index e36a5388fd..c9fb6207ec 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -1427,7 +1427,6 @@ static void fts5SegIterNextPage(
   fts5DataRelease(pIter->pLeaf);
   pIter->iLeafPgno++;
   if( pIter->pNextLeaf ){
-    assert( pIter->iLeafPgno<=pSeg->pgnoLast );
     pIter->pLeaf = pIter->pNextLeaf;
     pIter->pNextLeaf = 0;
   }else if( pIter->iLeafPgno<=pSeg->pgnoLast ){
@@ -1659,10 +1658,8 @@ static void fts5SegIterReverseNewPage(Fts5Index *p, Fts5SegIter *pIter){
     ));
     if( pNew ){
       if( pIter->iLeafPgno==pIter->iTermLeafPgno ){
-        if( pIter->iTermLeafOffsetszLeaf ){
-          pIter->pLeaf = pNew;
-          pIter->iLeafOffset = pIter->iTermLeafOffset;
-        }
+        pIter->pLeaf = pNew;
+        pIter->iLeafOffset = pIter->iTermLeafOffset;
       }else{
         int iRowidOff;
         iRowidOff = fts5LeafFirstRowidOff(pNew);
@@ -3168,7 +3165,7 @@ static void fts5WriteAppendTerm(
   Fts5PageWriter *pPage = &pWriter->writer;
   Fts5Buffer *pPgidx = &pWriter->writer.pgidx;
 
-  if( p->rc ) return;
+  assert( p->rc==SQLITE_OK );
   assert( pPage->buf.n>=4 );
   assert( pPage->buf.n>4 || pWriter->bFirstTermInPage );
 
@@ -3315,16 +3312,12 @@ static void fts5WriteFinish(
   int i;
   Fts5PageWriter *pLeaf = &pWriter->writer;
   if( p->rc==SQLITE_OK ){
-    if( pLeaf->pgno==1 && pLeaf->buf.n==0 ){
-      *pnLeaf = 0;
-    }else{
-      if( pLeaf->buf.n>4 ){
-        fts5WriteFlushLeaf(p, pWriter);
-      }
-      *pnLeaf = pLeaf->pgno-1;
-
-      fts5WriteFlushBtree(p, pWriter);
+    assert( pLeaf->pgno>=1 );
+    if( pLeaf->buf.n>4 ){
+      fts5WriteFlushLeaf(p, pWriter);
     }
+    *pnLeaf = pLeaf->pgno-1;
+    fts5WriteFlushBtree(p, pWriter);
   }
   fts5BufferFree(&pLeaf->term);
   fts5BufferFree(&pLeaf->buf);
@@ -3763,9 +3756,8 @@ static void fts5FlushOneHash(Fts5Index *p){
       sqlite3Fts5HashScanEntry(pHash, &zTerm, &pDoclist, &nDoclist);
       fts5WriteAppendTerm(p, &writer, strlen(zTerm), (const u8*)zTerm);
 
-      if( writer.bFirstRowidInPage==0 
-       && pgsz>=(pBuf->n + pPgidx->n + nDoclist + 1) 
-      ){
+      assert( writer.bFirstRowidInPage==0 );
+      if( pgsz>=(pBuf->n + pPgidx->n + nDoclist + 1) ){
         /* The entire doclist will fit on the current leaf. */
         fts5BufferSafeAppendBlob(pBuf, pDoclist, nDoclist);
       }else{
@@ -3773,8 +3765,6 @@ static void fts5FlushOneHash(Fts5Index *p){
         i64 iDelta = 0;
         int iOff = 0;
 
-        /*  writer.bFirstRowidInPage = 0; */
-
         /* The entire doclist will not fit on this leaf. The following 
         ** loop iterates through the poslists that make up the current 
         ** doclist.  */
@@ -4155,7 +4145,7 @@ static void fts5SetupPrefixIter(
       i64 iRowid = fts5MultiIterRowid(p1);
       int nTerm;
       const u8 *pTerm = fts5MultiIterTerm(p1, &nTerm);
-      assert( memcmp(pToken, pTerm, MIN(nToken, nTerm))<=0 );
+      assert_nc( memcmp(pToken, pTerm, MIN(nToken, nTerm))<=0 );
       if( nTerm0 && iRowid<=iLastRowid ){
@@ -5290,7 +5280,6 @@ static int fts5DecodeDoclist(int *pRc, Fts5Buffer *pBuf, const u8 *a, int n){
     if( iOff100"] {
+    incr i
+    do_test $tn.$i {
+      db eval BEGIN
+      db eval {DELETE FROM t1_data WHERE rowid = $::rowid}
+      list [
+        catch { db eval {SELECT rowid FROM t1 WHERE t1 MATCH 'x*'} } msg
+      ] $msg
+    } {1 {database disk image is malformed}}
+    catch { db eval ROLLBACK }
+  }
+}
+
+do_3_test 3.2
+
+do_execsql_test 3.3 {
+  INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
+  INSERT INTO t1 SELECT x FROM t1;
+  INSERT INTO t1(t1) VALUES('optimize');
+} {}
+
+do_3_test 3.4
+
+do_test 3.5 {
+  execsql { 
+    DELETE FROM t1;
+    INSERT INTO t1(t1, rank) VALUES('pgsz', 40);
+  }
+  for {set i 0} {$i < 1000} {incr i} {
+    set rnd [expr int(rand() * 1000)]
+    set doc [string repeat "x$rnd " [expr int(rand() * 3) + 1]]
+    execsql { INSERT INTO t1(rowid, x) VALUES($i, $doc) }
+  }
+} {}
+
+do_3_test 3.6
+
+do_test 3.7 {
+  execsql {
+    INSERT INTO t1(t1, rank) VALUES('pgsz', 40);
+    INSERT INTO t1 SELECT x FROM t1;
+    INSERT INTO t1(t1) VALUES('optimize');
+  }
+} {}
+
+do_3_test 3.8
+
+do_test 3.9 {
+  execsql { 
+    DELETE FROM t1;
+    INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
+  }
+  for {set i 0} {$i < 100} {incr i} {
+    set rnd [expr int(rand() * 100)]
+    set doc "x[string repeat $rnd 20]"
+    execsql { INSERT INTO t1(rowid, x) VALUES($i, $doc) }
+  }
+} {}
+
+do_3_test 3.10
+
+}
+
+#-------------------------------------------------------------------------
+# Test that segments that end unexpectedly are identified as corruption.
+#
+reset_db
+do_test 4.0 {
+  execsql { 
+    CREATE VIRTUAL TABLE t1 USING fts5(x);
+    INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
+  }
+  for {set i 0} {$i < 100} {incr i} {
+    set rnd [expr int(rand() * 100)]
+    set doc "x[string repeat $rnd 20]"
+    execsql { INSERT INTO t1(rowid, x) VALUES($i, $doc) }
+  }
+  execsql { INSERT INTO t1(t1) VALUES('optimize') }
+} {}
+
+set nErr 0
+for {set i 1} {1} {incr i} {
+  set struct [db one {SELECT block FROM t1_data WHERE id=10}]
+  binary scan $struct c* var
+  set end [lindex $var end]
+  if {$end<=$i} break
+  lset var end [expr $end - $i]
+  set struct [binary format c* $var]
+  db eval {
+    BEGIN;
+    UPDATE t1_data SET block = $struct WHERE id=10;
+  }
+  do_test 4.1.$i {
+    incr nErr [catch { db eval { SELECT rowid FROM t1 WHERE t1 MATCH 'x*' } }]
+    set {} {}
+  } {}
+  catch { db eval ROLLBACK }
+}
+do_test 4.1.x { expr $nErr>45 } 1
+
+#-------------------------------------------------------------------------
+#
+
+# The first argument passed to this command must be a binary blob 
+# containing an FTS5 leaf page. This command returns a copy of this
+# blob, with the pgidx of the leaf page replaced by a single varint
+# containing value $iVal.
+#
+proc rewrite_pgidx {blob iVal} {
+  binary scan $blob SS off1 szLeaf
+  if {$iVal<0 || $iVal>=128} {
+    error "$iVal out of range!"
+  } else {
+    set pgidx [binary format c $iVal]
+  }
+
+  binary format a${szLeaf}a* $blob $pgidx
+}
+
+reset_db
+do_execsql_test 5.1 {
+  CREATE VIRTUAL TABLE x1 USING fts5(x);
+  INSERT INTO x1(x1, rank) VALUES('pgsz', 40);
+  BEGIN;
+  INSERT INTO x1 VALUES('xaaa xabb xccc xcdd xeee xeff xggg xghh xiii xijj');
+  INSERT INTO x1 SELECT x FROM x1;
+  INSERT INTO x1 SELECT x FROM x1;
+  INSERT INTO x1 SELECT x FROM x1;
+  INSERT INTO x1 SELECT x FROM x1;
+  INSERT INTO x1(x1) VALUES('optimize');
+  COMMIT;
+}
+
+#db eval { SELECT fts5_decode(id, block) b from x1_data } { puts $b }
+#
+db func rewrite_pgidx rewrite_pgidx  
+set i 0
+foreach rowid [db eval {SELECT rowid FROM x1_data WHERE rowid>100}] {
+  foreach val {2 100} {
+    do_test 5.2.$val.[incr i] {
+      catchsql {
+        BEGIN;
+        UPDATE x1_data SET block=rewrite_pgidx(block, $val) WHERE id=$rowid;
+        SELECT rowid FROM x1 WHERE x1 MATCH 'xa*';
+        SELECT rowid FROM x1 WHERE x1 MATCH 'xb*';
+        SELECT rowid FROM x1 WHERE x1 MATCH 'xc*';
+        SELECT rowid FROM x1 WHERE x1 MATCH 'xd*';
+        SELECT rowid FROM x1 WHERE x1 MATCH 'xe*';
+        SELECT rowid FROM x1 WHERE x1 MATCH 'xf*';
+        SELECT rowid FROM x1 WHERE x1 MATCH 'xg*';
+        SELECT rowid FROM x1 WHERE x1 MATCH 'xh*';
+        SELECT rowid FROM x1 WHERE x1 MATCH 'xi*';
+      }
+      set {} {}
+    } {}
+    catch { db eval ROLLBACK }
+  }
+}
+
+
 sqlite3_fts5_may_be_corrupt 0
 finish_test
 
diff --git a/ext/fts5/test/fts5dlidx.test b/ext/fts5/test/fts5dlidx.test
index 07d7e2baeb..86fb8bb930 100644
--- a/ext/fts5/test/fts5dlidx.test
+++ b/ext/fts5/test/fts5dlidx.test
@@ -26,6 +26,8 @@ if { $tcl_platform(wordSize)<8 } {
   return
 }
 
+if 1 {
+
 proc do_fb_test {tn sql res} {
   set res2 [lsort -integer -decr $res]
   uplevel [list do_execsql_test $tn.1 $sql $res]
@@ -128,5 +130,60 @@ proc do_dlidx_test2 {tn nEntry iFirst nStep} {
 
 do_dlidx_test2 2.1 [expr 20] [expr 1<<57] [expr (1<<57) + 128]
 
+}
+
+#--------------------------------------------------------------------
+#
+reset_db
+
+set ::vocab [list \
+  IteratorpItercurrentlypointstothefirstrowidofadoclist \
+  Thereisadoclistindexassociatedwiththefinaltermonthecurrent \
+  pageIfthecurrenttermisthelasttermonthepageloadthe \
+  doclistindexfromdiskandinitializeaniteratoratpIterpDlidx \
+  IteratorpItercurrentlypointstothefirstrowidofadoclist \
+  Thereisadoclistindexassociatedwiththefinaltermonthecurrent \
+  pageIfthecurrenttermisthelasttermonthepageloadthe \
+  doclistindexfromdiskandinitializeaniteratoratpIterpDlidx \
+]
+proc rnddoc {} {
+  global vocab
+  set nVocab [llength $vocab]
+  set ret [list]
+  for {set i 0} {$i < 64} {incr i} {
+    lappend ret [lindex $vocab [expr $i % $nVocab]]
+  }
+  set ret
+}
+db func rnddoc rnddoc
+
+do_execsql_test 3.1 {
+  CREATE VIRTUAL TABLE abc USING fts5(a);
+  INSERT INTO abc(abc, rank) VALUES('pgsz', 32);
+
+  INSERT INTO abc VALUES ( rnddoc() );
+  INSERT INTO abc VALUES ( rnddoc() );
+  INSERT INTO abc VALUES ( rnddoc() );
+  INSERT INTO abc VALUES ( rnddoc() );
+
+  INSERT INTO abc SELECT rnddoc() FROM abc;
+  INSERT INTO abc SELECT rnddoc() FROM abc;
+}
+
+do_execsql_test 3.2 {
+  INSERT INTO abc(abc) VALUES('integrity-check');
+  INSERT INTO abc(abc) VALUES('optimize');
+  INSERT INTO abc(abc) VALUES('integrity-check');
+}
+
+set v [lindex $vocab 0]
+set i 0
+foreach v $vocab {
+  do_execsql_test 3.3.[incr i] {
+    SELECT rowid FROM abc WHERE abc MATCH $v
+  } {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16}
+}
+
+
 finish_test
 
diff --git a/ext/fts5/test/fts5fault7.test b/ext/fts5/test/fts5fault7.test
index 381f9ea6ad..9bce1c7caf 100644
--- a/ext/fts5/test/fts5fault7.test
+++ b/ext/fts5/test/fts5fault7.test
@@ -14,7 +14,7 @@
 
 source [file join [file dirname [info script]] fts5_common.tcl]
 source $testdir/malloc_common.tcl
-set testprefix fts5fault2
+set testprefix fts5fault7
 
 # If SQLITE_ENABLE_FTS3 is defined, omit this file.
 ifcapable !fts5 {
@@ -22,6 +22,8 @@ ifcapable !fts5 {
   return
 }
 
+if 1 {
+
 #-------------------------------------------------------------------------
 # Test fault-injection on a query that uses xColumnSize() on columnsize=0
 # table.
@@ -41,5 +43,49 @@ do_faultsim_test 1 -faults oom* -body {
   faultsim_test_result {0 {7 4 10}} {1 SQLITE_NOMEM}
 }
 
+}
+
+#-------------------------------------------------------------------------
+# Test fault-injection when a segment is promoted.
+#
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t2 USING fts5(a);
+  INSERT INTO t2(t2, rank) VALUES('automerge', 0);
+  INSERT INTO t2(t2, rank) VALUES('crisismerge', 4);
+  INSERT INTO t2(t2, rank) VALUES('pgsz', 40);
+
+  INSERT INTO t2 VALUES('a b c');
+  INSERT INTO t2 VALUES('d e f');
+  INSERT INTO t2 VALUES('f e d');
+  INSERT INTO t2 VALUES('c b a');
+
+  INSERT INTO t2 VALUES('a b c');
+  INSERT INTO t2 VALUES('d e f');
+  INSERT INTO t2 VALUES('f e d');
+  INSERT INTO t2 VALUES('c b a');
+} {}
+
+faultsim_save_and_close
+do_faultsim_test 1 -faults oom-t* -prep {
+  faultsim_restore_and_reopen
+  db eval {
+    BEGIN;
+    INSERT INTO t2 VALUES('c d c g g f');
+    INSERT INTO t2 VALUES('c d g b f d');
+    INSERT INTO t2 VALUES('c c f d e d');
+    INSERT INTO t2 VALUES('e a f c e f');
+    INSERT INTO t2 VALUES('c g f b b d');
+    INSERT INTO t2 VALUES('d a g a b b');
+    INSERT INTO t2 VALUES('e f a b c e');
+    INSERT INTO t2 VALUES('e c a g c d');
+    INSERT INTO t2 VALUES('g b d d e b');
+    INSERT INTO t2 VALUES('e a d a e d');
+  }
+} -body {
+  db eval COMMIT
+} -test {
+  faultsim_test_result {0 {}}
+}
+
 finish_test
 
diff --git a/manifest b/manifest
index 836476cb10..1f1b10325e 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Testability\simprovements\sfor\sthe\sONEPASS_MULTI\senhancement.
-D 2015-09-14T19:26:37.988
+C Improve\stest\scoverage\sof\sfts5_index.c.
+D 2015-09-14T19:51:05.852
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -112,7 +112,7 @@ F ext/fts5/fts5_buffer.c 64dcaf36a3ebda9e84b7c3b8788887ec325e12a4
 F ext/fts5/fts5_config.c 57ee5fe71578cb494574fc0e6e51acb9a22a8695
 F ext/fts5/fts5_expr.c 667faaf14a69a5683ac383acdc8d942cf32c3f93
 F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246
-F ext/fts5/fts5_index.c 62a682a70ea2e84fa67c7786ead892b201116ad1
+F ext/fts5/fts5_index.c fae675f4d60cfff93b31f85576fddb6ade7d344e
 F ext/fts5/fts5_main.c 3fa906f6c0177caf8f82862bc70f37b28bb3305c
 F ext/fts5/fts5_storage.c 120f7b143688b5b7710dacbd48cff211609b8059
 F ext/fts5/fts5_tcl.c 6da58d6e8f42a93c4486b5ba9b187a7f995dee37
@@ -146,8 +146,8 @@ F ext/fts5/test/fts5config.test ad2ff42ddc856aed2d05bf89dc1c578c8a39ea3b
 F ext/fts5/test/fts5content.test 9a952c95518a14182dc3b59e3c8fa71cda82a4e1
 F ext/fts5/test/fts5corrupt.test c2ad090192708150d50d961278df10ae7a4b8b62
 F ext/fts5/test/fts5corrupt2.test 26c0a39dd9ff73207e6229f83b50b21d37c7658c
-F ext/fts5/test/fts5corrupt3.test 1ccf575f5126e79f9fec7979fd02a1f40a076be3
-F ext/fts5/test/fts5dlidx.test 59b80bbe34169a082c575d9c26f0a7019a7b79c1
+F ext/fts5/test/fts5corrupt3.test 618a965b4fd2859f9ddb72413c9828a23828704b
+F ext/fts5/test/fts5dlidx.test 77259f6d8d671b486113b157bd30107ca9d6b0f6
 F ext/fts5/test/fts5doclist.test 8edb5b57e5f144030ed74ec00ef6fa4294fed79b
 F ext/fts5/test/fts5ea.test b01e3a18cdfabbff8104a96a5242a06a68a998a0
 F ext/fts5/test/fts5eb.test 3e5869af2008cbc4ad03a175a0b6f6e58134cd43
@@ -157,7 +157,7 @@ F ext/fts5/test/fts5fault3.test d6e9577d4312e331a913c72931bf131704efc8f3
 F ext/fts5/test/fts5fault4.test 762991d526ee67c2b374351a17248097ea38bee7
 F ext/fts5/test/fts5fault5.test 54da9fd4c3434a1d4f6abdcb6469299d91cf5875
 F ext/fts5/test/fts5fault6.test 97bce1a36b7a64e3203fea504ae8e5cfd5ada423
-F ext/fts5/test/fts5fault7.test f4a9b796f8b20c78ec7cf9f4e11144d15d7c3fd4
+F ext/fts5/test/fts5fault7.test f62ed4d98f137eb03f1db94d1fa41b17a771d971
 F ext/fts5/test/fts5full.test 6f6143af0c6700501d9fd597189dfab1555bb741
 F ext/fts5/test/fts5hash.test 42eb066f667e9a389a63437cb7038c51974d4fc6
 F ext/fts5/test/fts5integrity.test 29f41d2c7126c6122fbb5d54e556506456876145
@@ -1387,7 +1387,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 379455af9fdfb299a857d46f35f0a76ed6daa08a
-R 240d53905959924a39a1969f0c2c4fad
-U drh
-Z db76ee6590838fdbed88f2282d6173bf
+P d2df93f26fc0cf6fe01acfeaf2946972e9d8cca2
+R a767de21459f1cfc9798aa9a41d99081
+U dan
+Z 579f9e68a86182329093b3386940c228
diff --git a/manifest.uuid b/manifest.uuid
index 93b327f2ec..7a9c2af077 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-d2df93f26fc0cf6fe01acfeaf2946972e9d8cca2
\ No newline at end of file
+c1f76686cee3918b1be785a4071d68cb3afda0ef
\ No newline at end of file

From 128ea696eac4dc6619820b0386f7fd226b96eeec Mon Sep 17 00:00:00 2001
From: dan 
Date: Tue, 15 Sep 2015 11:58:33 +0000
Subject: [PATCH 098/100] Fix a problem with fts5 "ORDER BY rowid DESC" queries
 and large terms.

FossilOrigin-Name: b26d8f79c62cb71b4c3ea5df9d0e09558bbbd794
---
 ext/fts5/fts5_index.c        |  7 ++++++-
 ext/fts5/test/fts5dlidx.test |  8 ++++++++
 ext/fts5/test/fts5rowid.test |  4 ++++
 manifest                     | 16 ++++++++--------
 manifest.uuid                |  2 +-
 5 files changed, 27 insertions(+), 10 deletions(-)

diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index c9fb6207ec..adbf19530a 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -1657,7 +1657,12 @@ static void fts5SegIterReverseNewPage(Fts5Index *p, Fts5SegIter *pIter){
           pIter->pSeg->iSegid, pIter->iLeafPgno
     ));
     if( pNew ){
-      if( pIter->iLeafPgno==pIter->iTermLeafPgno ){
+      /* iTermLeafOffset may be equal to szLeaf if the term is the last
+      ** thing on the page - i.e. the first rowid is on the following page.
+      ** In this case leaf pIter->pLeaf==0, this iterator is at EOF. */
+      if( pIter->iLeafPgno==pIter->iTermLeafPgno 
+       && pIter->iTermLeafOffsetszLeaf 
+      ){
         pIter->pLeaf = pNew;
         pIter->iLeafOffset = pIter->iTermLeafOffset;
       }else{
diff --git a/ext/fts5/test/fts5dlidx.test b/ext/fts5/test/fts5dlidx.test
index 86fb8bb930..5a03c6989b 100644
--- a/ext/fts5/test/fts5dlidx.test
+++ b/ext/fts5/test/fts5dlidx.test
@@ -170,6 +170,14 @@ do_execsql_test 3.1 {
   INSERT INTO abc SELECT rnddoc() FROM abc;
 }
 
+
+
+do_execsql_test 3.2 {
+  SELECT rowid FROM abc WHERE abc 
+  MATCH 'IteratorpItercurrentlypointstothefirstrowidofadoclist' 
+  ORDER BY rowid DESC;
+} {16 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1}
+
 do_execsql_test 3.2 {
   INSERT INTO abc(abc) VALUES('integrity-check');
   INSERT INTO abc(abc) VALUES('optimize');
diff --git a/ext/fts5/test/fts5rowid.test b/ext/fts5/test/fts5rowid.test
index e9dffa5791..621934c0f2 100644
--- a/ext/fts5/test/fts5rowid.test
+++ b/ext/fts5/test/fts5rowid.test
@@ -91,6 +91,10 @@ do_execsql_test 2.6 {
 #  SELECT count(fts5_decode(rowid, block)) FROM x1_data;
 #} $res
 
+do_execsql_test 2.8 {
+  SELECT fts5_decode(fts5_rowid('segment', 1000, 1), X'AB')
+} {corrupt}
+
 #-------------------------------------------------------------------------
 # Tests with very large tokens.
 #
diff --git a/manifest b/manifest
index 1f1b10325e..af98c1d902 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Improve\stest\scoverage\sof\sfts5_index.c.
-D 2015-09-14T19:51:05.852
+C Fix\sa\sproblem\swith\sfts5\s"ORDER\sBY\srowid\sDESC"\squeries\sand\slarge\sterms.
+D 2015-09-15T11:58:33.318
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -112,7 +112,7 @@ F ext/fts5/fts5_buffer.c 64dcaf36a3ebda9e84b7c3b8788887ec325e12a4
 F ext/fts5/fts5_config.c 57ee5fe71578cb494574fc0e6e51acb9a22a8695
 F ext/fts5/fts5_expr.c 667faaf14a69a5683ac383acdc8d942cf32c3f93
 F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246
-F ext/fts5/fts5_index.c fae675f4d60cfff93b31f85576fddb6ade7d344e
+F ext/fts5/fts5_index.c 437c907fec8b72c08394cda73cee84afca700a00
 F ext/fts5/fts5_main.c 3fa906f6c0177caf8f82862bc70f37b28bb3305c
 F ext/fts5/fts5_storage.c 120f7b143688b5b7710dacbd48cff211609b8059
 F ext/fts5/fts5_tcl.c 6da58d6e8f42a93c4486b5ba9b187a7f995dee37
@@ -147,7 +147,7 @@ F ext/fts5/test/fts5content.test 9a952c95518a14182dc3b59e3c8fa71cda82a4e1
 F ext/fts5/test/fts5corrupt.test c2ad090192708150d50d961278df10ae7a4b8b62
 F ext/fts5/test/fts5corrupt2.test 26c0a39dd9ff73207e6229f83b50b21d37c7658c
 F ext/fts5/test/fts5corrupt3.test 618a965b4fd2859f9ddb72413c9828a23828704b
-F ext/fts5/test/fts5dlidx.test 77259f6d8d671b486113b157bd30107ca9d6b0f6
+F ext/fts5/test/fts5dlidx.test ecba5e62ea8b26c33829961602069c546228046d
 F ext/fts5/test/fts5doclist.test 8edb5b57e5f144030ed74ec00ef6fa4294fed79b
 F ext/fts5/test/fts5ea.test b01e3a18cdfabbff8104a96a5242a06a68a998a0
 F ext/fts5/test/fts5eb.test 3e5869af2008cbc4ad03a175a0b6f6e58134cd43
@@ -172,7 +172,7 @@ F ext/fts5/test/fts5prefix.test 552a462f0e8595676611f41643de217fb4ac2808
 F ext/fts5/test/fts5rank.test 11dcebba31d822f7e99685b4ea2c2ae3ec0b16f1
 F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b
 F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17
-F ext/fts5/test/fts5rowid.test 3e3b66670ca65540fa321250ac12f890b17f9312
+F ext/fts5/test/fts5rowid.test 400384798349d658eaf06aefa1e364957d5d4821
 F ext/fts5/test/fts5simple.test f629e24a35a9f31cfb16c9920e8c2316e3d93e94
 F ext/fts5/test/fts5synonym.test cf88c0a56d5ea9591e3939ef1f6e294f7f2d0671
 F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89
@@ -1387,7 +1387,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P d2df93f26fc0cf6fe01acfeaf2946972e9d8cca2
-R a767de21459f1cfc9798aa9a41d99081
+P c1f76686cee3918b1be785a4071d68cb3afda0ef
+R d67808eb365cc1292d83338a5a6fd99f
 U dan
-Z 579f9e68a86182329093b3386940c228
+Z c8f43a4f5db97e6a86a8018a67cf6238
diff --git a/manifest.uuid b/manifest.uuid
index 7a9c2af077..f00cc58580 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-c1f76686cee3918b1be785a4071d68cb3afda0ef
\ No newline at end of file
+b26d8f79c62cb71b4c3ea5df9d0e09558bbbd794
\ No newline at end of file

From b77ebd828bb75e0cf21c2f149fb84024bd6b5f3a Mon Sep 17 00:00:00 2001
From: drh 
Date: Tue, 15 Sep 2015 13:42:16 +0000
Subject: [PATCH 099/100] Reformat some code to make it easier to merge with
 sessions.  No logic changes.

FossilOrigin-Name: eade355fafec558fc13dc4b08ca0b07713a2cd84
---
 manifest      | 16 ++++++++--------
 manifest.uuid |  2 +-
 src/delete.c  |  6 +++++-
 src/insert.c  |  8 +++++---
 4 files changed, 19 insertions(+), 13 deletions(-)

diff --git a/manifest b/manifest
index af98c1d902..589e961845 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\sproblem\swith\sfts5\s"ORDER\sBY\srowid\sDESC"\squeries\sand\slarge\sterms.
-D 2015-09-15T11:58:33.318
+C Reformat\ssome\scode\sto\smake\sit\seasier\sto\smerge\swith\ssessions.\s\sNo\slogic\schanges.
+D 2015-09-15T13:42:16.601
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -291,7 +291,7 @@ F src/complete.c addcd8160b081131005d5bc2d34adf20c1c5c92f
 F src/ctime.c 5a0b735dc95604766f5dac73973658eef782ee8b
 F src/date.c fb1c99172017dcc8e237339132c91a21a0788584
 F src/dbstat.c e637e7a7ff40ef32132a418c6fdf1cfb63aa27c7
-F src/delete.c 249a287682dd6e7ab362dd81e4d65abe2d5c3ce4
+F src/delete.c 5ab483f15aaf202290227d5dd1f10a7ecce3d6b1
 F src/expr.c 3a76afcdac925294c39903b7002ddb9e5fd29863
 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb
 F src/fkey.c 83e1baba999bed3144ea5a2143fc922edf51135f
@@ -300,7 +300,7 @@ F src/global.c 508e4087f7b41d688e4762dcf4d4fe28cfbc87f9
 F src/hash.c 4263fbc955f26c2e8cdc0cf214bc42435aa4e4f5
 F src/hash.h c8f3c31722cf3277d03713909761e152a5b81094
 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08
-F src/insert.c ace93aa254f86b1bddb20eb9bf382f6b851bbda5
+F src/insert.c 9748a37e058256eb2ead69f028ab85ebf203ad15
 F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d
 F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e
 F src/lempar.c d344a95d60c24e2f490ee59db9784b1b17439012
@@ -1387,7 +1387,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P c1f76686cee3918b1be785a4071d68cb3afda0ef
-R d67808eb365cc1292d83338a5a6fd99f
-U dan
-Z c8f43a4f5db97e6a86a8018a67cf6238
+P b26d8f79c62cb71b4c3ea5df9d0e09558bbbd794
+R c60068330077cac7ec52227493b03ae0
+U drh
+Z 9a77e7bf5b2e60419df2346d70a9c648
diff --git a/manifest.uuid b/manifest.uuid
index f00cc58580..cdea9a8f40 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-b26d8f79c62cb71b4c3ea5df9d0e09558bbbd794
\ No newline at end of file
+eade355fafec558fc13dc4b08ca0b07713a2cd84
\ No newline at end of file
diff --git a/src/delete.c b/src/delete.c
index 7c2f3fcd00..860b2cf115 100644
--- a/src/delete.c
+++ b/src/delete.c
@@ -361,7 +361,11 @@ void sqlite3DeleteFrom(
   ** It is easier just to erase the whole table. Prior to version 3.6.5,
   ** this optimization caused the row change count (the value returned by 
   ** API function sqlite3_count_changes) to be set incorrectly.  */
-  if( rcauth==SQLITE_OK && pWhere==0 && !bComplex && !IsVirtual(pTab) ){
+  if( rcauth==SQLITE_OK
+   && pWhere==0
+   && !bComplex
+   && !IsVirtual(pTab)
+  ){
     assert( !isView );
     sqlite3TableLock(pParse, iDb, pTab->tnum, 1, pTab->zName);
     if( HasRowid(pTab) ){
diff --git a/src/insert.c b/src/insert.c
index 6132778355..53b429c1f4 100644
--- a/src/insert.c
+++ b/src/insert.c
@@ -1349,9 +1349,11 @@ void sqlite3GenerateConstraintChecks(
           sqlite3GenerateRowDelete(pParse, pTab, pTrigger, iDataCur, iIdxCur,
                                    regNewData, 1, 0, OE_Replace,
                                    ONEPASS_SINGLE, -1);
-        }else if( pTab->pIndex ){
-          sqlite3MultiWrite(pParse);
-          sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur, 0, -1);
+        }else{
+          if( pTab->pIndex ){
+            sqlite3MultiWrite(pParse);
+            sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur,0,-1);
+          }
         }
         seenReplace = 1;
         break;

From 6609bab24d84219140f28d9bda1ebee4389be5ca Mon Sep 17 00:00:00 2001
From: dan 
Date: Tue, 15 Sep 2015 14:39:17 +0000
Subject: [PATCH 100/100] Add test cases to cover fts5 integrity-check code.

FossilOrigin-Name: 1d018c35b9e81982df036f5e62a4a42219b54e02
---
 ext/fts5/fts5_index.c           |  3 +-
 ext/fts5/test/fts5corrupt3.test | 86 ++++++++++++++++++++++++++++++++-
 manifest                        | 16 +++---
 manifest.uuid                   |  2 +-
 4 files changed, 94 insertions(+), 13 deletions(-)

diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index adbf19530a..b8aa52c2f8 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -4854,7 +4854,6 @@ static void fts5IndexIntegrityCheckEmpty(
       if( i>=iNoRowid && 0!=fts5LeafFirstRowidOff(pLeaf) ) p->rc = FTS5_CORRUPT;
     }
     fts5DataRelease(pLeaf);
-    if( p->rc ) break;
   }
 }
 
@@ -4921,7 +4920,7 @@ static void fts5IndexIntegrityCheckSegment(
   if( pSeg->pgnoFirst==0 ) return;
 
   fts5IndexPrepareStmt(p, &pStmt, sqlite3_mprintf(
-      "SELECT segid, term, (pgno>>1), (pgno & 1) FROM '%q'.'%q_idx' WHERE segid=%d",
+      "SELECT segid, term, (pgno>>1), (pgno&1) FROM %Q.'%q_idx' WHERE segid=%d",
       pConfig->zDb, pConfig->zName, pSeg->iSegid
   ));
 
diff --git a/ext/fts5/test/fts5corrupt3.test b/ext/fts5/test/fts5corrupt3.test
index 9501e29ba9..ba1d6f9902 100644
--- a/ext/fts5/test/fts5corrupt3.test
+++ b/ext/fts5/test/fts5corrupt3.test
@@ -154,8 +154,6 @@ do_test 3.9 {
 
 do_3_test 3.10
 
-}
-
 #-------------------------------------------------------------------------
 # Test that segments that end unexpectedly are identified as corruption.
 #
@@ -252,6 +250,90 @@ foreach rowid [db eval {SELECT rowid FROM x1_data WHERE rowid>100}] {
   }
 }
 
+}
+
+#------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 6.1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a);
+  INSERT INTO t1 VALUES('bbbbb ccccc');
+  SELECT quote(block) FROM t1_data WHERE rowid>100;
+} {X'000000180630626262626201020201056363636363010203040A'}
+do_execsql_test 6.1.1 {
+  UPDATE t1_data SET block = 
+  X'000000180630626262626201020201056161616161010203040A'
+  WHERE rowid>100;
+}
+do_catchsql_test 6.1.2 {
+  INSERT INTO t1(t1) VALUES('integrity-check');
+} {1 {database disk image is malformed}}
+
+#-------
+reset_db
+do_execsql_test 6.2.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a);
+  INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
+  INSERT INTO t1 VALUES('aa bb cc dd ee');
+  SELECT pgno, quote(term) FROM t1_idx;
+} {2 X'' 4 X'3064'}
+do_execsql_test 6.2.1 {
+  UPDATE t1_idx SET term = X'3065' WHERE pgno=4;
+}
+do_catchsql_test 6.2.2 {
+  INSERT INTO t1(t1) VALUES('integrity-check');
+} {1 {database disk image is malformed}}
+
+#-------
+reset_db
+do_execsql_test 6.3.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a);
+  INSERT INTO t1 VALUES('abc abcdef abcdefghi');
+  SELECT quote(block) FROM t1_data WHERE id>100;
+}    {X'0000001C043061626301020204036465660102030703676869010204040808'}
+do_execsql_test 6.3.1 {
+  BEGIN;
+    UPDATE t1_data SET block = 
+      X'0000001C043061626301020204036465660102035003676869010204040808'
+      ------------------------------------------^^---------------------
+    WHERE id>100;
+}
+do_catchsql_test 6.3.2 {
+  INSERT INTO t1(t1) VALUES('integrity-check');
+} {1 {database disk image is malformed}}
+do_execsql_test 6.3.3 {
+  ROLLBACK;
+  BEGIN;
+    UPDATE t1_data SET block = 
+      X'0000001C043061626301020204036465660102030750676869010204040808'
+      --------------------------------------------^^-------------------
+    WHERE id>100;
+}
+do_catchsql_test 6.3.3 {
+  INSERT INTO t1(t1) VALUES('integrity-check');
+} {1 {database disk image is malformed}}
+do_execsql_test 6.3.4 {
+  ROLLBACK;
+  BEGIN;
+    UPDATE t1_data SET block = 
+      X'0000001C043061626301020204036465660102030707676869010204040850'
+      --------------------------------------------------------------^^-
+    WHERE id>100;
+}
+do_catchsql_test 6.3.5 {
+  INSERT INTO t1(t1) VALUES('integrity-check');
+} {1 {database disk image is malformed}}
+do_execsql_test 6.3.6 {
+  ROLLBACK;
+  BEGIN;
+    UPDATE t1_data SET block = 
+      X'0000001C503061626301020204036465660102030707676869010204040808'
+      ----------^^-----------------------------------------------------
+    WHERE id>100;
+}
+do_catchsql_test 6.3.5 {
+  INSERT INTO t1(t1) VALUES('integrity-check');
+} {1 {database disk image is malformed}}
 
 sqlite3_fts5_may_be_corrupt 0
 finish_test
diff --git a/manifest b/manifest
index 589e961845..e3335b827f 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Reformat\ssome\scode\sto\smake\sit\seasier\sto\smerge\swith\ssessions.\s\sNo\slogic\schanges.
-D 2015-09-15T13:42:16.601
+C Add\stest\scases\sto\scover\sfts5\sintegrity-check\scode.
+D 2015-09-15T14:39:17.597
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -112,7 +112,7 @@ F ext/fts5/fts5_buffer.c 64dcaf36a3ebda9e84b7c3b8788887ec325e12a4
 F ext/fts5/fts5_config.c 57ee5fe71578cb494574fc0e6e51acb9a22a8695
 F ext/fts5/fts5_expr.c 667faaf14a69a5683ac383acdc8d942cf32c3f93
 F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246
-F ext/fts5/fts5_index.c 437c907fec8b72c08394cda73cee84afca700a00
+F ext/fts5/fts5_index.c 4fdbc0a321e3a1d73741a623d7aea4db78d6a86d
 F ext/fts5/fts5_main.c 3fa906f6c0177caf8f82862bc70f37b28bb3305c
 F ext/fts5/fts5_storage.c 120f7b143688b5b7710dacbd48cff211609b8059
 F ext/fts5/fts5_tcl.c 6da58d6e8f42a93c4486b5ba9b187a7f995dee37
@@ -146,7 +146,7 @@ F ext/fts5/test/fts5config.test ad2ff42ddc856aed2d05bf89dc1c578c8a39ea3b
 F ext/fts5/test/fts5content.test 9a952c95518a14182dc3b59e3c8fa71cda82a4e1
 F ext/fts5/test/fts5corrupt.test c2ad090192708150d50d961278df10ae7a4b8b62
 F ext/fts5/test/fts5corrupt2.test 26c0a39dd9ff73207e6229f83b50b21d37c7658c
-F ext/fts5/test/fts5corrupt3.test 618a965b4fd2859f9ddb72413c9828a23828704b
+F ext/fts5/test/fts5corrupt3.test a1429635c30f922079189e35b2b631eb7463a2fb
 F ext/fts5/test/fts5dlidx.test ecba5e62ea8b26c33829961602069c546228046d
 F ext/fts5/test/fts5doclist.test 8edb5b57e5f144030ed74ec00ef6fa4294fed79b
 F ext/fts5/test/fts5ea.test b01e3a18cdfabbff8104a96a5242a06a68a998a0
@@ -1387,7 +1387,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P b26d8f79c62cb71b4c3ea5df9d0e09558bbbd794
-R c60068330077cac7ec52227493b03ae0
-U drh
-Z 9a77e7bf5b2e60419df2346d70a9c648
+P eade355fafec558fc13dc4b08ca0b07713a2cd84
+R 1694ed50f41ea37a05fbcbd009cf49d5
+U dan
+Z ab887f4a44598b0175c78de8cb058da9
diff --git a/manifest.uuid b/manifest.uuid
index cdea9a8f40..7bd70ebd90 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-eade355fafec558fc13dc4b08ca0b07713a2cd84
\ No newline at end of file
+1d018c35b9e81982df036f5e62a4a42219b54e02
\ No newline at end of file