From 85963f5474aaf9fb789a21c74600fa3e0018be0d Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 12 Nov 2013 17:46:44 +0000 Subject: [PATCH 01/14] When possible, have FTS use 32-bit operations to read varints from the database. FossilOrigin-Name: aa7ba302ed13aedde89b5bcbe9119799c0da8a42 --- ext/fts3/fts3.c | 77 +++++++++++++++++--------- ext/fts3/fts3Int.h | 4 ++ ext/fts3/fts3_snippet.c | 2 +- ext/fts3/fts3_test.c | 49 +++++++++++++++++ ext/fts3/fts3_write.c | 16 +++--- manifest | 25 +++++---- manifest.uuid | 2 +- test/fts3varint.test | 118 ++++++++++++++++++++++++++++++++++++++++ test/permutations.test | 1 + 9 files changed, 246 insertions(+), 48 deletions(-) create mode 100644 test/fts3varint.test diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index dd6d38a59d..9f6cc482af 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -330,21 +330,35 @@ int sqlite3Fts3PutVarint(char *p, sqlite_int64 v){ return (int) (q - (unsigned char *)p); } +#define GETVARINT_STEP(v, ptr, shift, mask1, mask2, var, ret) \ + v = (v & mask1) | ( (*ptr++) << shift ); \ + if( (v & mask2)==0 ){ var = v; return ret; } + /* ** Read a 64-bit variable-length integer from memory starting at p[0]. ** Return the number of bytes read, or 0 on error. ** The value is stored in *v. */ int sqlite3Fts3GetVarint(const char *p, sqlite_int64 *v){ - const unsigned char *q = (const unsigned char *) p; - sqlite_uint64 x = 0, y = 1; - while( (*q&0x80)==0x80 && q-(unsigned char *)pzEnd ){ rc = FTS_CORRUPT_VTAB; @@ -1806,7 +1831,7 @@ static int fts3SelectLeaf( assert( piLeaf || piLeaf2 ); - sqlite3Fts3GetVarint32(zNode, &iHeight); + fts3GetVarint32(zNode, &iHeight); rc = fts3ScanInteriorNode(zTerm, nTerm, zNode, nNode, piLeaf, piLeaf2); assert( !piLeaf2 || !piLeaf || rc!=SQLITE_OK || (*piLeaf<=*piLeaf2) ); @@ -2008,11 +2033,11 @@ static void fts3PoslistMerge( int iCol1; /* The current column index in pp1 */ int iCol2; /* The current column index in pp2 */ - if( *p1==POS_COLUMN ) sqlite3Fts3GetVarint32(&p1[1], &iCol1); + if( *p1==POS_COLUMN ) fts3GetVarint32(&p1[1], &iCol1); else if( *p1==POS_END ) iCol1 = POSITION_LIST_END; else iCol1 = 0; - if( *p2==POS_COLUMN ) sqlite3Fts3GetVarint32(&p2[1], &iCol2); + if( *p2==POS_COLUMN ) fts3GetVarint32(&p2[1], &iCol2); else if( *p2==POS_END ) iCol2 = POSITION_LIST_END; else iCol2 = 0; @@ -2105,11 +2130,11 @@ static int fts3PoslistPhraseMerge( assert( p!=0 && *p1!=0 && *p2!=0 ); if( *p1==POS_COLUMN ){ p1++; - p1 += sqlite3Fts3GetVarint32(p1, &iCol1); + p1 += fts3GetVarint32(p1, &iCol1); } if( *p2==POS_COLUMN ){ p2++; - p2 += sqlite3Fts3GetVarint32(p2, &iCol2); + p2 += fts3GetVarint32(p2, &iCol2); } while( 1 ){ @@ -2159,9 +2184,9 @@ static int fts3PoslistPhraseMerge( if( 0==*p1 || 0==*p2 ) break; p1++; - p1 += sqlite3Fts3GetVarint32(p1, &iCol1); + p1 += fts3GetVarint32(p1, &iCol1); p2++; - p2 += sqlite3Fts3GetVarint32(p2, &iCol2); + p2 += fts3GetVarint32(p2, &iCol2); } /* Advance pointer p1 or p2 (whichever corresponds to the smaller of @@ -2173,12 +2198,12 @@ static int fts3PoslistPhraseMerge( fts3ColumnlistCopy(0, &p1); if( 0==*p1 ) break; p1++; - p1 += sqlite3Fts3GetVarint32(p1, &iCol1); + p1 += fts3GetVarint32(p1, &iCol1); }else{ fts3ColumnlistCopy(0, &p2); if( 0==*p2 ) break; p2++; - p2 += sqlite3Fts3GetVarint32(p2, &iCol2); + p2 += fts3GetVarint32(p2, &iCol2); } } @@ -5345,7 +5370,7 @@ static void fts3EvalUpdateCounts(Fts3Expr *pExpr){ pExpr->aMI[iCol*3 + 2] += (iCnt>0); if( *p==0x00 ) break; p++; - p += sqlite3Fts3GetVarint32(p, &iCol); + p += fts3GetVarint32(p, &iCol); } } @@ -5646,7 +5671,7 @@ int sqlite3Fts3EvalPhrasePoslist( if( *pIter==0x01 ){ pIter++; - pIter += sqlite3Fts3GetVarint32(pIter, &iThis); + pIter += fts3GetVarint32(pIter, &iThis); }else{ iThis = 0; } @@ -5654,7 +5679,7 @@ int sqlite3Fts3EvalPhrasePoslist( fts3ColumnlistCopy(0, &pIter); if( *pIter==0x00 ) return 0; pIter++; - pIter += sqlite3Fts3GetVarint32(pIter, &iThis); + pIter += fts3GetVarint32(pIter, &iThis); } *ppOut = ((iCol==iThis)?pIter:0); diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index 6986acdec6..7f71736888 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -524,6 +524,10 @@ struct Fts3MultiSegReader { int sqlite3Fts3Incrmerge(Fts3Table*,int,int); +#define fts3GetVarint32(p, piVal) ( \ + (*(u8*)(p)&0x80) ? sqlite3Fts3GetVarint32(p, piVal) : (*piVal=*(u8*)(p), 1) \ +) + /* fts3.c */ int sqlite3Fts3PutVarint(char *, sqlite3_int64); int sqlite3Fts3GetVarint(const char *, sqlite_int64 *); diff --git a/ext/fts3/fts3_snippet.c b/ext/fts3/fts3_snippet.c index 3e62799bb7..aa8779fa61 100644 --- a/ext/fts3/fts3_snippet.c +++ b/ext/fts3/fts3_snippet.c @@ -128,7 +128,7 @@ struct StrBuffer { */ static void fts3GetDeltaPosition(char **pp, int *piPos){ int iVal; - *pp += sqlite3Fts3GetVarint32(*pp, &iVal); + *pp += fts3GetVarint32(*pp, &iVal); *piPos += (iVal-2); } diff --git a/ext/fts3/fts3_test.c b/ext/fts3/fts3_test.c index 75ec6bd01d..36dcc94e6d 100644 --- a/ext/fts3/fts3_test.c +++ b/ext/fts3/fts3_test.c @@ -517,6 +517,51 @@ static int fts3_test_tokenizer_cmd( return TCL_OK; } +static int fts3_test_varint_cmd( + ClientData clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ +#ifdef SQLITE_ENABLE_FTS3 + char aBuf[24]; + int rc; + Tcl_WideInt w, w2; + int nByte, nByte2; + + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "INTEGER"); + return TCL_ERROR; + } + + rc = Tcl_GetWideIntFromObj(interp, objv[1], &w); + if( rc!=TCL_OK ) return rc; + + nByte = sqlite3Fts3PutVarint(aBuf, w); + nByte2 = sqlite3Fts3GetVarint(aBuf, &w2); + if( w!=w2 || nByte!=nByte2 ){ + char *zErr = sqlite3_mprintf("error testing %lld", w); + Tcl_ResetResult(interp); + Tcl_AppendResult(interp, zErr, 0); + return TCL_ERROR; + } + + if( w<=2147483647 && w>=0 ){ + int i; + nByte2 = fts3GetVarint32(aBuf, &i); + if( (int)w!=i || nByte!=nByte2 ){ + char *zErr = sqlite3_mprintf("error testing %lld (32-bit)", w); + Tcl_ResetResult(interp); + Tcl_AppendResult(interp, zErr, 0); + return TCL_ERROR; + } + } + +#endif + UNUSED_PARAMETER(clientData); + return TCL_OK; +} + /* ** End of tokenizer code. **************************************************************************/ @@ -529,6 +574,10 @@ int Sqlitetestfts3_Init(Tcl_Interp *interp){ Tcl_CreateObjCommand( interp, "fts3_test_tokenizer", fts3_test_tokenizer_cmd, 0, 0 ); + + Tcl_CreateObjCommand( + interp, "fts3_test_varint", fts3_test_varint_cmd, 0, 0 + ); return TCL_OK; } #endif /* SQLITE_ENABLE_FTS3 || SQLITE_ENABLE_FTS4 */ diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 794b6dcbf1..3564cf8206 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -1344,8 +1344,8 @@ static int fts3SegReaderNext( /* Because of the FTS3_NODE_PADDING bytes of padding, the following is ** safe (no risk of overread) even if the node data is corrupted. */ - pNext += sqlite3Fts3GetVarint32(pNext, &nPrefix); - pNext += sqlite3Fts3GetVarint32(pNext, &nSuffix); + pNext += fts3GetVarint32(pNext, &nPrefix); + pNext += fts3GetVarint32(pNext, &nSuffix); if( nPrefix<0 || nSuffix<=0 || &pNext[nSuffix]>&pReader->aNode[pReader->nNode] ){ @@ -1368,7 +1368,7 @@ static int fts3SegReaderNext( memcpy(&pReader->zTerm[nPrefix], pNext, nSuffix); pReader->nTerm = nPrefix+nSuffix; pNext += nSuffix; - pNext += sqlite3Fts3GetVarint32(pNext, &pReader->nDoclist); + pNext += fts3GetVarint32(pNext, &pReader->nDoclist); pReader->aDoclist = pNext; pReader->pOffsetList = 0; @@ -2529,7 +2529,7 @@ static void fts3ColumnFilter( break; } p = &pList[1]; - p += sqlite3Fts3GetVarint32(p, &iCurrent); + p += fts3GetVarint32(p, &iCurrent); } if( bZero && &pList[nList]!=pEnd ){ @@ -3494,9 +3494,9 @@ static int nodeReaderNext(NodeReader *p){ p->aNode = 0; }else{ if( bFirst==0 ){ - p->iOff += sqlite3Fts3GetVarint32(&p->aNode[p->iOff], &nPrefix); + p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &nPrefix); } - p->iOff += sqlite3Fts3GetVarint32(&p->aNode[p->iOff], &nSuffix); + p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &nSuffix); blobGrowBuffer(&p->term, nPrefix+nSuffix, &rc); if( rc==SQLITE_OK ){ @@ -3504,7 +3504,7 @@ static int nodeReaderNext(NodeReader *p){ p->term.n = nPrefix+nSuffix; p->iOff += nSuffix; if( p->iChild==0 ){ - p->iOff += sqlite3Fts3GetVarint32(&p->aNode[p->iOff], &p->nDoclist); + p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &p->nDoclist); p->aDoclist = &p->aNode[p->iOff]; p->iOff += p->nDoclist; } @@ -4556,7 +4556,7 @@ static int fts3IncrmergeHintPop(Blob *pHint, i64 *piAbsLevel, int *pnInput){ pHint->n = i; i += sqlite3Fts3GetVarint(&pHint->a[i], piAbsLevel); - i += sqlite3Fts3GetVarint32(&pHint->a[i], pnInput); + i += fts3GetVarint32(&pHint->a[i], pnInput); if( i!=nHint ) return SQLITE_CORRUPT_VTAB; return SQLITE_OK; diff --git a/manifest b/manifest index 2a3835525f..31a3efca1e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C A\sbetter\s(simpler)\sfix\sto\sthe\scount(*)\sproblem\saddressed\sin\sthe\sprevious\ncheck-in. -D 2013-11-12T15:33:40.954 +C When\spossible,\shave\sFTS\suse\s32-bit\soperations\sto\sread\svarints\sfrom\sthe\sdatabase. +D 2013-11-12T17:46:44.081 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in d12e4455cf7a36e42d3949876c1c3b88ff70867a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -78,25 +78,25 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers e0a8b81383ea60d0334d274fadf305ea14a8c314 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c f25ae5729d40cc4e661c0a552685038f27e72bc9 +F ext/fts3/fts3.c 7eb3b2935f0bd5dca23c2215b296606bdd7760f6 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h 8689f7cf85020e7f88d1e761eeac480c3b0ea7ad +F ext/fts3/fts3Int.h eb5f8029589f3d8f1dc7fd50c773326a640388b1 F ext/fts3/fts3_aux.c 5c211e17a64885faeb16b9ba7772f9d5445c2365 F ext/fts3/fts3_expr.c 5165c365cb5a035f5be8bb296f7aa3211d43e4ac F ext/fts3/fts3_hash.c 8dd2d06b66c72c628c2732555a32bc0943114914 F ext/fts3/fts3_hash.h 39cf6874dc239d6b4e30479b1975fe5b22a3caaf F ext/fts3/fts3_icu.c e319e108661147bcca8dd511cd562f33a1ba81b5 F ext/fts3/fts3_porter.c a465b49fcb8249a755792f87516eff182efa42b3 -F ext/fts3/fts3_snippet.c e8ee8c101dd9cfbc9568d134e869d2bd2f7f6d4d +F ext/fts3/fts3_snippet.c 51beb5c1498176fd9caccaf1c75b55cb803a985a F ext/fts3/fts3_term.c a521f75132f9a495bdca1bdd45949b3191c52763 -F ext/fts3/fts3_test.c f9a1a1702db1bfad3e2d0064746eeb808f125489 +F ext/fts3/fts3_test.c 8a3a78c4458b2d7c631fcf4b152a5cd656fa7038 F ext/fts3/fts3_tokenize_vtab.c 011170fe9eba5ff062f1a31d3188e00267716706 F ext/fts3/fts3_tokenizer.c bbdc731bc91338050675c6d1da9ab82147391e16 F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 F ext/fts3/fts3_unicode.c 92391b4b4fb043564c6539ea9b8661e3bcba47b9 F ext/fts3/fts3_unicode2.c 0113d3acf13429e6dc38e0647d1bc71211c31a4d -F ext/fts3/fts3_write.c 17817f0cb6c8555e1be5e073fbddf676c60f4ea9 +F ext/fts3/fts3_write.c cdebe72171a217b1465032dec103f8d17f7dad4d F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/fts3/tool/fts3view.c 6cfc5b67a5f0e09c0d698f9fd012c784bfaa9197 @@ -558,6 +558,7 @@ F test/fts3snippet.test d524af6bcef4714e059ef559113dbdc924cd33d1 F test/fts3sort.test ed34c716a11cc2009a35210e84ad5f9c102362ca F test/fts3tok1.test b10d0a12a0ab5f905cea1200b745de233f37443f F test/fts3tok_err.test 52273cd193b9036282f7bacb43da78c6be87418d +F test/fts3varint.test 23e79002ebebd6fbb1235504f7acb087b0fea741 F test/fts4aa.test 0c3152322c7f0b548cc942ad763eaba0da87ccca F test/fts4check.test 66fa274cab2b615f2fb338b257713aba8fad88a8 F test/fts4content.test 2e7252557d6d24afa101d9ba1de710d6140e6d06 @@ -731,7 +732,7 @@ F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test b09104b03160aca0d968d99e8cd2c5b1921a993d F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 F test/percentile.test b98fc868d71eb5619d42a1702e9ab91718cbed54 -F test/permutations.test e154f5ed66d4d4913a99a110e870c9407f75b055 +F test/permutations.test af3278cbea3a19e025d5169be8193ff48dc3f862 F test/pragma.test e882183ecd21d064cec5c7aaea174fbd36293429 F test/pragma2.test aea7b3d82c76034a2df2b38a13745172ddc0bc13 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 @@ -1136,7 +1137,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 91174779786be07d63f3c4a5277602ddc5f0ba26 -R a014858d7aea6c178ec71eb38d26a678 -U drh -Z 90958247e5f1aff2cac4a81cce729911 +P 0f924c6ef6cf2ac5a61aafa8dd8e3309b3970499 +R 5351cba7d5954e18211d3eb00255913e +U dan +Z 8f4bd276a0e96490eb3d64253721bcb5 diff --git a/manifest.uuid b/manifest.uuid index 3da4669fd2..bf44e43c65 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0f924c6ef6cf2ac5a61aafa8dd8e3309b3970499 \ No newline at end of file +aa7ba302ed13aedde89b5bcbe9119799c0da8a42 \ No newline at end of file diff --git a/test/fts3varint.test b/test/fts3varint.test new file mode 100644 index 0000000000..229c37bac6 --- /dev/null +++ b/test/fts3varint.test @@ -0,0 +1,118 @@ +# 2007 November 23 +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file runs all tests. +# +# $Id: fts3.test,v 1.2 2008/07/23 18:17:32 drh Exp $ + +set testdir [file dirname $argv0] +source $testdir/permutations.test +set testprefix fts3varint + +ifcapable !fts3 { + finish_test + return +} + +proc test_list {list} { + foreach n $list { fts3_test_varint $n } +} + +proc do_fts3_varint_test {tn list} { + uplevel [list do_test $tn [list test_list $list] {}] +} + +do_fts3_varint_test 1.0 { + 1 10 100 1000 10000 100000 1000000 + 10000000 100000000 1000000000 10000000000 + 100000000000 1000000000000 10000000000000 +} + +do_fts3_varint_test 1.1 { + -1 -10 -100 -1000 -10000 -100000 -1000000 + -10000000 -100000000 -1000000000 -10000000000 + -100000000000 -1000000000000 -10000000000000 +} + +do_fts3_varint_test 2.0 { 0 1 2 } +do_fts3_varint_test 2.1 { 1 2 3 } +do_fts3_varint_test 2.2 { 3 4 5 } +do_fts3_varint_test 2.3 { 7 8 9 } +do_fts3_varint_test 2.4 { 15 16 17 } +do_fts3_varint_test 2.5 { 31 32 33 } +do_fts3_varint_test 2.6 { 63 64 65 } +do_fts3_varint_test 2.7 { 127 128 129 } +do_fts3_varint_test 2.8 { 255 256 257 } +do_fts3_varint_test 2.9 { 511 512 513 } +do_fts3_varint_test 2.10 { 1023 1024 1025 } +do_fts3_varint_test 2.11 { 2047 2048 2049 } +do_fts3_varint_test 2.12 { 4095 4096 4097 } +do_fts3_varint_test 2.13 { 8191 8192 8193 } +do_fts3_varint_test 2.14 { 16383 16384 16385 } +do_fts3_varint_test 2.15 { 32767 32768 32769 } +do_fts3_varint_test 2.16 { 65535 65536 65537 } +do_fts3_varint_test 2.17 { 131071 131072 131073 } +do_fts3_varint_test 2.18 { 262143 262144 262145 } +do_fts3_varint_test 2.19 { 524287 524288 524289 } +do_fts3_varint_test 2.20 { 1048575 1048576 1048577 } +do_fts3_varint_test 2.21 { 2097151 2097152 2097153 } +do_fts3_varint_test 2.22 { 4194303 4194304 4194305 } +do_fts3_varint_test 2.23 { 8388607 8388608 8388609 } +do_fts3_varint_test 2.24 { 16777215 16777216 16777217 } +do_fts3_varint_test 2.25 { 33554431 33554432 33554433 } +do_fts3_varint_test 2.26 { 67108863 67108864 67108865 } +do_fts3_varint_test 2.27 { 134217727 134217728 134217729 } +do_fts3_varint_test 2.28 { 268435455 268435456 268435457 } +do_fts3_varint_test 2.29 { 536870911 536870912 536870913 } +do_fts3_varint_test 2.30 { 1073741823 1073741824 1073741825 } +do_fts3_varint_test 2.31 { 2147483647 2147483648 2147483649 } +do_fts3_varint_test 2.32 { 4294967295 4294967296 4294967297 } +do_fts3_varint_test 2.33 { 8589934591 8589934592 8589934593 } +do_fts3_varint_test 2.34 { 17179869183 17179869184 17179869185 } +do_fts3_varint_test 2.35 { 34359738367 34359738368 34359738369 } +do_fts3_varint_test 2.36 { 68719476735 68719476736 68719476737 } +do_fts3_varint_test 2.37 { 137438953471 137438953472 137438953473 } +do_fts3_varint_test 2.38 { 274877906943 274877906944 274877906945 } +do_fts3_varint_test 2.39 { 549755813887 549755813888 549755813889 } +do_fts3_varint_test 2.40 { 1099511627775 1099511627776 1099511627777 } +do_fts3_varint_test 2.41 { 2199023255551 2199023255552 2199023255553 } +do_fts3_varint_test 2.42 { 4398046511103 4398046511104 4398046511105 } +do_fts3_varint_test 2.43 { 8796093022207 8796093022208 8796093022209 } +do_fts3_varint_test 2.44 { 17592186044415 17592186044416 17592186044417 } +do_fts3_varint_test 2.45 { 35184372088831 35184372088832 35184372088833 } +do_fts3_varint_test 2.46 { 70368744177663 70368744177664 70368744177665 } +do_fts3_varint_test 2.47 { 140737488355327 140737488355328 140737488355329 } +do_fts3_varint_test 2.48 { 281474976710655 281474976710656 281474976710657 } +do_fts3_varint_test 2.49 { 562949953421311 562949953421312 562949953421313 } +do_fts3_varint_test 2.50 { 1125899906842623 1125899906842624 1125899906842625 } +do_fts3_varint_test 2.51 { 2251799813685247 2251799813685248 2251799813685249 } +do_fts3_varint_test 2.52 { 4503599627370495 4503599627370496 4503599627370497 } +do_fts3_varint_test 2.53 { 9007199254740991 9007199254740992 9007199254740993 } +do_fts3_varint_test 2.54 { + 18014398509481983 18014398509481984 18014398509481985 } +do_fts3_varint_test 2.55 { + 36028797018963967 36028797018963968 36028797018963969 } +do_fts3_varint_test 2.56 { + 72057594037927935 72057594037927936 72057594037927937 } +do_fts3_varint_test 2.57 { + 144115188075855871 144115188075855872 144115188075855873 } +do_fts3_varint_test 2.58 { + 288230376151711743 288230376151711744 288230376151711745 } +do_fts3_varint_test 2.59 { + 576460752303423487 576460752303423488 576460752303423489 } +do_fts3_varint_test 2.60 { + 1152921504606846975 1152921504606846976 1152921504606846977 } +do_fts3_varint_test 2.61 { + 2305843009213693951 2305843009213693952 2305843009213693953 } +do_fts3_varint_test 2.62 { + 4611686018427387903 4611686018427387904 4611686018427387905 } +do_fts3_varint_test 2.63 { + 9223372036854775807 9223372036854775808 9223372036854775809 } + +do_fts3_varint_test 3.0 { 18446744073709551615 -18446744073709551615 } + +finish_test diff --git a/test/permutations.test b/test/permutations.test index 54e9ede0be..b72d7ad3c7 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -195,6 +195,7 @@ test_suite "fts3" -prefix "" -description { fts3conf.test fts3prefix.test fts3fault2.test fts3corrupt.test fts3corrupt2.test fts3first.test fts4langid.test fts4merge.test fts4check.test fts4unicode.test fts4noti.test + fts3varint.test } test_suite "nofaultsim" -prefix "" -description { From e54df42d873bf72c3e96c0fc9adcddbb3b649a54 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 12 Nov 2013 18:37:25 +0000 Subject: [PATCH 02/14] Break out the structure and macro definitions of where.c into a separate header file whereInt.h for easier editing and debugging. FossilOrigin-Name: c44467124623733aac64096d605f16139b733a7f --- Makefile.in | 4 +- Makefile.msc | 6 +- main.mk | 6 +- manifest | 23 +-- manifest.uuid | 2 +- src/where.c | 442 +----------------------------------------- src/whereInt.h | 456 ++++++++++++++++++++++++++++++++++++++++++++ tool/mksqlite3c.tcl | 1 + 8 files changed, 482 insertions(+), 458 deletions(-) create mode 100644 src/whereInt.h diff --git a/Makefile.in b/Makefile.in index deb5019e50..b38edda821 100644 --- a/Makefile.in +++ b/Makefile.in @@ -281,7 +281,8 @@ SRC = \ $(TOP)/src/wal.c \ $(TOP)/src/wal.h \ $(TOP)/src/walker.c \ - $(TOP)/src/where.c + $(TOP)/src/where.c \ + $(TOP)/src/whereInt.h # Source code for extensions # @@ -465,6 +466,7 @@ HDR = \ $(TOP)/src/sqliteLimit.h \ $(TOP)/src/vdbe.h \ $(TOP)/src/vdbeInt.h \ + $(TOP)/src/whereInt.h \ config.h # Header files used by extensions diff --git a/Makefile.msc b/Makefile.msc index 44428f0f97..3bc01c07b4 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -668,7 +668,8 @@ SRC = \ $(TOP)\src\wal.c \ $(TOP)\src\wal.h \ $(TOP)\src\walker.c \ - $(TOP)\src\where.c + $(TOP)\src\where.c \ + $(TOP)\src\whereInt.h # Source code for extensions # @@ -854,7 +855,8 @@ HDR = \ $(TOP)\src\sqliteInt.h \ $(TOP)\src\sqliteLimit.h \ $(TOP)\src\vdbe.h \ - $(TOP)\src\vdbeInt.h + $(TOP)\src\vdbeInt.h \ + $(TOP)\src\whereInt.h # Header files used by extensions # diff --git a/main.mk b/main.mk index 7571996906..9f06366566 100644 --- a/main.mk +++ b/main.mk @@ -163,7 +163,8 @@ SRC = \ $(TOP)/src/wal.c \ $(TOP)/src/wal.h \ $(TOP)/src/walker.c \ - $(TOP)/src/where.c + $(TOP)/src/where.c \ + $(TOP)/src/whereInt.h # Source code for extensions # @@ -346,7 +347,8 @@ HDR = \ $(TOP)/src/sqliteInt.h \ $(TOP)/src/sqliteLimit.h \ $(TOP)/src/vdbe.h \ - $(TOP)/src/vdbeInt.h + $(TOP)/src/vdbeInt.h \ + $(TOP)/src/whereInt.h # Header files used by extensions # diff --git a/manifest b/manifest index 31a3efca1e..a11a4561d0 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C When\spossible,\shave\sFTS\suse\s32-bit\soperations\sto\sread\svarints\sfrom\sthe\sdatabase. -D 2013-11-12T17:46:44.081 +C Break\sout\sthe\sstructure\sand\smacro\sdefinitions\sof\swhere.c\sinto\sa\sseparate\nheader\sfile\swhereInt.h\sfor\seasier\sediting\sand\sdebugging. +D 2013-11-12T18:37:25.924 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f -F Makefile.in d12e4455cf7a36e42d3949876c1c3b88ff70867a +F Makefile.in 8a07bebafbfda0eb67728f4bd15a36201662d1a1 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 -F Makefile.msc eaf39e18f37637111c6e39ed35b1da9092c8816f +F Makefile.msc 19243469fcd1073319ea5b7db97609c1cfb3b1c0 F Makefile.vxworks db21ed42a01d5740e656b16f92cb5d8d5e5dd315 F README cd04a36fbc7ea56932a4052d7d0b7f09f27c33d6 F VERSION 52f7e22bfcec71a462e34194b4ae1671380fde59 @@ -142,7 +142,7 @@ F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt f439556c5ce01ced70987e5ee86549a45165d9ff -F main.mk fbd5fed974f68cc312e4f73770df0f5221b3b460 +F main.mk f57b51e64f50e9924b3f14df3c2e8b66893a8ce5 F mkdll.sh 7d09b23c05d56532e9d44a50868eb4b12ff4f74a F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f F mkextw.sh d2a981497b404d6498f5ff3e3b1f3816bdfcb338 @@ -293,7 +293,8 @@ F src/vtab.c 5a423b042eb1402ef77697d03d6a67378d97bc8d F src/wal.c 7dc3966ef98b74422267e7e6e46e07ff6c6eb1b4 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c e9e593d5bb798c3e67fc3893dfe7055c9e7d8d74 -F src/where.c a80cab07e904b65a57849f7f2f6a1bc075fbdbec +F src/where.c 7ce9564fc41539a45fd9c416ea4282b3defc1311 +F src/whereInt.h fda11df00f43f9d4a16cc215c0675c83ba362efd F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1107,7 +1108,7 @@ F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e F tool/mkpragmatab.tcl 17d40faae6c4b865633bfc5763821402a1cbefc3 F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 F tool/mksqlite3c-noext.tcl 8bce31074e4cbe631bb7676526a048335f4c9f02 -F tool/mksqlite3c.tcl d8dc444d403019167260e5578f5c362741f03696 +F tool/mksqlite3c.tcl e2ba20d3f690990079d17f3e5a7417dfb7ada543 F tool/mksqlite3h.tcl ba24038056f51fde07c0079c41885ab85e2cff12 F tool/mksqlite3internalh.tcl 3dca7bb5374cee003379b8cbac73714f610ef795 F tool/mkvsix.tcl 6477fb9dab838b7eea1eed50658ff1cda04850b5 @@ -1137,7 +1138,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 0f924c6ef6cf2ac5a61aafa8dd8e3309b3970499 -R 5351cba7d5954e18211d3eb00255913e -U dan -Z 8f4bd276a0e96490eb3d64253721bcb5 +P aa7ba302ed13aedde89b5bcbe9119799c0da8a42 +R b1d0892fd01796ef0307192d25431345 +U drh +Z e9ed81f2d95ba49c17059d8287cc88cd diff --git a/manifest.uuid b/manifest.uuid index bf44e43c65..4ff3badfb8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -aa7ba302ed13aedde89b5bcbe9119799c0da8a42 \ No newline at end of file +c44467124623733aac64096d605f16139b733a7f \ No newline at end of file diff --git a/src/where.c b/src/where.c index 60a9471dad..77b9eed51d 100644 --- a/src/where.c +++ b/src/where.c @@ -17,447 +17,7 @@ ** indices, you might also think of this module as the "query optimizer". */ #include "sqliteInt.h" - - -/* -** Trace output macros -*/ -#if defined(SQLITE_TEST) || defined(SQLITE_DEBUG) -/***/ int sqlite3WhereTrace = 0; -#endif -#if defined(SQLITE_DEBUG) \ - && (defined(SQLITE_TEST) || defined(SQLITE_ENABLE_WHERETRACE)) -# define WHERETRACE(K,X) if(sqlite3WhereTrace&(K)) sqlite3DebugPrintf X -# define WHERETRACE_ENABLED 1 -#else -# define WHERETRACE(K,X) -#endif - -/* Forward references -*/ -typedef struct WhereClause WhereClause; -typedef struct WhereMaskSet WhereMaskSet; -typedef struct WhereOrInfo WhereOrInfo; -typedef struct WhereAndInfo WhereAndInfo; -typedef struct WhereLevel WhereLevel; -typedef struct WhereLoop WhereLoop; -typedef struct WherePath WherePath; -typedef struct WhereTerm WhereTerm; -typedef struct WhereLoopBuilder WhereLoopBuilder; -typedef struct WhereScan WhereScan; -typedef struct WhereOrCost WhereOrCost; -typedef struct WhereOrSet WhereOrSet; - -/* -** This object contains information needed to implement a single nested -** loop in WHERE clause. -** -** Contrast this object with WhereLoop. This object describes the -** implementation of the loop. WhereLoop describes the algorithm. -** This object contains a pointer to the WhereLoop algorithm as one of -** its elements. -** -** The WhereInfo object contains a single instance of this object for -** each term in the FROM clause (which is to say, for each of the -** nested loops as implemented). The order of WhereLevel objects determines -** the loop nested order, with WhereInfo.a[0] being the outer loop and -** WhereInfo.a[WhereInfo.nLevel-1] being the inner loop. -*/ -struct WhereLevel { - int iLeftJoin; /* Memory cell used to implement LEFT OUTER JOIN */ - int iTabCur; /* The VDBE cursor used to access the table */ - int iIdxCur; /* The VDBE cursor used to access pIdx */ - int addrBrk; /* Jump here to break out of the loop */ - int addrNxt; /* Jump here to start the next IN combination */ - int addrCont; /* Jump here to continue with the next loop cycle */ - int addrFirst; /* First instruction of interior of the loop */ - int addrBody; /* Beginning of the body of this loop */ - u8 iFrom; /* Which entry in the FROM clause */ - u8 op, p5; /* Opcode and P5 of the opcode that ends the loop */ - int p1, p2; /* Operands of the opcode used to ends the loop */ - union { /* Information that depends on pWLoop->wsFlags */ - struct { - int nIn; /* Number of entries in aInLoop[] */ - struct InLoop { - int iCur; /* The VDBE cursor used by this IN operator */ - int addrInTop; /* Top of the IN loop */ - u8 eEndLoopOp; /* IN Loop terminator. OP_Next or OP_Prev */ - } *aInLoop; /* Information about each nested IN operator */ - } in; /* Used when pWLoop->wsFlags&WHERE_IN_ABLE */ - Index *pCovidx; /* Possible covering index for WHERE_MULTI_OR */ - } u; - struct WhereLoop *pWLoop; /* The selected WhereLoop object */ - Bitmask notReady; /* FROM entries not usable at this level */ -}; - -/* -** Each instance of this object represents an algorithm for evaluating one -** term of a join. Every term of the FROM clause will have at least -** one corresponding WhereLoop object (unless INDEXED BY constraints -** prevent a query solution - which is an error) and many terms of the -** FROM clause will have multiple WhereLoop objects, each describing a -** potential way of implementing that FROM-clause term, together with -** dependencies and cost estimates for using the chosen algorithm. -** -** Query planning consists of building up a collection of these WhereLoop -** objects, then computing a particular sequence of WhereLoop objects, with -** one WhereLoop object per FROM clause term, that satisfy all dependencies -** and that minimize the overall cost. -*/ -struct WhereLoop { - Bitmask prereq; /* Bitmask of other loops that must run first */ - Bitmask maskSelf; /* Bitmask identifying table iTab */ -#ifdef SQLITE_DEBUG - char cId; /* Symbolic ID of this loop for debugging use */ -#endif - u8 iTab; /* Position in FROM clause of table for this loop */ - u8 iSortIdx; /* Sorting index number. 0==None */ - LogEst rSetup; /* One-time setup cost (ex: create transient index) */ - LogEst rRun; /* Cost of running each loop */ - LogEst nOut; /* Estimated number of output rows */ - union { - struct { /* Information for internal btree tables */ - int nEq; /* Number of equality constraints */ - Index *pIndex; /* Index used, or NULL */ - } btree; - struct { /* Information for virtual tables */ - int idxNum; /* Index number */ - u8 needFree; /* True if sqlite3_free(idxStr) is needed */ - u8 isOrdered; /* True if satisfies ORDER BY */ - u16 omitMask; /* Terms that may be omitted */ - char *idxStr; /* Index identifier string */ - } vtab; - } u; - u32 wsFlags; /* WHERE_* flags describing the plan */ - u16 nLTerm; /* Number of entries in aLTerm[] */ - /**** whereLoopXfer() copies fields above ***********************/ -# define WHERE_LOOP_XFER_SZ offsetof(WhereLoop,nLSlot) - u16 nLSlot; /* Number of slots allocated for aLTerm[] */ - WhereTerm **aLTerm; /* WhereTerms used */ - WhereLoop *pNextLoop; /* Next WhereLoop object in the WhereClause */ - WhereTerm *aLTermSpace[4]; /* Initial aLTerm[] space */ -}; - -/* This object holds the prerequisites and the cost of running a -** subquery on one operand of an OR operator in the WHERE clause. -** See WhereOrSet for additional information -*/ -struct WhereOrCost { - Bitmask prereq; /* Prerequisites */ - LogEst rRun; /* Cost of running this subquery */ - LogEst nOut; /* Number of outputs for this subquery */ -}; - -/* The WhereOrSet object holds a set of possible WhereOrCosts that -** correspond to the subquery(s) of OR-clause processing. Only the -** best N_OR_COST elements are retained. -*/ -#define N_OR_COST 3 -struct WhereOrSet { - u16 n; /* Number of valid a[] entries */ - WhereOrCost a[N_OR_COST]; /* Set of best costs */ -}; - - -/* Forward declaration of methods */ -static int whereLoopResize(sqlite3*, WhereLoop*, int); - -/* -** Each instance of this object holds a sequence of WhereLoop objects -** that implement some or all of a query plan. -** -** Think of each WhereLoop object as a node in a graph with arcs -** showing dependencies and costs for travelling between nodes. (That is -** not a completely accurate description because WhereLoop costs are a -** vector, not a scalar, and because dependencies are many-to-one, not -** one-to-one as are graph nodes. But it is a useful visualization aid.) -** Then a WherePath object is a path through the graph that visits some -** or all of the WhereLoop objects once. -** -** The "solver" works by creating the N best WherePath objects of length -** 1. Then using those as a basis to compute the N best WherePath objects -** of length 2. And so forth until the length of WherePaths equals the -** number of nodes in the FROM clause. The best (lowest cost) WherePath -** at the end is the choosen query plan. -*/ -struct WherePath { - Bitmask maskLoop; /* Bitmask of all WhereLoop objects in this path */ - Bitmask revLoop; /* aLoop[]s that should be reversed for ORDER BY */ - LogEst nRow; /* Estimated number of rows generated by this path */ - LogEst rCost; /* Total cost of this path */ - u8 isOrdered; /* True if this path satisfies ORDER BY */ - u8 isOrderedValid; /* True if the isOrdered field is valid */ - WhereLoop **aLoop; /* Array of WhereLoop objects implementing this path */ -}; - -/* -** The query generator uses an array of instances of this structure to -** help it analyze the subexpressions of the WHERE clause. Each WHERE -** clause subexpression is separated from the others by AND operators, -** usually, or sometimes subexpressions separated by OR. -** -** All WhereTerms are collected into a single WhereClause structure. -** The following identity holds: -** -** WhereTerm.pWC->a[WhereTerm.idx] == WhereTerm -** -** When a term is of the form: -** -** X -** -** where X is a column name and is one of certain operators, -** then WhereTerm.leftCursor and WhereTerm.u.leftColumn record the -** cursor number and column number for X. WhereTerm.eOperator records -** the using a bitmask encoding defined by WO_xxx below. The -** use of a bitmask encoding for the operator allows us to search -** quickly for terms that match any of several different operators. -** -** A WhereTerm might also be two or more subterms connected by OR: -** -** (t1.X ) OR (t1.Y ) OR .... -** -** In this second case, wtFlag has the TERM_ORINFO bit set and eOperator==WO_OR -** and the WhereTerm.u.pOrInfo field points to auxiliary information that -** is collected about the OR clause. -** -** If a term in the WHERE clause does not match either of the two previous -** categories, then eOperator==0. The WhereTerm.pExpr field is still set -** to the original subexpression content and wtFlags is set up appropriately -** but no other fields in the WhereTerm object are meaningful. -** -** When eOperator!=0, prereqRight and prereqAll record sets of cursor numbers, -** but they do so indirectly. A single WhereMaskSet structure translates -** cursor number into bits and the translated bit is stored in the prereq -** fields. The translation is used in order to maximize the number of -** bits that will fit in a Bitmask. The VDBE cursor numbers might be -** spread out over the non-negative integers. For example, the cursor -** numbers might be 3, 8, 9, 10, 20, 23, 41, and 45. The WhereMaskSet -** translates these sparse cursor numbers into consecutive integers -** beginning with 0 in order to make the best possible use of the available -** bits in the Bitmask. So, in the example above, the cursor numbers -** would be mapped into integers 0 through 7. -** -** The number of terms in a join is limited by the number of bits -** in prereqRight and prereqAll. The default is 64 bits, hence SQLite -** is only able to process joins with 64 or fewer tables. -*/ -struct WhereTerm { - Expr *pExpr; /* Pointer to the subexpression that is this term */ - int iParent; /* Disable pWC->a[iParent] when this term disabled */ - int leftCursor; /* Cursor number of X in "X " */ - union { - int leftColumn; /* Column number of X in "X " */ - WhereOrInfo *pOrInfo; /* Extra information if (eOperator & WO_OR)!=0 */ - WhereAndInfo *pAndInfo; /* Extra information if (eOperator& WO_AND)!=0 */ - } u; - LogEst truthProb; /* Probability of truth for this expression */ - u16 eOperator; /* A WO_xx value describing */ - u8 wtFlags; /* TERM_xxx bit flags. See below */ - u8 nChild; /* Number of children that must disable us */ - WhereClause *pWC; /* The clause this term is part of */ - Bitmask prereqRight; /* Bitmask of tables used by pExpr->pRight */ - Bitmask prereqAll; /* Bitmask of tables referenced by pExpr */ -}; - -/* -** Allowed values of WhereTerm.wtFlags -*/ -#define TERM_DYNAMIC 0x01 /* Need to call sqlite3ExprDelete(db, pExpr) */ -#define TERM_VIRTUAL 0x02 /* Added by the optimizer. Do not code */ -#define TERM_CODED 0x04 /* This term is already coded */ -#define TERM_COPIED 0x08 /* Has a child */ -#define TERM_ORINFO 0x10 /* Need to free the WhereTerm.u.pOrInfo object */ -#define TERM_ANDINFO 0x20 /* Need to free the WhereTerm.u.pAndInfo obj */ -#define TERM_OR_OK 0x40 /* Used during OR-clause processing */ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 -# define TERM_VNULL 0x80 /* Manufactured x>NULL or x<=NULL term */ -#else -# define TERM_VNULL 0x00 /* Disabled if not using stat3 */ -#endif - -/* -** An instance of the WhereScan object is used as an iterator for locating -** terms in the WHERE clause that are useful to the query planner. -*/ -struct WhereScan { - WhereClause *pOrigWC; /* Original, innermost WhereClause */ - WhereClause *pWC; /* WhereClause currently being scanned */ - char *zCollName; /* Required collating sequence, if not NULL */ - char idxaff; /* Must match this affinity, if zCollName!=NULL */ - unsigned char nEquiv; /* Number of entries in aEquiv[] */ - unsigned char iEquiv; /* Next unused slot in aEquiv[] */ - u32 opMask; /* Acceptable operators */ - int k; /* Resume scanning at this->pWC->a[this->k] */ - int aEquiv[22]; /* Cursor,Column pairs for equivalence classes */ -}; - -/* -** An instance of the following structure holds all information about a -** WHERE clause. Mostly this is a container for one or more WhereTerms. -** -** Explanation of pOuter: For a WHERE clause of the form -** -** a AND ((b AND c) OR (d AND e)) AND f -** -** There are separate WhereClause objects for the whole clause and for -** the subclauses "(b AND c)" and "(d AND e)". The pOuter field of the -** subclauses points to the WhereClause object for the whole clause. -*/ -struct WhereClause { - WhereInfo *pWInfo; /* WHERE clause processing context */ - WhereClause *pOuter; /* Outer conjunction */ - u8 op; /* Split operator. TK_AND or TK_OR */ - int nTerm; /* Number of terms */ - int nSlot; /* Number of entries in a[] */ - WhereTerm *a; /* Each a[] describes a term of the WHERE cluase */ -#if defined(SQLITE_SMALL_STACK) - WhereTerm aStatic[1]; /* Initial static space for a[] */ -#else - WhereTerm aStatic[8]; /* Initial static space for a[] */ -#endif -}; - -/* -** A WhereTerm with eOperator==WO_OR has its u.pOrInfo pointer set to -** a dynamically allocated instance of the following structure. -*/ -struct WhereOrInfo { - WhereClause wc; /* Decomposition into subterms */ - Bitmask indexable; /* Bitmask of all indexable tables in the clause */ -}; - -/* -** A WhereTerm with eOperator==WO_AND has its u.pAndInfo pointer set to -** a dynamically allocated instance of the following structure. -*/ -struct WhereAndInfo { - WhereClause wc; /* The subexpression broken out */ -}; - -/* -** An instance of the following structure keeps track of a mapping -** between VDBE cursor numbers and bits of the bitmasks in WhereTerm. -** -** The VDBE cursor numbers are small integers contained in -** SrcList_item.iCursor and Expr.iTable fields. For any given WHERE -** clause, the cursor numbers might not begin with 0 and they might -** contain gaps in the numbering sequence. But we want to make maximum -** use of the bits in our bitmasks. This structure provides a mapping -** from the sparse cursor numbers into consecutive integers beginning -** with 0. -** -** If WhereMaskSet.ix[A]==B it means that The A-th bit of a Bitmask -** corresponds VDBE cursor number B. The A-th bit of a bitmask is 1<3, 5->1, 8->2, 29->0, -** 57->5, 73->4. Or one of 719 other combinations might be used. It -** does not really matter. What is important is that sparse cursor -** numbers all get mapped into bit numbers that begin with 0 and contain -** no gaps. -*/ -struct WhereMaskSet { - int n; /* Number of assigned cursor values */ - int ix[BMS]; /* Cursor assigned to each bit */ -}; - -/* -** This object is a convenience wrapper holding all information needed -** to construct WhereLoop objects for a particular query. -*/ -struct WhereLoopBuilder { - WhereInfo *pWInfo; /* Information about this WHERE */ - WhereClause *pWC; /* WHERE clause terms */ - ExprList *pOrderBy; /* ORDER BY clause */ - WhereLoop *pNew; /* Template WhereLoop */ - WhereOrSet *pOrSet; /* Record best loops here, if not NULL */ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 - UnpackedRecord *pRec; /* Probe for stat4 (if required) */ - int nRecValid; /* Number of valid fields currently in pRec */ -#endif -}; - -/* -** The WHERE clause processing routine has two halves. The -** first part does the start of the WHERE loop and the second -** half does the tail of the WHERE loop. An instance of -** this structure is returned by the first half and passed -** into the second half to give some continuity. -** -** An instance of this object holds the complete state of the query -** planner. -*/ -struct WhereInfo { - Parse *pParse; /* Parsing and code generating context */ - SrcList *pTabList; /* List of tables in the join */ - ExprList *pOrderBy; /* The ORDER BY clause or NULL */ - ExprList *pResultSet; /* Result set. DISTINCT operates on these */ - WhereLoop *pLoops; /* List of all WhereLoop objects */ - Bitmask revMask; /* Mask of ORDER BY terms that need reversing */ - LogEst nRowOut; /* Estimated number of output rows */ - u16 wctrlFlags; /* Flags originally passed to sqlite3WhereBegin() */ - u8 bOBSat; /* ORDER BY satisfied by indices */ - u8 okOnePass; /* Ok to use one-pass algorithm for UPDATE/DELETE */ - u8 untestedTerms; /* Not all WHERE terms resolved by outer loop */ - u8 eDistinct; /* One of the WHERE_DISTINCT_* values below */ - u8 nLevel; /* Number of nested loop */ - int iTop; /* The very beginning of the WHERE loop */ - int iContinue; /* Jump here to continue with next record */ - int iBreak; /* Jump here to break out of the loop */ - int savedNQueryLoop; /* pParse->nQueryLoop outside the WHERE loop */ - int aiCurOnePass[2]; /* OP_OpenWrite cursors for the ONEPASS opt */ - WhereMaskSet sMaskSet; /* Map cursor numbers to bitmasks */ - WhereClause sWC; /* Decomposition of the WHERE clause */ - WhereLevel a[1]; /* Information about each nest loop in WHERE */ -}; - -/* -** Bitmasks for the operators on WhereTerm objects. These are all -** operators that are of interest to the query planner. An -** OR-ed combination of these values can be used when searching for -** particular WhereTerms within a WhereClause. -*/ -#define WO_IN 0x001 -#define WO_EQ 0x002 -#define WO_LT (WO_EQ<<(TK_LT-TK_EQ)) -#define WO_LE (WO_EQ<<(TK_LE-TK_EQ)) -#define WO_GT (WO_EQ<<(TK_GT-TK_EQ)) -#define WO_GE (WO_EQ<<(TK_GE-TK_EQ)) -#define WO_MATCH 0x040 -#define WO_ISNULL 0x080 -#define WO_OR 0x100 /* Two or more OR-connected terms */ -#define WO_AND 0x200 /* Two or more AND-connected terms */ -#define WO_EQUIV 0x400 /* Of the form A==B, both columns */ -#define WO_NOOP 0x800 /* This term does not restrict search space */ - -#define WO_ALL 0xfff /* Mask of all possible WO_* values */ -#define WO_SINGLE 0x0ff /* Mask of all non-compound WO_* values */ - -/* -** These are definitions of bits in the WhereLoop.wsFlags field. -** The particular combination of bits in each WhereLoop help to -** determine the algorithm that WhereLoop represents. -*/ -#define WHERE_COLUMN_EQ 0x00000001 /* x=EXPR */ -#define WHERE_COLUMN_RANGE 0x00000002 /* xEXPR */ -#define WHERE_COLUMN_IN 0x00000004 /* x IN (...) */ -#define WHERE_COLUMN_NULL 0x00000008 /* x IS NULL */ -#define WHERE_CONSTRAINT 0x0000000f /* Any of the WHERE_COLUMN_xxx values */ -#define WHERE_TOP_LIMIT 0x00000010 /* xEXPR or x>=EXPR constraint */ -#define WHERE_BOTH_LIMIT 0x00000030 /* Both x>EXPR and xwsFlags */ + struct { + int nIn; /* Number of entries in aInLoop[] */ + struct InLoop { + int iCur; /* The VDBE cursor used by this IN operator */ + int addrInTop; /* Top of the IN loop */ + u8 eEndLoopOp; /* IN Loop terminator. OP_Next or OP_Prev */ + } *aInLoop; /* Information about each nested IN operator */ + } in; /* Used when pWLoop->wsFlags&WHERE_IN_ABLE */ + Index *pCovidx; /* Possible covering index for WHERE_MULTI_OR */ + } u; + struct WhereLoop *pWLoop; /* The selected WhereLoop object */ + Bitmask notReady; /* FROM entries not usable at this level */ +}; + +/* +** Each instance of this object represents an algorithm for evaluating one +** term of a join. Every term of the FROM clause will have at least +** one corresponding WhereLoop object (unless INDEXED BY constraints +** prevent a query solution - which is an error) and many terms of the +** FROM clause will have multiple WhereLoop objects, each describing a +** potential way of implementing that FROM-clause term, together with +** dependencies and cost estimates for using the chosen algorithm. +** +** Query planning consists of building up a collection of these WhereLoop +** objects, then computing a particular sequence of WhereLoop objects, with +** one WhereLoop object per FROM clause term, that satisfy all dependencies +** and that minimize the overall cost. +*/ +struct WhereLoop { + Bitmask prereq; /* Bitmask of other loops that must run first */ + Bitmask maskSelf; /* Bitmask identifying table iTab */ +#ifdef SQLITE_DEBUG + char cId; /* Symbolic ID of this loop for debugging use */ +#endif + u8 iTab; /* Position in FROM clause of table for this loop */ + u8 iSortIdx; /* Sorting index number. 0==None */ + LogEst rSetup; /* One-time setup cost (ex: create transient index) */ + LogEst rRun; /* Cost of running each loop */ + LogEst nOut; /* Estimated number of output rows */ + union { + struct { /* Information for internal btree tables */ + int nEq; /* Number of equality constraints */ + Index *pIndex; /* Index used, or NULL */ + } btree; + struct { /* Information for virtual tables */ + int idxNum; /* Index number */ + u8 needFree; /* True if sqlite3_free(idxStr) is needed */ + u8 isOrdered; /* True if satisfies ORDER BY */ + u16 omitMask; /* Terms that may be omitted */ + char *idxStr; /* Index identifier string */ + } vtab; + } u; + u32 wsFlags; /* WHERE_* flags describing the plan */ + u16 nLTerm; /* Number of entries in aLTerm[] */ + /**** whereLoopXfer() copies fields above ***********************/ +# define WHERE_LOOP_XFER_SZ offsetof(WhereLoop,nLSlot) + u16 nLSlot; /* Number of slots allocated for aLTerm[] */ + WhereTerm **aLTerm; /* WhereTerms used */ + WhereLoop *pNextLoop; /* Next WhereLoop object in the WhereClause */ + WhereTerm *aLTermSpace[4]; /* Initial aLTerm[] space */ +}; + +/* This object holds the prerequisites and the cost of running a +** subquery on one operand of an OR operator in the WHERE clause. +** See WhereOrSet for additional information +*/ +struct WhereOrCost { + Bitmask prereq; /* Prerequisites */ + LogEst rRun; /* Cost of running this subquery */ + LogEst nOut; /* Number of outputs for this subquery */ +}; + +/* The WhereOrSet object holds a set of possible WhereOrCosts that +** correspond to the subquery(s) of OR-clause processing. Only the +** best N_OR_COST elements are retained. +*/ +#define N_OR_COST 3 +struct WhereOrSet { + u16 n; /* Number of valid a[] entries */ + WhereOrCost a[N_OR_COST]; /* Set of best costs */ +}; + + +/* Forward declaration of methods */ +static int whereLoopResize(sqlite3*, WhereLoop*, int); + +/* +** Each instance of this object holds a sequence of WhereLoop objects +** that implement some or all of a query plan. +** +** Think of each WhereLoop object as a node in a graph with arcs +** showing dependencies and costs for travelling between nodes. (That is +** not a completely accurate description because WhereLoop costs are a +** vector, not a scalar, and because dependencies are many-to-one, not +** one-to-one as are graph nodes. But it is a useful visualization aid.) +** Then a WherePath object is a path through the graph that visits some +** or all of the WhereLoop objects once. +** +** The "solver" works by creating the N best WherePath objects of length +** 1. Then using those as a basis to compute the N best WherePath objects +** of length 2. And so forth until the length of WherePaths equals the +** number of nodes in the FROM clause. The best (lowest cost) WherePath +** at the end is the choosen query plan. +*/ +struct WherePath { + Bitmask maskLoop; /* Bitmask of all WhereLoop objects in this path */ + Bitmask revLoop; /* aLoop[]s that should be reversed for ORDER BY */ + LogEst nRow; /* Estimated number of rows generated by this path */ + LogEst rCost; /* Total cost of this path */ + u8 isOrdered; /* True if this path satisfies ORDER BY */ + u8 isOrderedValid; /* True if the isOrdered field is valid */ + WhereLoop **aLoop; /* Array of WhereLoop objects implementing this path */ +}; + +/* +** The query generator uses an array of instances of this structure to +** help it analyze the subexpressions of the WHERE clause. Each WHERE +** clause subexpression is separated from the others by AND operators, +** usually, or sometimes subexpressions separated by OR. +** +** All WhereTerms are collected into a single WhereClause structure. +** The following identity holds: +** +** WhereTerm.pWC->a[WhereTerm.idx] == WhereTerm +** +** When a term is of the form: +** +** X +** +** where X is a column name and is one of certain operators, +** then WhereTerm.leftCursor and WhereTerm.u.leftColumn record the +** cursor number and column number for X. WhereTerm.eOperator records +** the using a bitmask encoding defined by WO_xxx below. The +** use of a bitmask encoding for the operator allows us to search +** quickly for terms that match any of several different operators. +** +** A WhereTerm might also be two or more subterms connected by OR: +** +** (t1.X ) OR (t1.Y ) OR .... +** +** In this second case, wtFlag has the TERM_ORINFO bit set and eOperator==WO_OR +** and the WhereTerm.u.pOrInfo field points to auxiliary information that +** is collected about the OR clause. +** +** If a term in the WHERE clause does not match either of the two previous +** categories, then eOperator==0. The WhereTerm.pExpr field is still set +** to the original subexpression content and wtFlags is set up appropriately +** but no other fields in the WhereTerm object are meaningful. +** +** When eOperator!=0, prereqRight and prereqAll record sets of cursor numbers, +** but they do so indirectly. A single WhereMaskSet structure translates +** cursor number into bits and the translated bit is stored in the prereq +** fields. The translation is used in order to maximize the number of +** bits that will fit in a Bitmask. The VDBE cursor numbers might be +** spread out over the non-negative integers. For example, the cursor +** numbers might be 3, 8, 9, 10, 20, 23, 41, and 45. The WhereMaskSet +** translates these sparse cursor numbers into consecutive integers +** beginning with 0 in order to make the best possible use of the available +** bits in the Bitmask. So, in the example above, the cursor numbers +** would be mapped into integers 0 through 7. +** +** The number of terms in a join is limited by the number of bits +** in prereqRight and prereqAll. The default is 64 bits, hence SQLite +** is only able to process joins with 64 or fewer tables. +*/ +struct WhereTerm { + Expr *pExpr; /* Pointer to the subexpression that is this term */ + int iParent; /* Disable pWC->a[iParent] when this term disabled */ + int leftCursor; /* Cursor number of X in "X " */ + union { + int leftColumn; /* Column number of X in "X " */ + WhereOrInfo *pOrInfo; /* Extra information if (eOperator & WO_OR)!=0 */ + WhereAndInfo *pAndInfo; /* Extra information if (eOperator& WO_AND)!=0 */ + } u; + LogEst truthProb; /* Probability of truth for this expression */ + u16 eOperator; /* A WO_xx value describing */ + u8 wtFlags; /* TERM_xxx bit flags. See below */ + u8 nChild; /* Number of children that must disable us */ + WhereClause *pWC; /* The clause this term is part of */ + Bitmask prereqRight; /* Bitmask of tables used by pExpr->pRight */ + Bitmask prereqAll; /* Bitmask of tables referenced by pExpr */ +}; + +/* +** Allowed values of WhereTerm.wtFlags +*/ +#define TERM_DYNAMIC 0x01 /* Need to call sqlite3ExprDelete(db, pExpr) */ +#define TERM_VIRTUAL 0x02 /* Added by the optimizer. Do not code */ +#define TERM_CODED 0x04 /* This term is already coded */ +#define TERM_COPIED 0x08 /* Has a child */ +#define TERM_ORINFO 0x10 /* Need to free the WhereTerm.u.pOrInfo object */ +#define TERM_ANDINFO 0x20 /* Need to free the WhereTerm.u.pAndInfo obj */ +#define TERM_OR_OK 0x40 /* Used during OR-clause processing */ +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +# define TERM_VNULL 0x80 /* Manufactured x>NULL or x<=NULL term */ +#else +# define TERM_VNULL 0x00 /* Disabled if not using stat3 */ +#endif + +/* +** An instance of the WhereScan object is used as an iterator for locating +** terms in the WHERE clause that are useful to the query planner. +*/ +struct WhereScan { + WhereClause *pOrigWC; /* Original, innermost WhereClause */ + WhereClause *pWC; /* WhereClause currently being scanned */ + char *zCollName; /* Required collating sequence, if not NULL */ + char idxaff; /* Must match this affinity, if zCollName!=NULL */ + unsigned char nEquiv; /* Number of entries in aEquiv[] */ + unsigned char iEquiv; /* Next unused slot in aEquiv[] */ + u32 opMask; /* Acceptable operators */ + int k; /* Resume scanning at this->pWC->a[this->k] */ + int aEquiv[22]; /* Cursor,Column pairs for equivalence classes */ +}; + +/* +** An instance of the following structure holds all information about a +** WHERE clause. Mostly this is a container for one or more WhereTerms. +** +** Explanation of pOuter: For a WHERE clause of the form +** +** a AND ((b AND c) OR (d AND e)) AND f +** +** There are separate WhereClause objects for the whole clause and for +** the subclauses "(b AND c)" and "(d AND e)". The pOuter field of the +** subclauses points to the WhereClause object for the whole clause. +*/ +struct WhereClause { + WhereInfo *pWInfo; /* WHERE clause processing context */ + WhereClause *pOuter; /* Outer conjunction */ + u8 op; /* Split operator. TK_AND or TK_OR */ + int nTerm; /* Number of terms */ + int nSlot; /* Number of entries in a[] */ + WhereTerm *a; /* Each a[] describes a term of the WHERE cluase */ +#if defined(SQLITE_SMALL_STACK) + WhereTerm aStatic[1]; /* Initial static space for a[] */ +#else + WhereTerm aStatic[8]; /* Initial static space for a[] */ +#endif +}; + +/* +** A WhereTerm with eOperator==WO_OR has its u.pOrInfo pointer set to +** a dynamically allocated instance of the following structure. +*/ +struct WhereOrInfo { + WhereClause wc; /* Decomposition into subterms */ + Bitmask indexable; /* Bitmask of all indexable tables in the clause */ +}; + +/* +** A WhereTerm with eOperator==WO_AND has its u.pAndInfo pointer set to +** a dynamically allocated instance of the following structure. +*/ +struct WhereAndInfo { + WhereClause wc; /* The subexpression broken out */ +}; + +/* +** An instance of the following structure keeps track of a mapping +** between VDBE cursor numbers and bits of the bitmasks in WhereTerm. +** +** The VDBE cursor numbers are small integers contained in +** SrcList_item.iCursor and Expr.iTable fields. For any given WHERE +** clause, the cursor numbers might not begin with 0 and they might +** contain gaps in the numbering sequence. But we want to make maximum +** use of the bits in our bitmasks. This structure provides a mapping +** from the sparse cursor numbers into consecutive integers beginning +** with 0. +** +** If WhereMaskSet.ix[A]==B it means that The A-th bit of a Bitmask +** corresponds VDBE cursor number B. The A-th bit of a bitmask is 1<3, 5->1, 8->2, 29->0, +** 57->5, 73->4. Or one of 719 other combinations might be used. It +** does not really matter. What is important is that sparse cursor +** numbers all get mapped into bit numbers that begin with 0 and contain +** no gaps. +*/ +struct WhereMaskSet { + int n; /* Number of assigned cursor values */ + int ix[BMS]; /* Cursor assigned to each bit */ +}; + +/* +** This object is a convenience wrapper holding all information needed +** to construct WhereLoop objects for a particular query. +*/ +struct WhereLoopBuilder { + WhereInfo *pWInfo; /* Information about this WHERE */ + WhereClause *pWC; /* WHERE clause terms */ + ExprList *pOrderBy; /* ORDER BY clause */ + WhereLoop *pNew; /* Template WhereLoop */ + WhereOrSet *pOrSet; /* Record best loops here, if not NULL */ +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + UnpackedRecord *pRec; /* Probe for stat4 (if required) */ + int nRecValid; /* Number of valid fields currently in pRec */ +#endif +}; + +/* +** The WHERE clause processing routine has two halves. The +** first part does the start of the WHERE loop and the second +** half does the tail of the WHERE loop. An instance of +** this structure is returned by the first half and passed +** into the second half to give some continuity. +** +** An instance of this object holds the complete state of the query +** planner. +*/ +struct WhereInfo { + Parse *pParse; /* Parsing and code generating context */ + SrcList *pTabList; /* List of tables in the join */ + ExprList *pOrderBy; /* The ORDER BY clause or NULL */ + ExprList *pResultSet; /* Result set. DISTINCT operates on these */ + WhereLoop *pLoops; /* List of all WhereLoop objects */ + Bitmask revMask; /* Mask of ORDER BY terms that need reversing */ + LogEst nRowOut; /* Estimated number of output rows */ + u16 wctrlFlags; /* Flags originally passed to sqlite3WhereBegin() */ + u8 bOBSat; /* ORDER BY satisfied by indices */ + u8 okOnePass; /* Ok to use one-pass algorithm for UPDATE/DELETE */ + u8 untestedTerms; /* Not all WHERE terms resolved by outer loop */ + u8 eDistinct; /* One of the WHERE_DISTINCT_* values below */ + u8 nLevel; /* Number of nested loop */ + int iTop; /* The very beginning of the WHERE loop */ + int iContinue; /* Jump here to continue with next record */ + int iBreak; /* Jump here to break out of the loop */ + int savedNQueryLoop; /* pParse->nQueryLoop outside the WHERE loop */ + int aiCurOnePass[2]; /* OP_OpenWrite cursors for the ONEPASS opt */ + WhereMaskSet sMaskSet; /* Map cursor numbers to bitmasks */ + WhereClause sWC; /* Decomposition of the WHERE clause */ + WhereLevel a[1]; /* Information about each nest loop in WHERE */ +}; + +/* +** Bitmasks for the operators on WhereTerm objects. These are all +** operators that are of interest to the query planner. An +** OR-ed combination of these values can be used when searching for +** particular WhereTerms within a WhereClause. +*/ +#define WO_IN 0x001 +#define WO_EQ 0x002 +#define WO_LT (WO_EQ<<(TK_LT-TK_EQ)) +#define WO_LE (WO_EQ<<(TK_LE-TK_EQ)) +#define WO_GT (WO_EQ<<(TK_GT-TK_EQ)) +#define WO_GE (WO_EQ<<(TK_GE-TK_EQ)) +#define WO_MATCH 0x040 +#define WO_ISNULL 0x080 +#define WO_OR 0x100 /* Two or more OR-connected terms */ +#define WO_AND 0x200 /* Two or more AND-connected terms */ +#define WO_EQUIV 0x400 /* Of the form A==B, both columns */ +#define WO_NOOP 0x800 /* This term does not restrict search space */ + +#define WO_ALL 0xfff /* Mask of all possible WO_* values */ +#define WO_SINGLE 0x0ff /* Mask of all non-compound WO_* values */ + +/* +** These are definitions of bits in the WhereLoop.wsFlags field. +** The particular combination of bits in each WhereLoop help to +** determine the algorithm that WhereLoop represents. +*/ +#define WHERE_COLUMN_EQ 0x00000001 /* x=EXPR */ +#define WHERE_COLUMN_RANGE 0x00000002 /* xEXPR */ +#define WHERE_COLUMN_IN 0x00000004 /* x IN (...) */ +#define WHERE_COLUMN_NULL 0x00000008 /* x IS NULL */ +#define WHERE_CONSTRAINT 0x0000000f /* Any of the WHERE_COLUMN_xxx values */ +#define WHERE_TOP_LIMIT 0x00000010 /* xEXPR or x>=EXPR constraint */ +#define WHERE_BOTH_LIMIT 0x00000030 /* Both x>EXPR and x Date: Tue, 12 Nov 2013 20:18:14 +0000 Subject: [PATCH 03/14] Minor enhancements to the auxiliary information added to EXPLAIN output with SQLITE_EXPLAIN_ENABLE_COMMENTS. FossilOrigin-Name: 0d1328e33ca761eddcf8a50e8e83c997861e9047 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/vdbeaux.c | 2 +- src/where.c | 6 ++++-- src/whereInt.h | 3 ++- 5 files changed, 16 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index a11a4561d0..496964f919 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Break\sout\sthe\sstructure\sand\smacro\sdefinitions\sof\swhere.c\sinto\sa\sseparate\nheader\sfile\swhereInt.h\sfor\seasier\sediting\sand\sdebugging. -D 2013-11-12T18:37:25.924 +C Minor\senhancements\sto\sthe\sauxiliary\sinformation\sadded\sto\sEXPLAIN\soutput\swith\nSQLITE_EXPLAIN_ENABLE_COMMENTS. +D 2013-11-12T20:18:14.367 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 8a07bebafbfda0eb67728f4bd15a36201662d1a1 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -284,7 +284,7 @@ F src/vdbe.c fda0d2ba5a3a5ab588f0467782e7fd64930ddb22 F src/vdbe.h 8d5a7351024d80374fc0acdbbe3cfe65c51ba8b6 F src/vdbeInt.h f2fa3ceccceeb757773921fb08af7c6e9f3caa1c F src/vdbeapi.c 93a22a9ba2abe292d5c2cf304d7eb2e894dde0ed -F src/vdbeaux.c 84d2d421c11c3e534d4948274e4d0a5d5816a695 +F src/vdbeaux.c 75c5fee307004739d814a840699df5e853e02b79 F src/vdbeblob.c ff60adf2953e0ffc1d59541b3101c9886b03a3de F src/vdbemem.c cc529bbf4f13e4e181bdb446bf6e6962ab030b4b F src/vdbesort.c 9d83601f9d6243fe70dd0169a2820c5ddfd48147 @@ -293,8 +293,8 @@ F src/vtab.c 5a423b042eb1402ef77697d03d6a67378d97bc8d F src/wal.c 7dc3966ef98b74422267e7e6e46e07ff6c6eb1b4 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c e9e593d5bb798c3e67fc3893dfe7055c9e7d8d74 -F src/where.c 7ce9564fc41539a45fd9c416ea4282b3defc1311 -F src/whereInt.h fda11df00f43f9d4a16cc215c0675c83ba362efd +F src/where.c 48404454d772af51f9075a33b485bd2270f31dd3 +F src/whereInt.h 63c8345d01d12ded6250b72e7c16855f0a358041 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1138,7 +1138,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P aa7ba302ed13aedde89b5bcbe9119799c0da8a42 -R b1d0892fd01796ef0307192d25431345 +P c44467124623733aac64096d605f16139b733a7f +R 6e0586ae35f6a5b0a0e54c1d5a933029 U drh -Z e9ed81f2d95ba49c17059d8287cc88cd +Z b01fa55693742bbd2cb099540a606861 diff --git a/manifest.uuid b/manifest.uuid index 4ff3badfb8..596080b252 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c44467124623733aac64096d605f16139b733a7f \ No newline at end of file +0d1328e33ca761eddcf8a50e8e83c997861e9047 \ No newline at end of file diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 3ada74e48b..7ce9c2a6c1 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -982,7 +982,7 @@ static char *displayP4(Op *pOp, char *zTemp, int nTemp){ } case P4_COLLSEQ: { CollSeq *pColl = pOp->p4.pColl; - sqlite3_snprintf(nTemp, zTemp, "collseq(%.20s)", pColl->zName); + sqlite3_snprintf(nTemp, zTemp, "(%.20s)", pColl->zName); break; } case P4_FUNCDEF: { diff --git a/src/where.c b/src/where.c index 77b9eed51d..10383ce411 100644 --- a/src/where.c +++ b/src/where.c @@ -2725,7 +2725,7 @@ static Bitmask codeOneLoopStart( bRev = (pWInfo->revMask>>iLevel)&1; omitTable = (pLoop->wsFlags & WHERE_IDX_ONLY)!=0 && (pWInfo->wctrlFlags & WHERE_FORCE_TABLE)==0; - VdbeNoopComment((v, "Begin WHERE-Loop %d: %s", iLevel,pTabItem->pTab->zName)); + VdbeNoopComment((v, "Begin WHERE-loop%d: %s",iLevel,pTabItem->pTab->zName)); /* Create labels for the "break" and "continue" instructions ** for the current loop. Jump to addrBrk to break out of a loop. @@ -5679,6 +5679,7 @@ WhereInfo *sqlite3WhereBegin( } /* Done. */ + VdbeNoopComment((v, "Begin WHERE-core")); return pWInfo; /* Jump here if malloc fails */ @@ -5705,6 +5706,7 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ /* Generate loop termination code. */ + VdbeNoopComment((v, "End WHERE-core")); sqlite3ExprCacheClear(pParse); for(i=pWInfo->nLevel-1; i>=0; i--){ pLevel = &pWInfo->a[i]; @@ -5744,7 +5746,7 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ } sqlite3VdbeJumpHere(v, addr); } - VdbeNoopComment((v, "End WHERE-Loop %d: %s", i, + VdbeNoopComment((v, "End WHERE-loop%d: %s", i, pWInfo->pTabList->a[pLevel->iFrom].pTab->zName)); } diff --git a/src/whereInt.h b/src/whereInt.h index 37ba416954..b73353f732 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -113,7 +113,8 @@ struct WhereLoop { LogEst nOut; /* Estimated number of output rows */ union { struct { /* Information for internal btree tables */ - int nEq; /* Number of equality constraints */ + u16 nEq; /* Number of equality constraints */ + u16 nSkip; /* Number of initial index columns to skip */ Index *pIndex; /* Index used, or NULL */ } btree; struct { /* Information for virtual tables */ From 3aa4be39a9967bc4c4bc793fa43d9d5fb97fee1e Mon Sep 17 00:00:00 2001 From: mistachkin Date: Tue, 12 Nov 2013 21:10:02 +0000 Subject: [PATCH 04/14] Fix harmless compiler warning. FossilOrigin-Name: ddacd10105c6df2d3a9d707947e72c62e88212eb --- ext/fts3/fts3.c | 1 - manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 9f6cc482af..1719bac7d1 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -343,7 +343,6 @@ int sqlite3Fts3GetVarint(const char *p, sqlite_int64 *v){ const char *pStart = p; u32 a; u64 b; - int ret; int shift; GETVARINT_STEP(a, p, 0, 0x00, 0x80, *v, 1); diff --git a/manifest b/manifest index 496964f919..4667406b87 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\senhancements\sto\sthe\sauxiliary\sinformation\sadded\sto\sEXPLAIN\soutput\swith\nSQLITE_EXPLAIN_ENABLE_COMMENTS. -D 2013-11-12T20:18:14.367 +C Fix\sharmless\scompiler\swarning. +D 2013-11-12T21:10:02.483 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 8a07bebafbfda0eb67728f4bd15a36201662d1a1 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -78,7 +78,7 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers e0a8b81383ea60d0334d274fadf305ea14a8c314 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 7eb3b2935f0bd5dca23c2215b296606bdd7760f6 +F ext/fts3/fts3.c dceaa5079833caa2e7945433e1eb93bb22f623a3 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h eb5f8029589f3d8f1dc7fd50c773326a640388b1 F ext/fts3/fts3_aux.c 5c211e17a64885faeb16b9ba7772f9d5445c2365 @@ -1138,7 +1138,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P c44467124623733aac64096d605f16139b733a7f -R 6e0586ae35f6a5b0a0e54c1d5a933029 -U drh -Z b01fa55693742bbd2cb099540a606861 +P 0d1328e33ca761eddcf8a50e8e83c997861e9047 +R 691489a4da1e379548bfdbb95fa25c81 +U mistachkin +Z 7d610a332d7c23a451ca45725a291be3 diff --git a/manifest.uuid b/manifest.uuid index 596080b252..618f0d117a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0d1328e33ca761eddcf8a50e8e83c997861e9047 \ No newline at end of file +ddacd10105c6df2d3a9d707947e72c62e88212eb \ No newline at end of file From 015a304f75dccd80d9045d157f6215d7b4522dfe Mon Sep 17 00:00:00 2001 From: mistachkin Date: Tue, 12 Nov 2013 21:37:04 +0000 Subject: [PATCH 05/14] Adjust the SQLITE_MALLOCSIZE defines, primarily to make sure _msize gets used with MSVC when appropriate. FossilOrigin-Name: 4e7e805e1139b2dc05d85e86e5c8254e5d361bf2 --- manifest | 12 +++++----- manifest.uuid | 2 +- src/mem1.c | 62 ++++++++++++++++++++++++++++++++------------------- 3 files changed, 46 insertions(+), 30 deletions(-) diff --git a/manifest b/manifest index 4667406b87..89efd23eb7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarning. -D 2013-11-12T21:10:02.483 +C Adjust\sthe\sSQLITE_MALLOCSIZE\sdefines,\sprimarily\sto\smake\ssure\s_msize\sgets\sused\swith\sMSVC\swhen\sappropriate. +D 2013-11-12T21:37:04.790 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 8a07bebafbfda0eb67728f4bd15a36201662d1a1 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -191,7 +191,7 @@ F src/loadext.c 867c7b330b740c6c917af9956b13b81d0a048303 F src/main.c 32bf1e6e164a6fa0ddf1bf2616c6eafbefd6e9b0 F src/malloc.c 543a8eb5508eaf4cadf55a9b503379eba2088128 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 -F src/mem1.c 437c7c4af964895d4650f29881df63535caaa1fa +F src/mem1.c c0c990fcaddff810ea277b4fb5d9138603dd5d4b F src/mem2.c dce31758da87ec2cfa52ba4c5df1aed6e07d8e8f F src/mem3.c 61c9d47b792908c532ca3a62b999cf21795c6534 F src/mem5.c 0025308a93838022bd5696cf9627ff4e40b19918 @@ -1138,7 +1138,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 0d1328e33ca761eddcf8a50e8e83c997861e9047 -R 691489a4da1e379548bfdbb95fa25c81 +P ddacd10105c6df2d3a9d707947e72c62e88212eb +R a1432779dd22915a7d5838c89a07f3c3 U mistachkin -Z 7d610a332d7c23a451ca45725a291be3 +Z f4a84974fe637056c9ba67de537af83a diff --git a/manifest.uuid b/manifest.uuid index 618f0d117a..54f0297c7a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ddacd10105c6df2d3a9d707947e72c62e88212eb \ No newline at end of file +4e7e805e1139b2dc05d85e86e5c8254e5d361bf2 \ No newline at end of file diff --git a/src/mem1.c b/src/mem1.c index 3578496f33..6dbf1058ea 100644 --- a/src/mem1.c +++ b/src/mem1.c @@ -49,16 +49,6 @@ ** macros. */ #ifdef SQLITE_SYSTEM_MALLOC - -/* -** The MSVCRT has malloc_usable_size() but it is called _msize(). -** The use of _msize() is automatic, but can be disabled by compiling -** with -DSQLITE_WITHOUT_MSIZE -*/ -#if defined(_MSC_VER) && !defined(SQLITE_WITHOUT_MSIZE) -# define SQLITE_MALLOCSIZE _msize -#endif - #if defined(__APPLE__) && !defined(SQLITE_WITHOUT_ZONEMALLOC) /* @@ -81,22 +71,48 @@ static malloc_zone_t* _sqliteZone_; ** Use standard C library malloc and free on non-Apple systems. ** Also used by Apple systems if SQLITE_WITHOUT_ZONEMALLOC is defined. */ -#define SQLITE_MALLOC(x) malloc(x) -#define SQLITE_FREE(x) free(x) -#define SQLITE_REALLOC(x,y) realloc((x),(y)) +#define SQLITE_MALLOC(x) malloc(x) +#define SQLITE_FREE(x) free(x) +#define SQLITE_REALLOC(x,y) realloc((x),(y)) -#if (defined(_MSC_VER) && !defined(SQLITE_WITHOUT_MSIZE)) \ - || (defined(HAVE_MALLOC_H) && defined(HAVE_MALLOC_USABLE_SIZE)) -# include /* Needed for malloc_usable_size on linux */ -#endif -#ifdef HAVE_MALLOC_USABLE_SIZE -# ifndef SQLITE_MALLOCSIZE -# define SQLITE_MALLOCSIZE(x) malloc_usable_size(x) -# endif -#else -# undef SQLITE_MALLOCSIZE +/* +** The malloc.h header file is needed for malloc_usable_size() function +** on some systems (e.g. Linux). +*/ +#if defined(HAVE_MALLOC_H) && defined(HAVE_MALLOC_USABLE_SIZE) +# define SQLITE_USE_MALLOC_H +# define SQLITE_USE_MALLOC_USABLE_SIZE +/* +** The MSVCRT has malloc_usable_size(), but it is called _msize(). The +** use of _msize() is automatic, but can be disabled by compiling with +** -DSQLITE_WITHOUT_MSIZE. Using the _msize() function also requires +** the malloc.h header file. +*/ +#elif defined(_MSC_VER) && !defined(SQLITE_WITHOUT_MSIZE) +# define SQLITE_USE_MALLOC_H +# define SQLITE_USE_MSIZE #endif +/* +** Include the malloc.h header file, if necessary. Also set define macro +** SQLITE_MALLOCSIZE to the appropriate function name, which is _msize() +** for MSVC and malloc_usable_size() for most other systems (e.g. Linux). +** The memory size function can always be overridden manually by defining +** the macro SQLITE_MALLOCSIZE to the desired function name. +*/ +#if defined(SQLITE_USE_MALLOC_H) +# include +# if defined(SQLITE_USE_MALLOC_USABLE_SIZE) +# if !defined(SQLITE_MALLOCSIZE) +# define SQLITE_MALLOCSIZE(x) malloc_usable_size(x) +# endif +# elif defined(SQLITE_USE_MSIZE) +# if !defined(SQLITE_MALLOCSIZE) +# define SQLITE_MALLOCSIZE _msize +# endif +# endif +#endif /* defined(SQLITE_USE_MALLOC_H) */ + #endif /* __APPLE__ or not __APPLE__ */ /* From c181c26cba2693ed88c52e4c0d7b96bf063901fe Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 13 Nov 2013 08:55:02 +0000 Subject: [PATCH 06/14] Avoid an unnecessary OP_IfNull while doing an indexed search. FossilOrigin-Name: 5196000930600d0cd931b87e864507791b9dab08 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 6 +++++- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 89efd23eb7..1bdb81fbcb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Adjust\sthe\sSQLITE_MALLOCSIZE\sdefines,\sprimarily\sto\smake\ssure\s_msize\sgets\sused\swith\sMSVC\swhen\sappropriate. -D 2013-11-12T21:37:04.790 +C Avoid\san\sunnecessary\sOP_IfNull\swhile\sdoing\san\sindexed\ssearch. +D 2013-11-13T08:55:02.907 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 8a07bebafbfda0eb67728f4bd15a36201662d1a1 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -293,7 +293,7 @@ F src/vtab.c 5a423b042eb1402ef77697d03d6a67378d97bc8d F src/wal.c 7dc3966ef98b74422267e7e6e46e07ff6c6eb1b4 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c e9e593d5bb798c3e67fc3893dfe7055c9e7d8d74 -F src/where.c 48404454d772af51f9075a33b485bd2270f31dd3 +F src/where.c 346944c20b9c1ac191f16cfc1100dfbe58fa98d4 F src/whereInt.h 63c8345d01d12ded6250b72e7c16855f0a358041 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1138,7 +1138,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P ddacd10105c6df2d3a9d707947e72c62e88212eb -R a1432779dd22915a7d5838c89a07f3c3 -U mistachkin -Z f4a84974fe637056c9ba67de537af83a +P 4e7e805e1139b2dc05d85e86e5c8254e5d361bf2 +R 24bac22755e297cd5d201f4faa2841ac +U drh +Z b52683f99c1f13f39fe1493c1b7d78a3 diff --git a/manifest.uuid b/manifest.uuid index 54f0297c7a..5a991f73f9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4e7e805e1139b2dc05d85e86e5c8254e5d361bf2 \ No newline at end of file +5196000930600d0cd931b87e864507791b9dab08 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 10383ce411..617d373cbb 100644 --- a/src/where.c +++ b/src/where.c @@ -3116,8 +3116,12 @@ static Bitmask codeOneLoopStart( r1 = sqlite3GetTempReg(pParse); testcase( pLoop->wsFlags & WHERE_BTM_LIMIT ); testcase( pLoop->wsFlags & WHERE_TOP_LIMIT ); - if( (pLoop->wsFlags & (WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))!=0 ){ + if( (pLoop->wsFlags & (WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))!=0 + && (j = pIdx->aiColumn[nEq])>=0 + && pIdx->pTable->aCol[j].notNull==0 + ){ sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, nEq, r1); + VdbeComment((v, "%s", pIdx->pTable->aCol[j].zName)); sqlite3VdbeAddOp2(v, OP_IsNull, r1, addrCont); } sqlite3ReleaseTempReg(pParse, r1); From cd8629e4bbc8f91e634337d72453b3963f50990e Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 13 Nov 2013 12:27:25 +0000 Subject: [PATCH 07/14] Add the ability to use an index even if the left-most columns of the index are unconstrainted, provided that the left-most columns have few distinct values. FossilOrigin-Name: 27dd5993d1ae5625eb94bf406421eb390d001be9 --- manifest | 17 ++++++---- manifest.uuid | 2 +- src/where.c | 87 +++++++++++++++++++++++++++++++++++++++++--------- src/whereInt.h | 5 +++ 4 files changed, 88 insertions(+), 23 deletions(-) diff --git a/manifest b/manifest index 1bdb81fbcb..5d4c908bea 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\san\sunnecessary\sOP_IfNull\swhile\sdoing\san\sindexed\ssearch. -D 2013-11-13T08:55:02.907 +C Add\sthe\sability\sto\suse\san\sindex\seven\sif\sthe\sleft-most\scolumns\sof\sthe\sindex\nare\sunconstrainted,\sprovided\sthat\sthe\sleft-most\scolumns\shave\sfew\sdistinct\nvalues. +D 2013-11-13T12:27:25.442 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 8a07bebafbfda0eb67728f4bd15a36201662d1a1 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -293,8 +293,8 @@ F src/vtab.c 5a423b042eb1402ef77697d03d6a67378d97bc8d F src/wal.c 7dc3966ef98b74422267e7e6e46e07ff6c6eb1b4 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c e9e593d5bb798c3e67fc3893dfe7055c9e7d8d74 -F src/where.c 346944c20b9c1ac191f16cfc1100dfbe58fa98d4 -F src/whereInt.h 63c8345d01d12ded6250b72e7c16855f0a358041 +F src/where.c 15170b152697e8819de5909031d72ad52a28a7ef +F src/whereInt.h a0e8fa5364122c6cb27c6fa340b257a05c592bfb F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1138,7 +1138,10 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 4e7e805e1139b2dc05d85e86e5c8254e5d361bf2 -R 24bac22755e297cd5d201f4faa2841ac +P 5196000930600d0cd931b87e864507791b9dab08 +R d1d503c972952b4e29bfacf424d1532a +T *branch * skip-scan +T *sym-skip-scan * +T -sym-trunk * U drh -Z b52683f99c1f13f39fe1493c1b7d78a3 +Z c64da1e6d225931e6198f9f21b69efb9 diff --git a/manifest.uuid b/manifest.uuid index 5a991f73f9..309dd4f350 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5196000930600d0cd931b87e864507791b9dab08 \ No newline at end of file +27dd5993d1ae5625eb94bf406421eb390d001be9 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 617d373cbb..021cab187d 100644 --- a/src/where.c +++ b/src/where.c @@ -2422,7 +2422,7 @@ static int codeEqualityTerm( /* ** Generate code that will evaluate all == and IN constraints for an -** index. +** index scan. ** ** For example, consider table t1(a,b,c,d,e,f) with index i1(a,b,c). ** Suppose the WHERE clause is this: a==5 AND b IN (1,2,3) AND c>5 AND c<10 @@ -2437,9 +2437,15 @@ static int codeEqualityTerm( ** The only thing it does is allocate the pLevel->iMem memory cell and ** compute the affinity string. ** -** This routine always allocates at least one memory cell and returns -** the index of that memory cell. The code that -** calls this routine will use that memory cell to store the termination +** The nExtraReg parameter is 0 or 1. It is 0 if all WHERE clause constraints +** are == or IN and are covered by the nEq. nExtraReg is 1 if there is +** an inequality constraint (such as the "c>=5 AND c<10" in the example) that +** occurs after the nEq quality constraints. +** +** This routine allocates a range of nEq+nExtraReg memory cells and returns +** the index of the first memory cell in that range. The code that +** calls this routine will use that memory range to store keys for +** start and termination conditions of the loop. ** key value of the loop. If one or more IN operators appear, then ** this routine allocates an additional nEq memory cells for internal ** use. @@ -2466,7 +2472,8 @@ static int codeAllEqualityTerms( int nExtraReg, /* Number of extra registers to allocate */ char **pzAff /* OUT: Set to point to affinity string */ ){ - int nEq; /* The number of == or IN constraints to code */ + u16 nEq; /* The number of == or IN constraints to code */ + u16 nSkip; /* Number of left-most columns to skip */ Vdbe *v = pParse->pVdbe; /* The vm under construction */ Index *pIdx; /* The index being used for this loop */ WhereTerm *pTerm; /* A single constraint term */ @@ -2480,6 +2487,7 @@ static int codeAllEqualityTerms( pLoop = pLevel->pWLoop; assert( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 ); nEq = pLoop->u.btree.nEq; + nSkip = pLoop->u.btree.nSkip; pIdx = pLoop->u.btree.pIndex; assert( pIdx!=0 ); @@ -2494,14 +2502,28 @@ static int codeAllEqualityTerms( pParse->db->mallocFailed = 1; } + if( nSkip ){ + int iIdxCur = pLevel->iIdxCur; + sqlite3VdbeAddOp2(v, (bRev?OP_Last:OP_Rewind), iIdxCur, pLevel->addrNxt); + pLevel->addrSkip = sqlite3VdbeCurrentAddr(v); + pLevel->opSkip = bRev ? OP_SeekLt : OP_SeekGt; + pLevel->p3Skip = regBase; + pLevel->p4Skip = nSkip; + for(j=0; jaiColumn[j]>=0 ); + VdbeComment((v, "%s", pIdx->pTable->aCol[pIdx->aiColumn[j]].zName)); + } + } + /* Evaluate the equality constraints */ assert( zAff==0 || (int)strlen(zAff)>=nEq ); - for(j=0; jaLTerm[j]; assert( pTerm!=0 ); - /* The following true for indices with redundant columns. + /* The following testcase is true for indices with redundant columns. ** Ex: CREATE INDEX i1 ON t1(a,b,a); SELECT * FROM t1 WHERE a=0 AND b=0; */ testcase( (pTerm->wtFlags & TERM_CODED)!=0 ); testcase( pTerm->wtFlags & TERM_VIRTUAL ); @@ -2575,7 +2597,8 @@ static void explainAppendTerm( */ static char *explainIndexRange(sqlite3 *db, WhereLoop *pLoop, Table *pTab){ Index *pIndex = pLoop->u.btree.pIndex; - int nEq = pLoop->u.btree.nEq; + u16 nEq = pLoop->u.btree.nEq; + u16 nSkip = pLoop->u.btree.nSkip; int i, j; Column *aCol = pTab->aCol; i16 *aiColumn = pIndex->aiColumn; @@ -2589,7 +2612,14 @@ static char *explainIndexRange(sqlite3 *db, WhereLoop *pLoop, Table *pTab){ sqlite3StrAccumAppend(&txt, " (", 2); for(i=0; inKeyCol ) ? "rowid" : aCol[aiColumn[i]].zName; - explainAppendTerm(&txt, i, z, "="); + if( i>=nSkip ){ + explainAppendTerm(&txt, i, z, "="); + }else{ + if( i ) sqlite3StrAccumAppend(&txt, " AND ", 5); + sqlite3StrAccumAppend(&txt, "ANY(", 4); + sqlite3StrAccumAppend(&txt, z, -1); + sqlite3StrAccumAppend(&txt, ")", 1); + } } j = i; @@ -2952,8 +2982,9 @@ static Bitmask codeOneLoopStart( OP_IdxGE, /* 1: (end_constraints && !bRev) */ OP_IdxLT /* 2: (end_constraints && bRev) */ }; - int nEq = pLoop->u.btree.nEq; /* Number of == or IN terms */ - int isMinQuery = 0; /* If this is an optimized SELECT min(x).. */ + u16 nEq = pLoop->u.btree.nEq; /* Number of == or IN terms */ + u16 nSkip = pLoop->u.btree.nSkip; /* Number of left index terms to skip */ + int isMinQuery = 0; /* If this is an optimized SELECT min(x).. */ int regBase; /* Base register holding constraint values */ int r1; /* Temp register */ WhereTerm *pRangeStart = 0; /* Inequality constraint at range start */ @@ -2971,6 +3002,7 @@ static Bitmask codeOneLoopStart( pIdx = pLoop->u.btree.pIndex; iIdxCur = pLevel->iIdxCur; + assert( nEq>=nSkip ); /* If this loop satisfies a sort order (pOrderBy) request that ** was passed to this function to implement a "SELECT min(x) ..." @@ -2984,8 +3016,7 @@ static Bitmask codeOneLoopStart( && (pWInfo->bOBSat!=0) && (pIdx->nKeyCol>nEq) ){ - /* assert( pOrderBy->nExpr==1 ); */ - /* assert( pOrderBy->a[0].pExpr->iColumn==pIdx->aiColumn[nEq] ); */ + assert( nSkip==0 ); isMinQuery = 1; nExtraReg = 1; } @@ -3536,6 +3567,7 @@ static void whereLoopPrint(WhereLoop *p, WhereClause *pWC){ sqlite3ExplainBegin(v); for(i=0; inLTerm; i++){ WhereTerm *pTerm = p->aLTerm[i]; + if( pTerm==0 ) continue; sqlite3ExplainPrintf(v, " (%d) #%-2d ", i+1, (int)(pTerm-pWC->a)); sqlite3ExplainPush(v); whereExplainTerm(v, pTerm); @@ -3848,7 +3880,8 @@ static int whereLoopAddBtreeIndex( WhereScan scan; /* Iterator for WHERE terms */ Bitmask saved_prereq; /* Original value of pNew->prereq */ u16 saved_nLTerm; /* Original value of pNew->nLTerm */ - int saved_nEq; /* Original value of pNew->u.btree.nEq */ + u16 saved_nEq; /* Original value of pNew->u.btree.nEq */ + u16 saved_nSkip; /* Original value of pNew->u.btree.nSkip */ u32 saved_wsFlags; /* Original value of pNew->wsFlags */ LogEst saved_nOut; /* Original value of pNew->nOut */ int iCol; /* Index of the column in the table */ @@ -3883,12 +3916,24 @@ static int whereLoopAddBtreeIndex( pTerm = whereScanInit(&scan, pBuilder->pWC, pSrc->iCursor, iCol, opMask, pProbe); saved_nEq = pNew->u.btree.nEq; + saved_nSkip = pNew->u.btree.nSkip; saved_nLTerm = pNew->nLTerm; saved_wsFlags = pNew->wsFlags; saved_prereq = pNew->prereq; saved_nOut = pNew->nOut; pNew->rSetup = 0; rLogSize = estLog(sqlite3LogEst(pProbe->aiRowEst[0])); + if( pTerm==0 + && saved_nEq==saved_nSkip + && saved_nEq+1nKeyCol + && pProbe->aiRowEst[saved_nEq+1]>50 + ){ + pNew->u.btree.nEq++; + pNew->u.btree.nSkip++; + pNew->aLTerm[pNew->nLTerm++] = 0; + pNew->wsFlags |= WHERE_SKIP_SCAN; + whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nInMul); + } for(; rc==SQLITE_OK && pTerm!=0; pTerm = whereScanNext(&scan)){ int nIn = 0; #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 @@ -4006,6 +4051,7 @@ static int whereLoopAddBtreeIndex( } pNew->prereq = saved_prereq; pNew->u.btree.nEq = saved_nEq; + pNew->u.btree.nSkip = saved_nSkip; pNew->wsFlags = saved_wsFlags; pNew->nOut = saved_nOut; pNew->nLTerm = saved_nLTerm; @@ -4152,6 +4198,7 @@ static int whereLoopAddBtree( if( pTerm->prereqRight & pNew->maskSelf ) continue; if( termCanDriveIndex(pTerm, pSrc, 0) ){ pNew->u.btree.nEq = 1; + pNew->u.btree.nSkip = 0; pNew->u.btree.pIndex = 0; pNew->nLTerm = 1; pNew->aLTerm[0] = pTerm; @@ -4181,6 +4228,7 @@ static int whereLoopAddBtree( continue; /* Partial index inappropriate for this query */ } pNew->u.btree.nEq = 0; + pNew->u.btree.nSkip = 0; pNew->nLTerm = 0; pNew->iSortIdx = 0; pNew->rSetup = 0; @@ -4715,6 +4763,7 @@ static int wherePathSatisfiesOrderBy( /* Skip over == and IS NULL terms */ if( ju.btree.nEq + && pLoop->u.btree.nSkip==0 && ((i = pLoop->aLTerm[j]->eOperator) & (WO_EQ|WO_ISNULL))!=0 ){ if( i & WO_ISNULL ){ @@ -5140,6 +5189,7 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){ pWC = &pWInfo->sWC; pLoop = pBuilder->pNew; pLoop->wsFlags = 0; + pLoop->u.btree.nSkip = 0; pTerm = findTerm(pWC, iCur, -1, 0, WO_EQ, 0); if( pTerm ){ pLoop->wsFlags = WHERE_COLUMN_EQ|WHERE_IPK|WHERE_ONEROW; @@ -5713,6 +5763,7 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ VdbeNoopComment((v, "End WHERE-core")); sqlite3ExprCacheClear(pParse); for(i=pWInfo->nLevel-1; i>=0; i--){ + int addr; pLevel = &pWInfo->a[i]; pLoop = pLevel->pWLoop; sqlite3VdbeResolveLabel(v, pLevel->addrCont); @@ -5732,8 +5783,14 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ sqlite3DbFree(db, pLevel->u.in.aInLoop); } sqlite3VdbeResolveLabel(v, pLevel->addrBrk); + if( pLevel->addrSkip ){ + addr = sqlite3VdbeAddOp4Int(v, pLevel->opSkip, pLevel->iIdxCur, 0, + pLevel->p3Skip, pLevel->p4Skip); + sqlite3VdbeAddOp2(v, OP_Goto, 0, pLevel->addrSkip); + sqlite3VdbeJumpHere(v, pLevel->addrSkip-1); + sqlite3VdbeJumpHere(v, addr); + } if( pLevel->iLeftJoin ){ - int addr; addr = sqlite3VdbeAddOp1(v, OP_IfPos, pLevel->iLeftJoin); assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 || (pLoop->wsFlags & WHERE_INDEXED)!=0 ); diff --git a/src/whereInt.h b/src/whereInt.h index b73353f732..62c93471e5 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -65,12 +65,16 @@ struct WhereLevel { int iIdxCur; /* The VDBE cursor used to access pIdx */ int addrBrk; /* Jump here to break out of the loop */ int addrNxt; /* Jump here to start the next IN combination */ + int addrSkip; /* Jump here for next iteration of skip-scan */ int addrCont; /* Jump here to continue with the next loop cycle */ int addrFirst; /* First instruction of interior of the loop */ int addrBody; /* Beginning of the body of this loop */ u8 iFrom; /* Which entry in the FROM clause */ u8 op, p5; /* Opcode and P5 of the opcode that ends the loop */ + u8 opSkip; /* Opcode to terminate the skip-scan */ int p1, p2; /* Operands of the opcode used to ends the loop */ + int p3Skip; /* P3 operand for the skip-scan terminator */ + u16 p4Skip; /* P4 operand for the skip-scan terminator */ union { /* Information that depends on pWLoop->wsFlags */ struct { int nIn; /* Number of entries in aInLoop[] */ @@ -455,3 +459,4 @@ struct WhereInfo { #define WHERE_ONEROW 0x00001000 /* Selects no more than one row */ #define WHERE_MULTI_OR 0x00002000 /* OR using multiple indices */ #define WHERE_AUTO_INDEX 0x00004000 /* Uses an ephemeral index */ +#define WHERE_SKIP_SCAN 0x00008000 /* Uses the skip-scan algorithm */ From c2b23e7a9832fdeb2658171f39a5cbdaf5ccdb67 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 13 Nov 2013 15:32:15 +0000 Subject: [PATCH 08/14] Add test cases for skip-scan. Enhance "do_test" so that if the expected result is of the form "/*..*/" or "~/*..*/" it treats the expected result as a glob pattern rather than as a regular expression. Fix a bug in ANALYZE result loading associated with WITHOUT ROWID tables. FossilOrigin-Name: d3e6e9b2a74074c05429d3c341c23525504351ab --- manifest | 18 +++--- manifest.uuid | 2 +- src/analyze.c | 8 ++- test/skipscan1.test | 148 ++++++++++++++++++++++++++++++++++++++++++++ test/tester.tcl | 21 +++++-- 5 files changed, 179 insertions(+), 18 deletions(-) create mode 100644 test/skipscan1.test diff --git a/manifest b/manifest index 5d4c908bea..7158caf1b6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sability\sto\suse\san\sindex\seven\sif\sthe\sleft-most\scolumns\sof\sthe\sindex\nare\sunconstrainted,\sprovided\sthat\sthe\sleft-most\scolumns\shave\sfew\sdistinct\nvalues. -D 2013-11-13T12:27:25.442 +C Add\stest\scases\sfor\sskip-scan.\s\sEnhance\s"do_test"\sso\sthat\sif\sthe\sexpected\sresult\nis\sof\sthe\sform\s"/*..*/"\sor\s"~/*..*/"\sit\streats\sthe\sexpected\sresult\sas\sa\sglob\npattern\srather\sthan\sas\sa\sregular\sexpression.\s\sFix\sa\sbug\sin\sANALYZE\sresult\nloading\sassociated\swith\sWITHOUT\sROWID\stables. +D 2013-11-13T15:32:15.331 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 8a07bebafbfda0eb67728f4bd15a36201662d1a1 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -160,7 +160,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 6be1ad09113570e1fc8dcaff84c9b0b337db5ffc F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c 2af0330bb1b601af7a7789bf7229675fd772a083 -F src/analyze.c 27f0c132aa0679189837e0addf8762e7fd6831b6 +F src/analyze.c 581d5c18ce89c6f45d4dca65914d0de5b4dad41f F src/attach.c 0a17c9364895316ca4f52d06a97a72c0af1ae8b3 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c 2f1987981139bd2f6d8c728d64bf09fb387443c3 @@ -805,6 +805,7 @@ F test/shell5.test 46c8c18d62732415c4fe084816c13d559831705e F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3 F test/shrink.test 8c70f62b6e8eb4d54533de6d65bd06b1b9a17868 F test/sidedelete.test f0ad71abe6233e3b153100f3b8d679b19a488329 +F test/skipscan1.test 63af32c300be545417410ea2ce44e78c5b2e34b1 F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24 F test/sort.test 0e4456e729e5a92a625907c63dcdedfbe72c5dc5 @@ -835,7 +836,7 @@ F test/tclsqlite.test 37a61c2da7e3bfe3b8c1a2867199f6b860df5d43 F test/tempdb.test 19d0f66e2e3eeffd68661a11c83ba5e6ace9128c F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 F test/temptrigger.test 0a48d94222d50e6e50d72ac103606c4f8e7cbb81 -F test/tester.tcl 3f675f00d22de3595be25dc2d2cba7ff623ef058 +F test/tester.tcl bce6b929932498383ce92431da6a96432c690bf7 F test/thread001.test 9f22fd3525a307ff42a326b6bc7b0465be1745a5 F test/thread002.test e630504f8a06c00bf8bbe68528774dd96aeb2e58 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7 @@ -1138,10 +1139,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 5196000930600d0cd931b87e864507791b9dab08 -R d1d503c972952b4e29bfacf424d1532a -T *branch * skip-scan -T *sym-skip-scan * -T -sym-trunk * +P 27dd5993d1ae5625eb94bf406421eb390d001be9 +R 56958c69a5559324c31384471b37b6dc U drh -Z c64da1e6d225931e6198f9f21b69efb9 +Z 2ff45f7229fb8bf180b3b10c0362f0e0 diff --git a/manifest.uuid b/manifest.uuid index 309dd4f350..1dae906437 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -27dd5993d1ae5625eb94bf406421eb390d001be9 \ No newline at end of file +d3e6e9b2a74074c05429d3c341c23525504351ab \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index f1094f79f9..3d5c4f6bec 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1428,10 +1428,12 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ if( pTable==0 ){ return 0; } - if( argv[1] ){ - pIndex = sqlite3FindIndex(pInfo->db, argv[1], pInfo->zDatabase); - }else{ + if( argv[1]==0 ){ pIndex = 0; + }else if( sqlite3_stricmp(argv[0],argv[1])==0 ){ + pIndex = sqlite3PrimaryKeyIndex(pTable); + }else{ + pIndex = sqlite3FindIndex(pInfo->db, argv[1], pInfo->zDatabase); } z = argv[2]; diff --git a/test/skipscan1.test b/test/skipscan1.test new file mode 100644 index 0000000000..0931592f94 --- /dev/null +++ b/test/skipscan1.test @@ -0,0 +1,148 @@ +# 2013-11-13 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file implements tests of the "skip-scan" query strategy. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +do_execsql_test skipscan1-1.1 { + CREATE TABLE t1(a TEXT, b INT, c INT, d INT); + CREATE INDEX t1abc ON t1(a,b,c); + INSERT INTO t1 VALUES('abc',123,4,5); + INSERT INTO t1 VALUES('abc',234,5,6); + INSERT INTO t1 VALUES('abc',234,6,7); + INSERT INTO t1 VALUES('abc',345,7,8); + INSERT INTO t1 VALUES('def',567,8,9); + INSERT INTO t1 VALUES('def',345,9,10); + INSERT INTO t1 VALUES('bcd',100,6,11); + + /* Fake the sqlite_stat1 table so that the query planner believes + ** the table contains thousands of rows and that the first few + ** columns are not selective. */ + ANALYZE; + DELETE FROM sqlite_stat1; + INSERT INTO sqlite_stat1 VALUES('t1','t1abc','10000 5000 2000 10'); + ANALYZE sqlite_master; +} {} + +# Simple queries that leave the first one or two columns of the +# index unconstrainted. +# +do_execsql_test skipscan1-1.2 { + SELECT a,b,c,d,'|' FROM t1 WHERE b=345 ORDER BY a; +} {abc 345 7 8 | def 345 9 10 |} +do_execsql_test skipscan1-1.2eqp { + EXPLAIN QUERY PLAN + SELECT a,b,c,d,'|' FROM t1 WHERE b=345 ORDER BY a; +} {/* USING INDEX t1abc (ANY(a) AND b=?)*/} +do_execsql_test skipscan1-1.2sort { + EXPLAIN QUERY PLAN + SELECT a,b,c,d,'|' FROM t1 WHERE b=345 ORDER BY a; +} {~/*ORDER BY*/} + +do_execsql_test skipscan1-1.3 { + SELECT a,b,c,d,'|' FROM t1 WHERE b=345 ORDER BY a DESC; +} {def 345 9 10 | abc 345 7 8 |} +do_execsql_test skipscan1-1.3eqp { + EXPLAIN QUERY PLAN + SELECT a,b,c,d,'|' FROM t1 WHERE b=345 ORDER BY a; +} {/* USING INDEX t1abc (ANY(a) AND b=?)*/} +do_execsql_test skipscan1-1.3sort { + EXPLAIN QUERY PLAN + SELECT a,b,c,d,'|' FROM t1 WHERE b=345 ORDER BY a; +} {~/*ORDER BY*/} + +do_execsql_test skipscan1-1.4 { + SELECT a,b,c,d,'|' FROM t1 WHERE c=6 ORDER BY a, b, c; +} {abc 234 6 7 | bcd 100 6 11 |} +do_execsql_test skipscan1-1.4eqp { + EXPLAIN QUERY PLAN + SELECT a,b,c,d,'|' FROM t1 WHERE c=6 ORDER BY a, b, c; +} {/* USING INDEX t1abc (ANY(a) AND ANY(b) AND c=?)*/} +do_execsql_test skipscan1-1.4sort { + EXPLAIN QUERY PLAN + SELECT a,b,c,d,'|' FROM t1 WHERE c=6 ORDER BY a, b, c; +} {~/*ORDER BY*/} + +# Joins +# +do_execsql_test skipscan1-1.5 { + CREATE TABLE t1j(x TEXT, y INTEGER); + INSERT INTO t1j VALUES('one',1),('six',6),('ninty-nine',99); + SELECT x, a, b, c, d, '|' FROM t1j, t1 WHERE c=y ORDER BY +a; +} {six abc 234 6 7 | six bcd 100 6 11 |} +do_execsql_test skipscan1-1.5eqp { + EXPLAIN QUERY PLAN + SELECT x, a, b, c, d, '|' FROM t1j, t1 WHERE c=y ORDER BY +a; +} {/* INDEX t1abc (ANY(a) AND ANY(b) AND c=?)*/} + +do_execsql_test skipscan1-1.6 { + SELECT x, a, b, c, d, '|' FROM t1j LEFT JOIN t1 ON c=y ORDER BY +y, +a; +} {one {} {} {} {} | six abc 234 6 7 | six bcd 100 6 11 | ninty-nine {} {} {} {} |} +do_execsql_test skipscan1-1.6eqp { + EXPLAIN QUERY PLAN + SELECT x, a, b, c, d, '|' FROM t1j LEFT JOIN t1 ON c=y ORDER BY +y, +a; +} {/* INDEX t1abc (ANY(a) AND ANY(b) AND c=?)*/} + +do_execsql_test skipscan1-2.1 { + CREATE TABLE t2(a TEXT, b INT, c INT, d INT, + PRIMARY KEY(a,b,c)); + INSERT INTO t2 SELECT * FROM t1; + + /* Fake the sqlite_stat1 table so that the query planner believes + ** the table contains thousands of rows and that the first few + ** columns are not selective. */ + ANALYZE; + UPDATE sqlite_stat1 SET stat='10000 5000 2000 10' WHERE idx NOT NULL; + ANALYZE sqlite_master; +} {} + +do_execsql_test skipscan1-2.2 { + SELECT a,b,c,d,'|' FROM t2 WHERE b=345 ORDER BY a; +} {abc 345 7 8 | def 345 9 10 |} +do_execsql_test skipscan1-2.2eqp { + EXPLAIN QUERY PLAN + SELECT a,b,c,d,'|' FROM t2 WHERE b=345 ORDER BY a; +} {/* USING INDEX sqlite_autoindex_t2_1 (ANY(a) AND b=?)*/} +do_execsql_test skipscan1-2.2sort { + EXPLAIN QUERY PLAN + SELECT a,b,c,d,'|' FROM t2 WHERE b=345 ORDER BY a; +} {~/*ORDER BY*/} + + +do_execsql_test skipscan1-3.1 { + CREATE TABLE t3(a TEXT, b INT, c INT, d INT, + PRIMARY KEY(a,b,c)) WITHOUT ROWID; + INSERT INTO t3 SELECT * FROM t1; + + /* Fake the sqlite_stat1 table so that the query planner believes + ** the table contains thousands of rows and that the first few + ** columns are not selective. */ + ANALYZE; + UPDATE sqlite_stat1 SET stat='10000 5000 2000 10' WHERE idx NOT NULL; + ANALYZE sqlite_master; +} {} + +do_execsql_test skipscan1-3.2 { + SELECT a,b,c,d,'|' FROM t3 WHERE b=345 ORDER BY a; +} {abc 345 7 8 | def 345 9 10 |} +do_execsql_test skipscan1-3.2eqp { + EXPLAIN QUERY PLAN + SELECT a,b,c,d,'|' FROM t3 WHERE b=345 ORDER BY a; +} {/* INDEX sqlite_autoindex_t3_1 (ANY(a) AND b=?)*/} +do_execsql_test skipscan1-3.2sort { + EXPLAIN QUERY PLAN + SELECT a,b,c,d,'|' FROM t3 WHERE b=345 ORDER BY a; +} {~/*ORDER BY*/} + +finish_test diff --git a/test/tester.tcl b/test/tester.tcl index 2d047be5f4..c1272210d0 100644 --- a/test/tester.tcl +++ b/test/tester.tcl @@ -616,11 +616,24 @@ proc do_test {name cmd expected} { # regular expression PATTERN matches the result. "~/PATTERN/" means # the regular expression must not match. if {[string index $expected 0]=="~"} { - set re [string map {# {[-0-9.]+}} [string range $expected 2 end-1]] - set ok [expr {![regexp $re $result]}] + set re [string range $expected 2 end-1] + if {[string index $re 0]=="*"} { + # If the regular expression begins with * then treat it as a glob instead + set ok [string match $re $result] + } else { + set re [string map {# {[-0-9.]+}} $re] + set ok [regexp $re $result] + } + set ok [expr {!$ok}] } else { - set re [string map {# {[-0-9.]+}} [string range $expected 1 end-1]] - set ok [regexp $re $result] + set re [string range $expected 1 end-1] + if {[string index $re 0]=="*"} { + # If the regular expression begins with * then treat it as a glob instead + set ok [string match $re $result] + } else { + set re [string map {# {[-0-9.]+}} $re] + set ok [regexp $re $result] + } } } elseif {[regexp {^~?\*.*\*$} $expected]} { # "expected" is of the form "*GLOB*" then the result if correct if From 2e5ef4ed77dbace183fff5add3bb6e017f4a8417 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 13 Nov 2013 16:58:54 +0000 Subject: [PATCH 09/14] Improve the way that skip-scan loops are constructued. Add test cases. Improved the scoring of skip-scan loops. FossilOrigin-Name: 5e75ab93881b85801cb4ebf70f2063ff7c51ac19 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/where.c | 28 +++++++++++++++------------- src/whereInt.h | 5 +---- test/skipscan1.test | 35 +++++++++++++++++++++++++++++++---- 5 files changed, 56 insertions(+), 30 deletions(-) diff --git a/manifest b/manifest index 7158caf1b6..7626e0a77f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stest\scases\sfor\sskip-scan.\s\sEnhance\s"do_test"\sso\sthat\sif\sthe\sexpected\sresult\nis\sof\sthe\sform\s"/*..*/"\sor\s"~/*..*/"\sit\streats\sthe\sexpected\sresult\sas\sa\sglob\npattern\srather\sthan\sas\sa\sregular\sexpression.\s\sFix\sa\sbug\sin\sANALYZE\sresult\nloading\sassociated\swith\sWITHOUT\sROWID\stables. -D 2013-11-13T15:32:15.331 +C Improve\sthe\sway\sthat\sskip-scan\sloops\sare\sconstructued.\s\sAdd\stest\scases.\nImproved\sthe\sscoring\sof\sskip-scan\sloops. +D 2013-11-13T16:58:54.547 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 8a07bebafbfda0eb67728f4bd15a36201662d1a1 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -293,8 +293,8 @@ F src/vtab.c 5a423b042eb1402ef77697d03d6a67378d97bc8d F src/wal.c 7dc3966ef98b74422267e7e6e46e07ff6c6eb1b4 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c e9e593d5bb798c3e67fc3893dfe7055c9e7d8d74 -F src/where.c 15170b152697e8819de5909031d72ad52a28a7ef -F src/whereInt.h a0e8fa5364122c6cb27c6fa340b257a05c592bfb +F src/where.c f50428d2e93ca3e5c541cc1c2084833b48db947c +F src/whereInt.h 96a75c61f1d2b9d4a8e4bb17d89deb0cf7cba358 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -805,7 +805,7 @@ F test/shell5.test 46c8c18d62732415c4fe084816c13d559831705e F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3 F test/shrink.test 8c70f62b6e8eb4d54533de6d65bd06b1b9a17868 F test/sidedelete.test f0ad71abe6233e3b153100f3b8d679b19a488329 -F test/skipscan1.test 63af32c300be545417410ea2ce44e78c5b2e34b1 +F test/skipscan1.test bc65ecb228eba396c404aa038f49798340f0677f F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24 F test/sort.test 0e4456e729e5a92a625907c63dcdedfbe72c5dc5 @@ -1139,7 +1139,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 27dd5993d1ae5625eb94bf406421eb390d001be9 -R 56958c69a5559324c31384471b37b6dc +P d3e6e9b2a74074c05429d3c341c23525504351ab +R 2b49a424145c7566c72f185654c1a259 U drh -Z 2ff45f7229fb8bf180b3b10c0362f0e0 +Z 03bb67f899f79fcc8cd9bd00c60852e3 diff --git a/manifest.uuid b/manifest.uuid index 1dae906437..514af577f1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d3e6e9b2a74074c05429d3c341c23525504351ab \ No newline at end of file +5e75ab93881b85801cb4ebf70f2063ff7c51ac19 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 021cab187d..c2824862ea 100644 --- a/src/where.c +++ b/src/where.c @@ -2504,11 +2504,11 @@ static int codeAllEqualityTerms( if( nSkip ){ int iIdxCur = pLevel->iIdxCur; - sqlite3VdbeAddOp2(v, (bRev?OP_Last:OP_Rewind), iIdxCur, pLevel->addrNxt); - pLevel->addrSkip = sqlite3VdbeCurrentAddr(v); - pLevel->opSkip = bRev ? OP_SeekLt : OP_SeekGt; - pLevel->p3Skip = regBase; - pLevel->p4Skip = nSkip; + sqlite3VdbeAddOp1(v, (bRev?OP_Last:OP_Rewind), iIdxCur); + j = sqlite3VdbeAddOp0(v, OP_Goto); + pLevel->addrSkip = sqlite3VdbeAddOp4Int(v, (bRev?OP_SeekLt:OP_SeekGt), + iIdxCur, 0, regBase, nSkip); + sqlite3VdbeJumpHere(v, j); for(j=0; jaiColumn[j]>=0 ); @@ -3928,11 +3928,13 @@ static int whereLoopAddBtreeIndex( && saved_nEq+1nKeyCol && pProbe->aiRowEst[saved_nEq+1]>50 ){ + LogEst nIter; pNew->u.btree.nEq++; pNew->u.btree.nSkip++; pNew->aLTerm[pNew->nLTerm++] = 0; - pNew->wsFlags |= WHERE_SKIP_SCAN; - whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nInMul); + pNew->wsFlags |= WHERE_SKIPSCAN; + nIter = sqlite3LogEst(pProbe->aiRowEst[0]/pProbe->aiRowEst[saved_nEq+1]); + whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nIter); } for(; rc==SQLITE_OK && pTerm!=0; pTerm = whereScanNext(&scan)){ int nIn = 0; @@ -3969,8 +3971,10 @@ static int whereLoopAddBtreeIndex( pNew->u.btree.nEq++; pNew->nOut = nRowEst + nInMul + nIn; }else if( pTerm->eOperator & (WO_EQ) ){ - assert( (pNew->wsFlags & (WHERE_COLUMN_NULL|WHERE_COLUMN_IN))!=0 - || nInMul==0 ); + assert( + (pNew->wsFlags & (WHERE_COLUMN_NULL|WHERE_COLUMN_IN|WHERE_SKIPSCAN))!=0 + || nInMul==0 + ); pNew->wsFlags |= WHERE_COLUMN_EQ; if( iCol<0 || (pProbe->onError!=OE_None && nInMul==0 @@ -5784,11 +5788,9 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ } sqlite3VdbeResolveLabel(v, pLevel->addrBrk); if( pLevel->addrSkip ){ - addr = sqlite3VdbeAddOp4Int(v, pLevel->opSkip, pLevel->iIdxCur, 0, - pLevel->p3Skip, pLevel->p4Skip); sqlite3VdbeAddOp2(v, OP_Goto, 0, pLevel->addrSkip); - sqlite3VdbeJumpHere(v, pLevel->addrSkip-1); - sqlite3VdbeJumpHere(v, addr); + sqlite3VdbeJumpHere(v, pLevel->addrSkip); + sqlite3VdbeJumpHere(v, pLevel->addrSkip-2); } if( pLevel->iLeftJoin ){ addr = sqlite3VdbeAddOp1(v, OP_IfPos, pLevel->iLeftJoin); diff --git a/src/whereInt.h b/src/whereInt.h index 62c93471e5..56646c55e6 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -71,10 +71,7 @@ struct WhereLevel { int addrBody; /* Beginning of the body of this loop */ u8 iFrom; /* Which entry in the FROM clause */ u8 op, p5; /* Opcode and P5 of the opcode that ends the loop */ - u8 opSkip; /* Opcode to terminate the skip-scan */ int p1, p2; /* Operands of the opcode used to ends the loop */ - int p3Skip; /* P3 operand for the skip-scan terminator */ - u16 p4Skip; /* P4 operand for the skip-scan terminator */ union { /* Information that depends on pWLoop->wsFlags */ struct { int nIn; /* Number of entries in aInLoop[] */ @@ -459,4 +456,4 @@ struct WhereInfo { #define WHERE_ONEROW 0x00001000 /* Selects no more than one row */ #define WHERE_MULTI_OR 0x00002000 /* OR using multiple indices */ #define WHERE_AUTO_INDEX 0x00004000 /* Uses an ephemeral index */ -#define WHERE_SKIP_SCAN 0x00008000 /* Uses the skip-scan algorithm */ +#define WHERE_SKIPSCAN 0x00008000 /* Uses the skip-scan algorithm */ diff --git a/test/skipscan1.test b/test/skipscan1.test index 0931592f94..eba7383eaa 100644 --- a/test/skipscan1.test +++ b/test/skipscan1.test @@ -74,22 +74,49 @@ do_execsql_test skipscan1-1.4sort { SELECT a,b,c,d,'|' FROM t1 WHERE c=6 ORDER BY a, b, c; } {~/*ORDER BY*/} +do_execsql_test skipscan1-1.5 { + SELECT a,b,c,d,'|' FROM t1 WHERE c IN (6,7) ORDER BY a, b, c; +} {abc 234 6 7 | abc 345 7 8 | bcd 100 6 11 |} +do_execsql_test skipscan1-1.5eqp { + EXPLAIN QUERY PLAN + SELECT a,b,c,d,'|' FROM t1 WHERE c IN (6,7) ORDER BY a, b, c; +} {/* USING INDEX t1abc (ANY(a) AND ANY(b) AND c=?)*/} +do_execsql_test skipscan1-1.5sort { + EXPLAIN QUERY PLAN + SELECT a,b,c,d,'|' FROM t1 WHERE c IN (6,7) ORDER BY a, b, c; +} {~/*ORDER BY*/} + +do_execsql_test skipscan1-1.6 { + SELECT a,b,c,d,'|' FROM t1 WHERE c BETWEEN 6 AND 7 ORDER BY a, b, c; +} {abc 234 6 7 | abc 345 7 8 | bcd 100 6 11 |} +do_execsql_test skipscan1-1.6eqp { + EXPLAIN QUERY PLAN + SELECT a,b,c,d,'|' FROM t1 WHERE c BETWEEN 6 AND 7 ORDER BY a, b, c; +} {/* USING INDEX t1abc (ANY(a) AND ANY(b) AND c>? AND c Date: Wed, 13 Nov 2013 17:24:38 +0000 Subject: [PATCH 10/14] Add VDBE comments to the beginning and end of skip-scan loops. FossilOrigin-Name: 0c85d93b52311dee7980d977be6ed0dc70b060c1 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 2 ++ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 7626e0a77f..390483589c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improve\sthe\sway\sthat\sskip-scan\sloops\sare\sconstructued.\s\sAdd\stest\scases.\nImproved\sthe\sscoring\sof\sskip-scan\sloops. -D 2013-11-13T16:58:54.547 +C Add\sVDBE\scomments\sto\sthe\sbeginning\sand\send\sof\sskip-scan\sloops. +D 2013-11-13T17:24:39.000 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 8a07bebafbfda0eb67728f4bd15a36201662d1a1 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -293,7 +293,7 @@ F src/vtab.c 5a423b042eb1402ef77697d03d6a67378d97bc8d F src/wal.c 7dc3966ef98b74422267e7e6e46e07ff6c6eb1b4 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c e9e593d5bb798c3e67fc3893dfe7055c9e7d8d74 -F src/where.c f50428d2e93ca3e5c541cc1c2084833b48db947c +F src/where.c 1d19a1d49a608441fa697ae32627399073be8dcf F src/whereInt.h 96a75c61f1d2b9d4a8e4bb17d89deb0cf7cba358 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1139,7 +1139,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P d3e6e9b2a74074c05429d3c341c23525504351ab -R 2b49a424145c7566c72f185654c1a259 +P 5e75ab93881b85801cb4ebf70f2063ff7c51ac19 +R 324d2d53e7ac48858b72f1751f7e7610 U drh -Z 03bb67f899f79fcc8cd9bd00c60852e3 +Z 7cfb6326eedc7a21a0bd15f40c4f97e2 diff --git a/manifest.uuid b/manifest.uuid index 514af577f1..ac2ea40722 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5e75ab93881b85801cb4ebf70f2063ff7c51ac19 \ No newline at end of file +0c85d93b52311dee7980d977be6ed0dc70b060c1 \ No newline at end of file diff --git a/src/where.c b/src/where.c index c2824862ea..2fd7c600c8 100644 --- a/src/where.c +++ b/src/where.c @@ -2505,6 +2505,7 @@ static int codeAllEqualityTerms( if( nSkip ){ int iIdxCur = pLevel->iIdxCur; sqlite3VdbeAddOp1(v, (bRev?OP_Last:OP_Rewind), iIdxCur); + VdbeComment((v, "begin skip-scan on %s", pIdx->zName)); j = sqlite3VdbeAddOp0(v, OP_Goto); pLevel->addrSkip = sqlite3VdbeAddOp4Int(v, (bRev?OP_SeekLt:OP_SeekGt), iIdxCur, 0, regBase, nSkip); @@ -5789,6 +5790,7 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ sqlite3VdbeResolveLabel(v, pLevel->addrBrk); if( pLevel->addrSkip ){ sqlite3VdbeAddOp2(v, OP_Goto, 0, pLevel->addrSkip); + VdbeComment((v, "next skip-scan on %s", pLoop->u.btree.pIndex->zName)); sqlite3VdbeJumpHere(v, pLevel->addrSkip); sqlite3VdbeJumpHere(v, pLevel->addrSkip-2); } From 84e55a80dbe18cfce16db1cd7858f110d39b8dd7 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 13 Nov 2013 17:58:23 +0000 Subject: [PATCH 11/14] Add the "PRAGMA vdbe_eqp" command, only available with SQLITE_DEBUG. Simplify some of the other debugging logic. FossilOrigin-Name: 8ce33f4c818e1c785a1c176f6f631b8184e1166b --- manifest | 28 +++++++++---------- manifest.uuid | 2 +- src/build.c | 4 --- src/pragma.c | 6 +++- src/sqliteInt.h | 1 + src/vdbe.c | 66 +++++++++++++++++++++++++------------------- src/vdbe.h | 1 - src/vdbeInt.h | 3 -- src/vdbeaux.c | 27 +++++++----------- src/where.c | 5 +++- tool/mkpragmatab.tcl | 6 ++++ 11 files changed, 79 insertions(+), 70 deletions(-) diff --git a/manifest b/manifest index 1bdb81fbcb..3448386df9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\san\sunnecessary\sOP_IfNull\swhile\sdoing\san\sindexed\ssearch. -D 2013-11-13T08:55:02.907 +C Add\sthe\s"PRAGMA\svdbe_eqp"\scommand,\sonly\savailable\swith\sSQLITE_DEBUG.\s\sSimplify\nsome\sof\sthe\sother\sdebugging\slogic. +D 2013-11-13T17:58:23.573 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 8a07bebafbfda0eb67728f4bd15a36201662d1a1 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -169,7 +169,7 @@ F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c 260dedc13119e6fb7930380bd3d294b98362bf5a F src/btree.h bfe0e8c5759b4ec77b0d18390064a6ef3cdffaaf F src/btreeInt.h f038e818bfadf75afbd09819ed93c26a333d39e0 -F src/build.c 4d740243144d9e5058a46df7d05d1243f449b4d7 +F src/build.c 2baeed38bdaa9f1199f101c63db41fdcc4b39ba5 F src/callback.c f99a8957ba2adf369645fac0db09ad8adcf1caa2 F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c ea4b7f3623a0fcb1146e7f245d7410033e86859c @@ -213,7 +213,7 @@ F src/parse.y 073a8294e1826f1b1656e84806b77e4199f4bb57 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c a467393909a4ed7ca9de066d85ba5c5b04a5be63 -F src/pragma.c 3b7b766382ac679d3c1a7ba368aa008f9a756d59 +F src/pragma.c c8d70c47ec8d8ba93575d92e34d30ddff8e9b517 F src/prepare.c fa6988589f39af8504a61731614cd4f6ae71554f F src/printf.c da9119eb31a187a4b99f60aa4a225141c0ebb74b F src/random.c 0b2dbc37fdfbfa6bd455b091dfcef5bdb32dba68 @@ -224,7 +224,7 @@ F src/shell.c 03d8d9b4052430343ff30d646334621f980f1202 F src/sqlite.h.in 4dedcab5b32358bf7a596badffe7363be1f1a82d F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 3da1940a2ba05a663e9016d57f1ea1f79ffcb03e +F src/sqliteInt.h 2ec1d71220307fa5ee85a918a6240cf0b4635b0e F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -280,11 +280,11 @@ F src/update.c 3de7e657b98ac67338d775c114a4068faf732402 F src/utf.c 6fc6c88d50448c469c5c196acf21617a24f90269 F src/util.c 2fa6c821d28bbdbeec1b2a7b091a281c9ef8f918 F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c fda0d2ba5a3a5ab588f0467782e7fd64930ddb22 -F src/vdbe.h 8d5a7351024d80374fc0acdbbe3cfe65c51ba8b6 -F src/vdbeInt.h f2fa3ceccceeb757773921fb08af7c6e9f3caa1c +F src/vdbe.c 3d73013b4ef54061e768443fb35a36b3dd289663 +F src/vdbe.h c06f0813f853566457ce9cfb1a4a4bc39a5da644 +F src/vdbeInt.h 62eb680327011f3a4b0336642b0ca9d6ecc6eb91 F src/vdbeapi.c 93a22a9ba2abe292d5c2cf304d7eb2e894dde0ed -F src/vdbeaux.c 75c5fee307004739d814a840699df5e853e02b79 +F src/vdbeaux.c dd0f6ab9dc159911facfc0a7a2164af44779bdda F src/vdbeblob.c ff60adf2953e0ffc1d59541b3101c9886b03a3de F src/vdbemem.c cc529bbf4f13e4e181bdb446bf6e6962ab030b4b F src/vdbesort.c 9d83601f9d6243fe70dd0169a2820c5ddfd48147 @@ -293,7 +293,7 @@ F src/vtab.c 5a423b042eb1402ef77697d03d6a67378d97bc8d F src/wal.c 7dc3966ef98b74422267e7e6e46e07ff6c6eb1b4 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c e9e593d5bb798c3e67fc3893dfe7055c9e7d8d74 -F src/where.c 346944c20b9c1ac191f16cfc1100dfbe58fa98d4 +F src/where.c 537f4d7e869f41b7aef851ad0b050983b4d6508d F src/whereInt.h 63c8345d01d12ded6250b72e7c16855f0a358041 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1105,7 +1105,7 @@ F tool/logest.c 7ad625cac3d54012b27d468b7af6612f78b9ba75 F tool/mkautoconfamal.sh f8d8dbf7d62f409ebed5134998bf5b51d7266383 F tool/mkkeywordhash.c 189d76644e373c7d0864c628deb8ce7b4f403591 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e -F tool/mkpragmatab.tcl 17d40faae6c4b865633bfc5763821402a1cbefc3 +F tool/mkpragmatab.tcl 78a77b2c554d534c6f2dc903130186ed15715460 F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 F tool/mksqlite3c-noext.tcl 8bce31074e4cbe631bb7676526a048335f4c9f02 F tool/mksqlite3c.tcl e2ba20d3f690990079d17f3e5a7417dfb7ada543 @@ -1138,7 +1138,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 4e7e805e1139b2dc05d85e86e5c8254e5d361bf2 -R 24bac22755e297cd5d201f4faa2841ac +P 5196000930600d0cd931b87e864507791b9dab08 +R f5035ff28ea611640fd4871588416d7f U drh -Z b52683f99c1f13f39fe1493c1b7d78a3 +Z f0bce2e671cb5c826c0cf7f8a42a33dc diff --git a/manifest.uuid b/manifest.uuid index 5a991f73f9..834dfca91b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5196000930600d0cd931b87e864507791b9dab08 \ No newline at end of file +8ce33f4c818e1c785a1c176f6f631b8184e1166b \ No newline at end of file diff --git a/src/build.c b/src/build.c index c3999f2b9a..d1615a128d 100644 --- a/src/build.c +++ b/src/build.c @@ -193,10 +193,6 @@ void sqlite3FinishCoding(Parse *pParse){ /* Get the VDBE program ready for execution */ if( v && ALWAYS(pParse->nErr==0) && !db->mallocFailed ){ -#ifdef SQLITE_DEBUG - FILE *trace = (db->flags & SQLITE_VdbeTrace)!=0 ? stdout : 0; - sqlite3VdbeTrace(v, trace); -#endif assert( pParse->iCacheLevel==0 ); /* Disables and re-enables match */ /* A minimum of one cursor is required if autoincrement is used * See ticket [a696379c1f08866] */ diff --git a/src/pragma.c b/src/pragma.c index 9211a2cb07..76a452c466 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -434,6 +434,10 @@ static const struct sPragmaNames { /* ePragTyp: */ PragTyp_FLAG, /* ePragFlag: */ 0, /* iArg: */ SQLITE_SqlTrace|SQLITE_VdbeListing|SQLITE_VdbeTrace }, + { /* zName: */ "vdbe_eqp", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlag: */ 0, + /* iArg: */ SQLITE_VdbeEQP }, { /* zName: */ "vdbe_listing", /* ePragTyp: */ PragTyp_FLAG, /* ePragFlag: */ 0, @@ -461,7 +465,7 @@ static const struct sPragmaNames { /* iArg: */ SQLITE_WriteSchema|SQLITE_RecoveryMode }, #endif }; -/* Number of pragmas: 56 on by default, 68 total. */ +/* Number of pragmas: 56 on by default, 69 total. */ /* End of the automatically generated pragma table. ***************************************************************************/ diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 0ae55edf3d..8838f7aa92 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1043,6 +1043,7 @@ struct sqlite3 { #define SQLITE_EnableTrigger 0x00800000 /* True to enable triggers */ #define SQLITE_DeferFKs 0x01000000 /* Defer all FK constraints */ #define SQLITE_QueryOnly 0x02000000 /* Disable database changes */ +#define SQLITE_VdbeEQP 0x04000000 /* Debug EXPLAIN QUERY PLAN */ /* diff --git a/src/vdbe.c b/src/vdbe.c index d3fe0a4500..c73bf59639 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -417,37 +417,36 @@ void sqlite3VdbeMemPrettyPrint(Mem *pMem, char *zBuf){ /* ** Print the value of a register for tracing purposes: */ -static void memTracePrint(FILE *out, Mem *p){ +static void memTracePrint(Mem *p){ if( p->flags & MEM_Invalid ){ - fprintf(out, " undefined"); + printf(" undefined"); }else if( p->flags & MEM_Null ){ - fprintf(out, " NULL"); + printf(" NULL"); }else if( (p->flags & (MEM_Int|MEM_Str))==(MEM_Int|MEM_Str) ){ - fprintf(out, " si:%lld", p->u.i); + printf(" si:%lld", p->u.i); }else if( p->flags & MEM_Int ){ - fprintf(out, " i:%lld", p->u.i); + printf(" i:%lld", p->u.i); #ifndef SQLITE_OMIT_FLOATING_POINT }else if( p->flags & MEM_Real ){ - fprintf(out, " r:%g", p->r); + printf(" r:%g", p->r); #endif }else if( p->flags & MEM_RowSet ){ - fprintf(out, " (rowset)"); + printf(" (rowset)"); }else{ char zBuf[200]; sqlite3VdbeMemPrettyPrint(p, zBuf); - fprintf(out, " "); - fprintf(out, "%s", zBuf); + printf(" %s", zBuf); } } -static void registerTrace(FILE *out, int iReg, Mem *p){ - fprintf(out, "REG[%d] = ", iReg); - memTracePrint(out, p); - fprintf(out, "\n"); +static void registerTrace(int iReg, Mem *p){ + printf("REG[%d] = ", iReg); + memTracePrint(p); + printf("\n"); } #endif #ifdef SQLITE_DEBUG -# define REGISTER_TRACE(R,M) if(p->trace)registerTrace(p->trace,R,M) +# define REGISTER_TRACE(R,M) if(db->flags&SQLITE_VdbeTrace)registerTrace(R,M) #else # define REGISTER_TRACE(R,M) #endif @@ -586,13 +585,28 @@ int sqlite3VdbeExec( #endif #ifdef SQLITE_DEBUG sqlite3BeginBenignMalloc(); - if( p->pc==0 && (p->db->flags & SQLITE_VdbeListing)!=0 ){ + if( p->pc==0 + && (p->db->flags & (SQLITE_VdbeListing|SQLITE_VdbeEQP|SQLITE_VdbeTrace))!=0 + ){ int i; - printf("VDBE Program Listing:\n"); + int once = 1; sqlite3VdbePrintSql(p); - for(i=0; inOp; i++){ - sqlite3VdbePrintOp(stdout, i, &aOp[i]); + if( p->db->flags & SQLITE_VdbeListing ){ + printf("VDBE Program Listing:\n"); + for(i=0; inOp; i++){ + sqlite3VdbePrintOp(stdout, i, &aOp[i]); + } } + if( p->db->flags & SQLITE_VdbeEQP ){ + for(i=0; inOp; i++){ + if( aOp[i].opcode==OP_Explain ){ + if( once ) printf("VDBE Query Plan:\n"); + printf("%s\n", aOp[i].p4.z); + once = 0; + } + } + } + if( p->db->flags & SQLITE_VdbeTrace ) printf("VDBE Trace:\n"); } sqlite3EndBenignMalloc(); #endif @@ -609,12 +623,8 @@ int sqlite3VdbeExec( /* Only allow tracing if SQLITE_DEBUG is defined. */ #ifdef SQLITE_DEBUG - if( p->trace ){ - if( pc==0 ){ - printf("VDBE Execution Trace:\n"); - sqlite3VdbePrintSql(p); - } - sqlite3VdbePrintOp(p->trace, pc, pOp); + if( db->flags & SQLITE_VdbeTrace ){ + sqlite3VdbePrintOp(stdout, pc, pOp); } #endif @@ -6255,13 +6265,13 @@ default: { /* This is really OP_Noop and OP_Explain */ assert( pc>=-1 && pcnOp ); #ifdef SQLITE_DEBUG - if( p->trace ){ - if( rc!=0 ) fprintf(p->trace,"rc=%d\n",rc); + if( db->flags & SQLITE_VdbeTrace ){ + if( rc!=0 ) printf("rc=%d\n",rc); if( pOp->opflags & (OPFLG_OUT2_PRERELEASE|OPFLG_OUT2) ){ - registerTrace(p->trace, pOp->p2, &aMem[pOp->p2]); + registerTrace(pOp->p2, &aMem[pOp->p2]); } if( pOp->opflags & OPFLG_OUT3 ){ - registerTrace(p->trace, pOp->p3, &aMem[pOp->p3]); + registerTrace(pOp->p3, &aMem[pOp->p3]); } } #endif /* SQLITE_DEBUG */ diff --git a/src/vdbe.h b/src/vdbe.h index 91d6a0f794..62d9aa2711 100644 --- a/src/vdbe.h +++ b/src/vdbe.h @@ -189,7 +189,6 @@ void sqlite3VdbeResolveLabel(Vdbe*, int); int sqlite3VdbeCurrentAddr(Vdbe*); #ifdef SQLITE_DEBUG int sqlite3VdbeAssertMayAbort(Vdbe *, int); - void sqlite3VdbeTrace(Vdbe*,FILE*); #endif void sqlite3VdbeResetStepResult(Vdbe*); void sqlite3VdbeRewind(Vdbe*); diff --git a/src/vdbeInt.h b/src/vdbeInt.h index b7f5ab1a31..4f63189f5e 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -355,9 +355,6 @@ struct Vdbe { i64 nStmtDefImmCons; /* Number of def. imm constraints when stmt started */ char *zSql; /* Text of the SQL statement that generated this */ void *pFree; /* Free this when deleting the vdbe */ -#ifdef SQLITE_DEBUG - FILE *trace; /* Write an execution trace here, if not NULL */ -#endif #ifdef SQLITE_ENABLE_TREE_EXPLAIN Explain *pExplain; /* The explainer */ char *zExplain; /* Explanation of data structures */ diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 7ce9c2a6c1..166cf7508f 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -78,15 +78,6 @@ void sqlite3VdbeSwap(Vdbe *pA, Vdbe *pB){ pB->isPrepareV2 = pA->isPrepareV2; } -#ifdef SQLITE_DEBUG -/* -** Turn tracing on or off -*/ -void sqlite3VdbeTrace(Vdbe *p, FILE *trace){ - p->trace = trace; -} -#endif - /* ** Resize the Vdbe.aOp array so that it is at least one op larger than ** it was. @@ -1415,15 +1406,17 @@ int sqlite3VdbeList( ** Print the SQL that was used to generate a VDBE program. */ void sqlite3VdbePrintSql(Vdbe *p){ - int nOp = p->nOp; - VdbeOp *pOp; - if( nOp<1 ) return; - pOp = &p->aOp[0]; - if( pOp->opcode==OP_Trace && pOp->p4.z!=0 ){ - const char *z = pOp->p4.z; - while( sqlite3Isspace(*z) ) z++; - printf("SQL: [%s]\n", z); + const char *z = 0; + if( p->zSql ){ + z = p->zSql; + }else if( p->nOp>=1 ){ + const VdbeOp *pOp = &p->aOp[0]; + if( pOp->opcode==OP_Trace && pOp->p4.z!=0 ){ + z = pOp->p4.z; + while( sqlite3Isspace(*z) ) z++; + } } + if( z ) printf("SQL: [%s]\n", z); } #endif diff --git a/src/where.c b/src/where.c index 617d373cbb..f5030b6a68 100644 --- a/src/where.c +++ b/src/where.c @@ -2619,7 +2619,10 @@ static void explainOneScan( int iFrom, /* Value for "from" column of output */ u16 wctrlFlags /* Flags passed to sqlite3WhereBegin() */ ){ - if( pParse->explain==2 ){ +#ifndef SQLITE_DEBUG + if( pParse->explain==2 ) +#endif + { struct SrcList_item *pItem = &pTabList->a[pLevel->iFrom]; Vdbe *v = pParse->pVdbe; /* VM being constructed */ sqlite3 *db = pParse->db; /* Database handle */ diff --git a/tool/mkpragmatab.tcl b/tool/mkpragmatab.tcl index f5ca6d8867..28a1e468b8 100644 --- a/tool/mkpragmatab.tcl +++ b/tool/mkpragmatab.tcl @@ -97,6 +97,12 @@ set pragma_def { IF: !defined(SQLITE_OMIT_FLAG_PRAGMAS) IF: defined(SQLITE_DEBUG) + NAME: vdbe_eqp + TYPE: FLAG + ARG: SQLITE_VdbeEQP + IF: !defined(SQLITE_OMIT_FLAG_PRAGMAS) + IF: defined(SQLITE_DEBUG) + NAME: ignore_check_constraints TYPE: FLAG ARG: SQLITE_IgnoreChecks From a98bf365fe4e73aa8d4249f39a9b0433f2605794 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 13 Nov 2013 18:35:01 +0000 Subject: [PATCH 12/14] In the shell tool, if an "EXPLAIN" command is executed in ".explain on" mode, attempt to automatically indent the bodies of loops in the output VDBE program. FossilOrigin-Name: e7d34ec6814ed4606a6d5d7f68c218ae4d25e666 --- manifest | 14 +++---- manifest.uuid | 2 +- src/shell.c | 102 +++++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 108 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 3448386df9..78bd21a267 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s"PRAGMA\svdbe_eqp"\scommand,\sonly\savailable\swith\sSQLITE_DEBUG.\s\sSimplify\nsome\sof\sthe\sother\sdebugging\slogic. -D 2013-11-13T17:58:23.573 +C In\sthe\sshell\stool,\sif\san\s"EXPLAIN"\scommand\sis\sexecuted\sin\s".explain\son"\smode,\sattempt\sto\sautomatically\sindent\sthe\sbodies\sof\sloops\sin\sthe\soutput\sVDBE\sprogram. +D 2013-11-13T18:35:01.894 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 8a07bebafbfda0eb67728f4bd15a36201662d1a1 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -220,7 +220,7 @@ F src/random.c 0b2dbc37fdfbfa6bd455b091dfcef5bdb32dba68 F src/resolve.c fc4673cc49b116e51e7f12de074c0acf8f2388f9 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 7317406831ecced390edba972818f3c5f82238c0 -F src/shell.c 03d8d9b4052430343ff30d646334621f980f1202 +F src/shell.c 3b23017da75118da0a7e3518c6ce78a7b747fb05 F src/sqlite.h.in 4dedcab5b32358bf7a596badffe7363be1f1a82d F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc @@ -1138,7 +1138,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 5196000930600d0cd931b87e864507791b9dab08 -R f5035ff28ea611640fd4871588416d7f -U drh -Z f0bce2e671cb5c826c0cf7f8a42a33dc +P 8ce33f4c818e1c785a1c176f6f631b8184e1166b +R 5d7c6b4eb6f0cadad72d4cdb7ed00f79 +U dan +Z 87b1c90d92386c54cb5a5e86314bc13e diff --git a/manifest.uuid b/manifest.uuid index 834dfca91b..e811c26432 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8ce33f4c818e1c785a1c176f6f631b8184e1166b \ No newline at end of file +e7d34ec6814ed4606a6d5d7f68c218ae4d25e666 \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index c3aee04633..6761750816 100644 --- a/src/shell.c +++ b/src/shell.c @@ -464,6 +464,8 @@ struct callback_data { const char *zVfs; /* Name of VFS to use */ sqlite3_stmt *pStmt; /* Current statement if any. */ FILE *pLog; /* Write log output here */ + int *aiIndent; /* Array of indents used in MODE_Explain */ + int nIndent; /* Size of array aiIndent[] */ }; /* @@ -765,10 +767,15 @@ static int shell_callback(void *pArg, int nArg, char **azArg, char **azCol, int }else{ w = 10; } - if( p->mode==MODE_Explain && azArg[i] && - strlen30(azArg[i])>w ){ + if( p->mode==MODE_Explain && azArg[i] && strlen30(azArg[i])>w ){ w = strlen30(azArg[i]); } + if( i==1 && p->aiIndent && p->pStmt ){ + int iOp = sqlite3_column_int(p->pStmt, 0); + if( iOpnIndent ){ + fprintf(p->out, "%*.s", p->aiIndent[iOp], ""); + } + } if( w<0 ){ fprintf(p->out,"%*.*s%s",-w,-w, azArg[i] ? azArg[i] : p->nullvalue, i==nArg-1 ? "\n": " "); @@ -1141,6 +1148,89 @@ static int display_stats( return 0; } +/* +** Parameter azArray points to a zero-terminated array of strings. zStr +** points to a single nul-terminated string. Return non-zero if zStr +** is equal, according to strcmp(), to any of the strings in the array. +** Otherwise, return zero. +*/ +static int str_in_array(const char *zStr, const char **azArray){ + int i; + for(i=0; azArray[i]; i++){ + if( 0==strcmp(zStr, azArray[i]) ) return 1; + } + return 0; +} + +/* +** If compiled statement pSql appears to be an EXPLAIN statement, allocate +** and populate the callback_data.aiIndent[] array with the number of +** spaces each opcode should be indented before it is output. +** +** The indenting rules are: +** +** * For each "Next", "Prev", "VNext" or "VPrev" instruction, indent +** all opcodes that occur between the p2 jump destination and the opcode +** itself by 2 spaces. +** +** * For each "Goto", if the jump destination is a "Yield" instruction +** that occurs earlier in the program than the Goto itself, indent +** all opcodes between the "Yield" and "Goto" by 2 spaces. +*/ +static void explain_data_prepare(struct callback_data *p, sqlite3_stmt *pSql){ + const char *zSql; /* The text of the SQL statement */ + const char *z; /* Used to check if this is an EXPLAIN */ + int *abYield = 0; /* True if op is an OP_Yield */ + int nAlloc = 0; /* Allocated size of p->aiIndent[], abYield */ + int iOp; + + const char *azNext[] = { "Next", "Prev", "VPrev", "VNext", 0 }; + const char *azYield[] = { "Yield", 0 }; + const char *azGoto[] = { "Goto", 0 }; + + /* Try to figure out if this is really an EXPLAIN statement. If this + ** cannot be verified, return early. */ + zSql = sqlite3_sql(pSql); + if( zSql==0 ) return; + for(z=zSql; *z==' ' || *z=='\t' || *z=='\n' || *z=='\f' || *z=='\r'; z++); + if( sqlite3_strnicmp(z, "explain", 7) ) return; + + for(iOp=0; SQLITE_ROW==sqlite3_step(pSql); iOp++){ + int i; + const char *zOp = (const char*)sqlite3_column_text(pSql, 1); + int p2 = sqlite3_column_int(pSql, 3); + + /* Grow the p->aiIndent array as required */ + if( iOp>=nAlloc ){ + nAlloc += 100; + p->aiIndent = (int*)sqlite3_realloc(p->aiIndent, nAlloc*sizeof(int)); + abYield = (int*)sqlite3_realloc(abYield, nAlloc*sizeof(int)); + } + abYield[iOp] = str_in_array(zOp, azYield); + p->aiIndent[iOp] = 0; + p->nIndent = iOp+1; + + if( str_in_array(zOp, azNext) ){ + for(i=p2; iaiIndent[i] += 2; + } + if( str_in_array(zOp, azGoto) && p2nIndent && abYield[p2] ){ + for(i=p2+1; iaiIndent[i] += 2; + } + } + + sqlite3_free(abYield); + sqlite3_reset(pSql); +} + +/* +** Free the array allocated by explain_data_prepare(). +*/ +static void explain_data_delete(struct callback_data *p){ + sqlite3_free(p->aiIndent); + p->aiIndent = 0; + p->nIndent = 0; +} + /* ** Execute a statement or set of statements. Print ** any result rows/columns depending on the current mode @@ -1202,6 +1292,12 @@ static int shell_exec( } } + /* If the shell is currently in ".explain" mode, gather the extra + ** data required to add indents to the output.*/ + if( pArg->mode==MODE_Explain ){ + explain_data_prepare(pArg, pStmt); + } + /* perform the first step. this will tell us if we ** have a result set or not and how wide it is. */ @@ -1259,6 +1355,8 @@ static int shell_exec( } } + explain_data_delete(pArg); + /* print usage stats if stats on */ if( pArg && pArg->statsOn ){ display_stats(db, pArg, 0); From e6400b9901cdcc13578b4e3901f8eb1c36216f4e Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 13 Nov 2013 23:48:46 +0000 Subject: [PATCH 13/14] Make sure the progress callback is invoked prior to an SQLITE_ROW return if it is overdue to be called. FossilOrigin-Name: 21f59b04f74738d08ebad693646bbaea24dc45ef --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbe.c | 12 ++++++++++++ 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 581e75d42a..03e87575ae 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\sskip-scan\senhancement\sinto\strunk. -D 2013-11-13T20:46:11.977 +C Make\ssure\sthe\sprogress\scallback\sis\sinvoked\sprior\sto\san\sSQLITE_ROW\sreturn\sif\nit\sis\soverdue\sto\sbe\scalled. +D 2013-11-13T23:48:46.535 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 8a07bebafbfda0eb67728f4bd15a36201662d1a1 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -280,7 +280,7 @@ F src/update.c 3de7e657b98ac67338d775c114a4068faf732402 F src/utf.c 6fc6c88d50448c469c5c196acf21617a24f90269 F src/util.c 2fa6c821d28bbdbeec1b2a7b091a281c9ef8f918 F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c 3d73013b4ef54061e768443fb35a36b3dd289663 +F src/vdbe.c 0057612c2678b08812fc4a9d5019351cbeb9ee97 F src/vdbe.h c06f0813f853566457ce9cfb1a4a4bc39a5da644 F src/vdbeInt.h 62eb680327011f3a4b0336642b0ca9d6ecc6eb91 F src/vdbeapi.c 93a22a9ba2abe292d5c2cf304d7eb2e894dde0ed @@ -1139,7 +1139,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P e7d34ec6814ed4606a6d5d7f68c218ae4d25e666 f668616a29686f3ce532731c534b168e536adbb5 -R 0f42dd23f60246e0b55a84d51974156a +P b0bb975c0986fe01f1184c1d4888fe397174ad0f +R 51ebd76e04274e826e1e2a1145d2a6ab U drh -Z e8f824146c4a7790bef9ac829b4cd088 +Z c57556eaf40c3f587321800fb5d2a1a5 diff --git a/manifest.uuid b/manifest.uuid index a213e48cb9..4387c022e5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b0bb975c0986fe01f1184c1d4888fe397174ad0f \ No newline at end of file +21f59b04f74738d08ebad693646bbaea24dc45ef \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index c73bf59639..6c5735b77a 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -1186,6 +1186,18 @@ case OP_ResultRow: { assert( pOp->p1>0 ); assert( pOp->p1+pOp->p2<=(p->nMem-p->nCursor)+1 ); +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK + /* Run the progress counter just before returning. + */ + if( db->xProgress!=0 + && nVmStep>=nProgressLimit + && db->xProgress(db->pProgressArg)!=0 + ){ + rc = SQLITE_INTERRUPT; + goto vdbe_error_halt; + } +#endif + /* If this statement has violated immediate foreign key constraints, do ** not return the number of rows modified. And do not RELEASE the statement ** transaction. It needs to be rolled back. */ From 400fcbad717b673501925e504e3cca9cbe31f533 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 14 Nov 2013 00:09:48 +0000 Subject: [PATCH 14/14] Simplification to the progress callback check. On branch removed. FossilOrigin-Name: 24ef16548eebcdb9d8b40308f6a16dabf8f8d474 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbe.c | 9 +++------ 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 03e87575ae..efb3c6127d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\ssure\sthe\sprogress\scallback\sis\sinvoked\sprior\sto\san\sSQLITE_ROW\sreturn\sif\nit\sis\soverdue\sto\sbe\scalled. -D 2013-11-13T23:48:46.535 +C Simplification\sto\sthe\sprogress\scallback\scheck.\sOn\sbranch\sremoved. +D 2013-11-14T00:09:48.126 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 8a07bebafbfda0eb67728f4bd15a36201662d1a1 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -280,7 +280,7 @@ F src/update.c 3de7e657b98ac67338d775c114a4068faf732402 F src/utf.c 6fc6c88d50448c469c5c196acf21617a24f90269 F src/util.c 2fa6c821d28bbdbeec1b2a7b091a281c9ef8f918 F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c 0057612c2678b08812fc4a9d5019351cbeb9ee97 +F src/vdbe.c f9a4e08114f97e453a5f13a62c31a357301137cc F src/vdbe.h c06f0813f853566457ce9cfb1a4a4bc39a5da644 F src/vdbeInt.h 62eb680327011f3a4b0336642b0ca9d6ecc6eb91 F src/vdbeapi.c 93a22a9ba2abe292d5c2cf304d7eb2e894dde0ed @@ -1139,7 +1139,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P b0bb975c0986fe01f1184c1d4888fe397174ad0f -R 51ebd76e04274e826e1e2a1145d2a6ab +P 21f59b04f74738d08ebad693646bbaea24dc45ef +R 5afe90b6493eb21de1d0af561cfa69c0 U drh -Z c57556eaf40c3f587321800fb5d2a1a5 +Z d0339b99c91e151f95c1864799fb36e8 diff --git a/manifest.uuid b/manifest.uuid index 4387c022e5..56997654db 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -21f59b04f74738d08ebad693646bbaea24dc45ef \ No newline at end of file +24ef16548eebcdb9d8b40308f6a16dabf8f8d474 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 6c5735b77a..f70870ca63 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -755,15 +755,12 @@ check_for_interrupt: ** a return code SQLITE_ABORT. */ if( db->xProgress!=0 && nVmStep>=nProgressLimit ){ - int prc; - prc = db->xProgress(db->pProgressArg); - if( prc!=0 ){ + assert( db->nProgressOps!=0 ); + nProgressLimit = nVmStep + db->nProgressOps - (nVmStep%db->nProgressOps); + if( db->xProgress(db->pProgressArg) ){ rc = SQLITE_INTERRUPT; goto vdbe_error_halt; } - if( db->xProgress!=0 ){ - nProgressLimit = nVmStep + db->nProgressOps - (nVmStep%db->nProgressOps); - } } #endif