diff --git a/manifest b/manifest index 168990dfe6..32086b8fc1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Experimental\simplementation\sof\sNULLS\sFIRST/LAST.\sThis\sbranch\sstill\shas\sproblems\s-\sthe\smost\ssignificant\sof\swhich\sis\sthat\sORDER\sBY\sclauses\swith\sa\snon-default\sNULLS\sFIRST/LAST\squalifier\scan\snever\suse\san\sindex. -D 2019-08-12T16:36:38.041 +C Add\ssupport\sfor\susing\sindexes\sfor\ssome\sORDER\sBY\sclauses\sthat\suse\snon-default\sNULL\shandling.\sStill\ssome\sproblems\son\sthis\sbranch. +D 2019-08-16T21:07:19.697 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -609,9 +609,9 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c bbd6838bd79c0a32144d482fb0b6a9d2d1a252fb3b16d5005ec30f2f80413b0d F src/wal.h 606292549f5a7be50b6227bd685fa76e3a4affad71bb8ac5ce4cb5c79f6a176a F src/walker.c d5a94907dcac990e31976be9dc769d17f6a806782593d6aec9d760ee01ec22cd -F src/where.c 83fc2acadfbb1c86501fc0847fd068040ecd9a250a2fc0b81bab5698aa4bc72d -F src/whereInt.h 2082fc2bd1eb66cb236a1a3c4b250e33d2bad9e43a0486a2cf9e4e211c58f3eb -F src/wherecode.c cf885ea2d439af9827c5cbab7d4c12be5c079439b7bd12e97151ccfe088c13c0 +F src/where.c c4ec116264555c512edf49ef7244113cad5028bd1ea70f3500982b7c10f9a121 +F src/whereInt.h 807766a6a92893dbba8a887e5b13466d2257ce07a84cf1be0aae2c41aee3bf4f +F src/wherecode.c 58889def15cb57375a5f4f83db6d2b28b372d87cf1b4f23e47928a4847a94ae4 F src/whereexpr.c 5cce1fd11876086890a27c05e0cb75ca97ba64ba6984f72154039f1cfd2e69cc F src/window.c 4d56fc1e3dbd3a4fa0653b3f48a3ad7066d0da91d0273cff8bab13c3412ddaf5 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 @@ -1182,7 +1182,7 @@ F test/notify2.test 2ecabaa1305083856b7c39cf32816b612740c161 F test/notify3.test 10ff25cde502e72a92053a2f215d64bece4ef934 F test/notnull.test a37b663d5bb728d66fc182016613fb8e4a0a4bbf3d75b8876a7527f7d4ed3f18 F test/null.test 0dcce4f04284ec66108c503327ad6d224c0752b3 -F test/nulls1.test eac9f46d9bf7a3883700bbc063e8a64fa2f4677ea64f12e173d3052d635f6b23 +F test/nulls1.test 522f0da68881b6ac616b1361fbd5a9897bd366597809495143968af743e3318c F test/numcast.test 5d126f7f581432e86a90d1e35cac625164aec4a1 F test/numindex1.test 20a5450d4b056e48cd5db30e659f13347a099823 F test/offset1.test f06b83657bcf26f9ce805e67450e189e282143b2 @@ -1837,11 +1837,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 636ca4472c9f41eb3989f28854d4968867837399a2092f389d1b814d98cccbae -R 6d8a33d8edc53ea630705f876c1c3795 -T *branch * nulls-last -T *sym-nulls-last * -T +closed 8174b2ca587e87083950ab21a47b4b4fe4bf309a8e16a82d4dc26d2c471e28cc -T -sym-trunk * +P 07babb0f897fc8c9cb5b30481899c32fdd743f3f3ca508d8d957826252107dd5 +R 39b3bc3a5371ac212f22834b463e006c U dan -Z 02a41ef53191d3d08ca3291791a40934 +Z d691c31e642275ddd26770c7946ed411 diff --git a/manifest.uuid b/manifest.uuid index 3ff2df6757..ef8a25f596 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -07babb0f897fc8c9cb5b30481899c32fdd743f3f3ca508d8d957826252107dd5 \ No newline at end of file +81069d7196857e909c94195d67388f71bc9f832eafd9156d8c5cdddb63513b4a \ No newline at end of file diff --git a/src/where.c b/src/where.c index 202db50bfc..663dcdc4e0 100644 --- a/src/where.c +++ b/src/where.c @@ -3798,7 +3798,7 @@ static i8 wherePathSatisfiesOrderBy( */ if( pIndex ){ iColumn = pIndex->aiColumn[j]; - revIdx = pIndex->aSortOrder[j]; + revIdx = pIndex->aSortOrder[j] & KEYINFO_ORDER_DESC; if( iColumn==pIndex->pTable->iPKey ) iColumn = XN_ROWID; }else{ iColumn = XN_ROWID; @@ -3836,7 +3836,6 @@ static i8 wherePathSatisfiesOrderBy( continue; } } - if( pOrderBy->a[i].sortFlags & KEYINFO_ORDER_BIGNULL ) continue; if( iColumn!=XN_ROWID ){ pColl = sqlite3ExprNNCollSeq(pWInfo->pParse, pOrderBy->a[i].pExpr); if( sqlite3StrICmp(pColl->zName, pIndex->azColl[j])!=0 ) continue; @@ -3850,24 +3849,29 @@ static i8 wherePathSatisfiesOrderBy( if( isMatch && (wctrlFlags & WHERE_GROUPBY)==0 ){ /* Make sure the sort order is compatible in an ORDER BY clause. ** Sort order is irrelevant for a GROUP BY clause. */ - assert( (pOrderBy->a[i].sortFlags & KEYINFO_ORDER_BIGNULL)==0 ); if( revSet ){ - if( (rev ^ revIdx)!=pOrderBy->a[i].sortFlags ) isMatch = 0; + if( (rev ^ revIdx)!=(pOrderBy->a[i].sortFlags&KEYINFO_ORDER_DESC) ){ + isMatch = 0; + } }else{ - rev = revIdx ^ pOrderBy->a[i].sortFlags; + rev = revIdx ^ (pOrderBy->a[i].sortFlags & KEYINFO_ORDER_DESC); if( rev ) *pRevMask |= MASKBIT(iLoop); revSet = 1; } } + if( isMatch && (pOrderBy->a[i].sortFlags & KEYINFO_ORDER_BIGNULL) ){ + if( j==pLoop->u.btree.nEq ){ + pLoop->wsFlags |= WHERE_BIGNULL_SORT; + }else{ + isMatch = 0; + } + } if( isMatch ){ if( iColumn==XN_ROWID ){ testcase( distinctColumns==0 ); distinctColumns = 1; } obSat |= MASKBIT(i); - if( (wctrlFlags & WHERE_ORDERBY_MIN) && j==pLoop->u.btree.nEq ){ - pLoop->wsFlags |= WHERE_MIN_ORDERED; - } }else{ /* No match found */ if( j==0 || jwsFlags & WHERE_CONSTRAINT)!=0 && (pLoop->wsFlags & (WHERE_COLUMN_RANGE|WHERE_SKIPSCAN))==0 + && (pLoop->wsFlags & WHERE_BIGNULL_SORT)==0 && (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)==0 && pWInfo->eDistinct!=WHERE_DISTINCT_ORDERED ){ @@ -5202,6 +5207,17 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ VdbeCoverageIf(v, pLevel->op==OP_Next); VdbeCoverageIf(v, pLevel->op==OP_Prev); VdbeCoverageIf(v, pLevel->op==OP_VNext); + if( pLevel->regBignull ){ + sqlite3VdbeResolveLabel(v, pLevel->addrBignull); + addr = sqlite3VdbeAddOp1(v, OP_If, pLevel->regBignull); + sqlite3VdbeAddOp2(v, OP_Integer, 1, pLevel->regBignull); + sqlite3VdbeAddOp2(v, OP_Goto, 0, pLevel->p2-1); + sqlite3VdbeChangeP5(v, pLevel->p5); + VdbeCoverage(v); + VdbeCoverageIf(v, pLevel->op==OP_Next); + VdbeCoverageIf(v, pLevel->op==OP_Prev); + sqlite3VdbeJumpHere(v, addr); + } #ifndef SQLITE_DISABLE_SKIPAHEAD_DISTINCT if( addrSeek ) sqlite3VdbeJumpHere(v, addrSeek); #endif diff --git a/src/whereInt.h b/src/whereInt.h index 09e45024c1..f402a18f8c 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -71,13 +71,15 @@ struct WhereLevel { int addrCont; /* Jump here to continue with the next loop cycle */ int addrFirst; /* First instruction of interior of the loop */ int addrBody; /* Beginning of the body of this loop */ + int regBignull; /* big-null flag register */ + int addrBignull; /* Jump here for next part of big-null scan */ #ifndef SQLITE_LIKE_DOESNT_MATCH_BLOBS u32 iLikeRepCntr; /* LIKE range processing counter register (times 2) */ int addrLikeRep; /* LIKE range processing address */ #endif u8 iFrom; /* Which entry in the FROM clause */ u8 op, p3, p5; /* Opcode, P3 & P5 of the opcode that ends the loop */ - int p1, p2; /* Operands of the opcode used to ends the loop */ + int p1, p2; /* Operands of the opcode used to end the loop */ union { /* Information that depends on pWLoop->wsFlags */ struct { int nIn; /* Number of entries in aInLoop[] */ @@ -586,6 +588,6 @@ void sqlite3WhereTabFuncArgs(Parse*, struct SrcList_item*, WhereClause*); #define WHERE_UNQ_WANTED 0x00010000 /* WHERE_ONEROW would have been helpful*/ #define WHERE_PARTIALIDX 0x00020000 /* The automatic index is partial */ #define WHERE_IN_EARLYOUT 0x00040000 /* Perhaps quit IN loops early */ -#define WHERE_MIN_ORDERED 0x00080000 /* Column nEq of index is min() expr */ +#define WHERE_BIGNULL_SORT 0x00080000 /* Column nEq of index is BIGNULL */ #endif /* !defined(SQLITE_WHEREINT_H) */ diff --git a/src/wherecode.c b/src/wherecode.c index c781b06c15..d4f26a3af0 100644 --- a/src/wherecode.c +++ b/src/wherecode.c @@ -1549,31 +1549,12 @@ Bitmask sqlite3WhereCodeOneLoopStart( u8 bSeekPastNull = 0; /* True to seek past initial nulls */ u8 bStopAtNull = 0; /* Add condition to terminate at NULLs */ int omitTable; /* True if we use the index only */ - + int regBignull = 0; pIdx = pLoop->u.btree.pIndex; iIdxCur = pLevel->iIdxCur; assert( nEq>=pLoop->nSkip ); - /* If this loop satisfies a sort order (pOrderBy) request that - ** was passed to this function to implement a "SELECT min(x) ..." - ** query, then the caller will only allow the loop to run for - ** a single iteration. This means that the first row returned - ** should not have a NULL value stored in 'x'. If column 'x' is - ** the first one after the nEq equality constraints in the index, - ** this requires some special handling. - */ - assert( (pWInfo->pOrderBy!=0 && pWInfo->pOrderBy->nExpr==1) - || (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)==0 ); - if( pLoop->wsFlags & WHERE_MIN_ORDERED ){ - assert( (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN) ); - assert( pWInfo->nOBSat ); - assert( pIdx->nColumn>nEq ); - assert( pLoop->nSkip==0 ); - bSeekPastNull = 1; - nExtraReg = 1; - } - /* Find any inequality constraint terms for the start and end ** of the range. */ @@ -1614,6 +1595,26 @@ Bitmask sqlite3WhereCodeOneLoopStart( } assert( pRangeEnd==0 || (pRangeEnd->wtFlags & TERM_VNULL)==0 ); + /* If the WHERE_BIGNULL_SORT flag is set, then index column nEq uses + ** a non-default "big-null" sort (either ASC NULLS LAST or DESC NULLS + ** FIRST). In both cases separate ordered scans are made of those + ** index entries for which the column is null and for those for which + ** it is not. For an ASC sort, the non-NULL entries are scanned first. + ** For DESC, NULL entries are scanned first. + */ + addrNxt = pLevel->addrNxt; + if( (pLoop->wsFlags & (WHERE_TOP_LIMIT|WHERE_BTM_LIMIT))==0 + && (pLoop->wsFlags & WHERE_BIGNULL_SORT)!=0 + ){ + assert( bSeekPastNull==0 && nExtraReg==0 && nBtm==0 && nTop==0 ); + assert( pRangeEnd==0 && pRangeStart==0 ); + assert( pLoop->nSkip==0 ); + nExtraReg = 1; + bSeekPastNull = 1; + pLevel->regBignull = regBignull = ++pParse->nMem; + addrNxt = pLevel->addrBignull = sqlite3VdbeMakeLabel(pParse); + } + /* If we are doing a reverse order scan on an ascending index, or ** a forward order scan on a descending index, interchange the ** start and end terms (pRangeStart and pRangeEnd). @@ -1636,7 +1637,6 @@ Bitmask sqlite3WhereCodeOneLoopStart( if( zStartAff && nTop ){ zEndAff = sqlite3DbStrDup(db, &zStartAff[nEq]); } - addrNxt = pLevel->addrNxt; testcase( pRangeStart && (pRangeStart->eOperator & WO_LE)!=0 ); testcase( pRangeStart && (pRangeStart->eOperator & WO_GE)!=0 ); @@ -1674,6 +1674,10 @@ Bitmask sqlite3WhereCodeOneLoopStart( nConstraint++; startEq = 0; start_constraints = 1; + }else if( regBignull ){ + sqlite3VdbeAddOp2(v, OP_Null, 0, regBase+nEq); + start_constraints = 1; + nConstraint++; } codeApplyAffinity(pParse, regBase, nConstraint - bSeekPastNull, zStartAff); if( pLoop->nSkip>0 && nConstraint==pLoop->nSkip ){ @@ -1684,6 +1688,10 @@ Bitmask sqlite3WhereCodeOneLoopStart( if( pLoop->wsFlags & WHERE_IN_EARLYOUT ){ sqlite3VdbeAddOp1(v, OP_SeekHit, iIdxCur); } + if( regBignull ){ + sqlite3VdbeAddOp2(v, OP_Integer, 0, regBignull); + } + op = aStartOp[(start_constraints<<2) + (startEq<<1) + bRev]; assert( op!=0 ); sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint); @@ -1695,23 +1703,16 @@ Bitmask sqlite3WhereCodeOneLoopStart( VdbeCoverageIf(v, op==OP_SeekLE); testcase( op==OP_SeekLE ); VdbeCoverageIf(v, op==OP_SeekLT); testcase( op==OP_SeekLT ); - if( bSeekPastNull && (pLoop->wsFlags & WHERE_TOP_LIMIT)==0 ){ - /* If bSeekPastNull is set only to skip past the NULL values for - ** a query like "SELECT min(a), b FROM t1", then add code so that - ** if there are no rows with (a IS NOT NULL), then do the seek - ** without jumping past NULLs instead. This allows the code in - ** select.c to pick a value for "b" in the above query. */ - assert( startEq==0 && (op==OP_SeekGT || op==OP_SeekLT) ); - assert( (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)!=0 && pWInfo->nOBSat>0 ); - sqlite3VdbeChangeP2(v, -1, sqlite3VdbeCurrentAddr(v)+1); + if( regBignull ){ sqlite3VdbeAddOp2(v, OP_Goto, 0, sqlite3VdbeCurrentAddr(v)+2); - - op = aStartOp[(start_constraints<<2) + (1<<1) + bRev]; - assert( op==OP_SeekGE || op==OP_SeekLE ); - sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint); - VdbeCoverage(v); - VdbeCoverageIf(v, op==OP_SeekGE); testcase( op==OP_SeekGE ); - VdbeCoverageIf(v, op==OP_SeekLE); testcase( op==OP_SeekLE ); + if( bStopAtNull ){ + start_constraints = (nConstraint>1); + op = aStartOp[(start_constraints<<2) + (startEq<<1) + bRev]; + sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint-1); + }else{ + op = aStartOp[(start_constraints<<2) + ((!startEq)<<1) + bRev]; + sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint); + } } } @@ -1744,8 +1745,10 @@ Bitmask sqlite3WhereCodeOneLoopStart( endEq = 1; } }else if( bStopAtNull ){ - sqlite3VdbeAddOp2(v, OP_Null, 0, regBase+nEq); - endEq = 0; + if( regBignull==0 ){ + sqlite3VdbeAddOp2(v, OP_Null, 0, regBase+nEq); + endEq = 0; + } nConstraint++; } sqlite3DbFree(db, zStartAff); @@ -1756,6 +1759,9 @@ Bitmask sqlite3WhereCodeOneLoopStart( /* Check if the index cursor is past the end of the range. */ if( nConstraint ){ + if( regBignull ){ + sqlite3VdbeAddOp2(v, OP_If, regBignull, sqlite3VdbeCurrentAddr(v)+3); + } op = aEndOp[bRev*2 + endEq]; sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint); testcase( op==OP_IdxGT ); VdbeCoverageIf(v, op==OP_IdxGT ); @@ -1763,6 +1769,16 @@ Bitmask sqlite3WhereCodeOneLoopStart( testcase( op==OP_IdxLT ); VdbeCoverageIf(v, op==OP_IdxLT ); testcase( op==OP_IdxLE ); VdbeCoverageIf(v, op==OP_IdxLE ); } + if( regBignull ){ + sqlite3VdbeAddOp2(v, OP_IfNot, regBignull, sqlite3VdbeCurrentAddr(v)+2); + if( bStopAtNull ){ + op = aEndOp[bRev*2 + 0]; + sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint); + }else{ + op = aEndOp[bRev*2 + endEq]; + sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint+1); + } + } if( pLoop->wsFlags & WHERE_IN_EARLYOUT ){ sqlite3VdbeAddOp2(v, OP_SeekHit, iIdxCur, 1); diff --git a/test/nulls1.test b/test/nulls1.test index 7b355e9a85..866ed2b296 100644 --- a/test/nulls1.test +++ b/test/nulls1.test @@ -11,21 +11,19 @@ # This file implements regression tests for SQLite library. # -#################################################### -# DO NOT EDIT! THIS FILE IS AUTOMATICALLY GENERATED! -#################################################### - set testdir [file dirname $argv0] source $testdir/tester.tcl set testprefix nulls1 +if 1 { + do_execsql_test 1.0 { DROP TABLE IF EXISTS t3; CREATE TABLE t3(a INTEGER); INSERT INTO t3 VALUES(NULL), (10), (30), (20), (NULL); } {} -for {set a 0} {$a < 2} {incr a} { +for {set a 0} {$a < 3} {incr a} { foreach {tn limit} { 1 "" 2 "LIMIT 10" @@ -47,7 +45,43 @@ for {set a 0} {$a < 2} {incr a} { " {30 20 10 {} {}} } - catchsql { CREATE INDEX i1 ON t3(a) } + switch $a { + 0 { + execsql { CREATE INDEX i1 ON t3(a) } + } + 1 { + execsql { DROP INDEX i1 ; CREATE INDEX i1 ON t3(a DESC) } + } + } } +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 2.0 { + CREATE TABLE t2(a, b, c); + CREATE INDEX i2 ON t2(a, b); + INSERT INTO t2 VALUES(1, 1, 1); + INSERT INTO t2 VALUES(1, NULL, 2); + INSERT INTO t2 VALUES(1, NULL, 3); + INSERT INTO t2 VALUES(1, 4, 4); +} + +do_execsql_test 2.1 { + SELECT * FROM t2 WHERE a=1 ORDER BY b NULLS LAST +} { + 1 1 1 1 4 4 1 {} 2 1 {} 3 +} + +do_execsql_test 2.2 { + SELECT * FROM t2 WHERE a=1 ORDER BY b DESC NULLS FIRST +} { + 1 {} 3 + 1 {} 2 + 1 4 4 + 1 1 1 +} + + finish_test