When stat4 information is available, try to use it to improve the truth

probability of WHERE clause terms that do not participate in the index.

FossilOrigin-Name: 1babd6ec5d60e2c34aa1c0285ead768a88004218468e97262411973fe3487022

manifest

@@ -1,5 +1,5 @@
-C In\sthe\sOP_Column\sopcode,\sif\sthe\scursor\sis\smarked\sNullRow\s(due\sto\sbeing\sthe\nright\stable\sof\sa\sLEFT\sJOIN\sthat\sdoes\snot\smatch)\sand\sthe\scursor\sis\sthe\stable\ncursor\sfor\san\sOR-optimization\swith\sa\scovering\sindex,\sthen\sdo\snot\ssubstitute\nthe\scovering\sindex\scursor,\ssince\sthe\scovering\sindex\scursor\sdoes\snot\shave\nthe\sNullRow\sflag\sset.\s\sTicket\s[aa4378693018aa99]
-D 2020-02-22T13:01:19.240
+C When\sstat4\sinformation\sis\savailable,\stry\sto\suse\sit\sto\simprove\sthe\struth\nprobability\sof\sWHERE\sclause\sterms\sthat\sdo\snot\sparticipate\sin\sthe\sindex.
+D 2020-02-22T16:58:49.287
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -617,8 +617,8 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c 697424314e40d99f93f548c7bfa526c10e87f4bdf64d5a76a96b999dd7133ebc
 F src/wal.h 606292549f5a7be50b6227bd685fa76e3a4affad71bb8ac5ce4cb5c79f6a176a
 F src/walker.c a137468bf36c92e64d2275caa80c83902e3a0fc59273591b96c6416d3253d05d
-F src/where.c cbad14f1d8e11b9f052e937274315c7c17266a89eda408c86084ee894debb7d5
-F src/whereInt.h 9157228db086f436a574589f8cc5749bd971e94017c552305ad9ec472ed2e098
+F src/where.c 74a2fc5a900eab9a2fdda2017a290f0eeaa9c5597fdb86322ea2ccbc3758c71d
+F src/whereInt.h 94e3aadcf43b4d16279182d147c9e4f8ef6ed5a5bd1ecc021639c29336b0a3eb
 F src/wherecode.c f5df56e395ade2240cabb2d39500c681bd29f8cc0636c3301c4996ad160df94d
 F src/whereexpr.c 264d58971eaf8256eb5b0917bcd7fc7a1f1109fdda183a8382308a1b18a2dce7
 F src/window.c f8ba2ee12a19b51d3ba42c16277c74185ee9215306bc0d5a03974ade8b5bc98f
@@ -1858,7 +1858,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 14d14eb537075c6ac77513b1e7305bed8bc01a9034dfb763fd96f76400f2b705
-R 1f345124b6aafeeb9e600617cf39817f
+P f02030b3403d67734bba471a91ad5bfdb03ddf6fdc3ef14808a04495e43b0470
+R 0b7e29ed07d82aa79f09869cc21d3459
+T *branch * stat4-truthprob
+T *sym-stat4-truthprob *
+T -sym-trunk *
 U drh
-Z 38c81c56b51d8e5ae42e4e25b041736c
+Z b67db2665ec4d9de443f342c9e446178

manifest.uuid

@@ -1 +1 @@
-f02030b3403d67734bba471a91ad5bfdb03ddf6fdc3ef14808a04495e43b0470
+1babd6ec5d60e2c34aa1c0285ead768a88004218468e97262411973fe3487022

src/where.c

@@ -2307,7 +2307,10 @@ static void whereLoopOutputAdjust(
           }else{
             k = 20;
           }
-          if( iReduce<k ) iReduce = k;
+          if( iReduce<k ){
+            pTerm->wtFlags |= TERM_HEURTRUTH;
+            iReduce = k;
+          }
         }
       }
     }
@@ -2489,9 +2492,9 @@ static int whereLoopAddBtreeIndex(
     }
 
     if( IsUniqueIndex(pProbe) && saved_nEq==pProbe->nKeyCol-1 ){
-      pBuilder->bldFlags |= SQLITE_BLDF_UNIQUE;
+      pBuilder->bldFlags1 |= SQLITE_BLDF1_UNIQUE;
     }else{
-      pBuilder->bldFlags |= SQLITE_BLDF_INDEXED;
+      pBuilder->bldFlags1 |= SQLITE_BLDF1_INDEXED;
     }
     pNew->wsFlags = saved_wsFlags;
     pNew->u.btree.nEq = saved_nEq;
@@ -2656,6 +2659,21 @@ static int whereLoopAddBtreeIndex(
           if( rc!=SQLITE_OK ) break;          /* Jump out of the pTerm loop */
           if( nOut ){
             pNew->nOut = sqlite3LogEst(nOut);
+            if( nEq==1 && pTerm->truthProb>0 ){
+#if WHERETRACE_ENABLED /* 0x01 */
+              if( sqlite3WhereTrace & 0x01 ){
+                sqlite3DebugPrintf("Update truthProb from %d to %d:\n",
+                       pTerm->truthProb, pNew->nOut - pProbe->aiRowLogEst[0]);
+                sqlite3WhereTermPrint(pTerm, 999);
+              }
+#endif
+              pTerm->truthProb = pNew->nOut - pProbe->aiRowLogEst[0];
+              if( pTerm->wtFlags & TERM_HEURTRUTH ){
+                /* If the old heuristic truthProb was previously used, signal
+                ** that all loops will need to be recomputed */
+                pBuilder->bldFlags2 |= SQLITE_BLDF2_2NDPASS;
+              }
+            }
             if( pNew->nOut>saved_nOut ) pNew->nOut = saved_nOut;
             pNew->nOut -= nIn;
           }
@@ -3080,9 +3098,9 @@ static int whereLoopAddBtree(
       }
     }
 
-    pBuilder->bldFlags = 0;
+    pBuilder->bldFlags1 = 0;
     rc = whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, 0);
-    if( pBuilder->bldFlags==SQLITE_BLDF_INDEXED ){
+    if( pBuilder->bldFlags1==SQLITE_BLDF1_INDEXED ){
       /* If a non-unique index is used, or if a prefix of the key for
       ** unique index is used (making the index functionally non-unique)
       ** then the sqlite_stat1 data becomes important for scoring the
@@ -4838,6 +4856,25 @@ WhereInfo *sqlite3WhereBegin(
   if( nTabList!=1 || whereShortCut(&sWLB)==0 ){
     rc = whereLoopAddAll(&sWLB);
     if( rc ) goto whereBeginError;
+
+#ifdef SQLITE_ENABLE_STAT4
+    /* If one or more WhereTerm.truthProb values were used in estimating
+    ** loop parameters, but then those truthProb values were subsequently
+    ** changed based on STAT4 information while computing subsequent loops,
+    ** then we need to rerun the whole loop building process so that all
+    ** loops will be built using the revised truthProb values. */
+    if( sWLB.bldFlags2 & SQLITE_BLDF2_2NDPASS ){
+      WHERETRACE(0xffff, 
+         ("**** Redo all loop computations due to truthProb changes ****\n"));
+      while( pWInfo->pLoops ){
+        WhereLoop *p = pWInfo->pLoops;
+        pWInfo->pLoops = p->pNextLoop;
+        whereLoopDelete(db, p);
+      }
+      rc = whereLoopAddAll(&sWLB);
+      if( rc ) goto whereBeginError;
+    }
+#endif
   
 #ifdef WHERETRACE_ENABLED
     if( sqlite3WhereTrace ){    /* Display all of the WhereLoop objects */

src/whereInt.h

@@ -291,6 +291,7 @@ struct WhereTerm {
 #define TERM_LIKE       0x0400 /* The original LIKE operator */
 #define TERM_IS         0x0800 /* Term.pExpr is an IS operator */
 #define TERM_VARSELECT  0x1000 /* Term.pExpr contains a correlated sub-query */
+#define TERM_HEURTRUTH  0x2000 /* Heuristic truthProb used */
 
 /*
 ** An instance of the WhereScan object is used as an iterator for locating
@@ -405,13 +406,16 @@ struct WhereLoopBuilder {
   UnpackedRecord *pRec;     /* Probe for stat4 (if required) */
   int nRecValid;            /* Number of valid fields currently in pRec */
 #endif
-  unsigned int bldFlags;    /* SQLITE_BLDF_* flags */
+  unsigned char bldFlags1;  /* First set of SQLITE_BLDF_* flags */
+  unsigned char bldFlags2;  /* Second set of SQLITE_BLDF_* flags */
   unsigned int iPlanLimit;  /* Search limiter */
 };
 
 /* Allowed values for WhereLoopBuider.bldFlags */
-#define SQLITE_BLDF_INDEXED  0x0001   /* An index is used */
-#define SQLITE_BLDF_UNIQUE   0x0002   /* All keys of a UNIQUE index used */
+#define SQLITE_BLDF1_INDEXED  0x0001   /* An index is used */
+#define SQLITE_BLDF1_UNIQUE   0x0002   /* All keys of a UNIQUE index used */
+
+#define SQLITE_BLDF2_2NDPASS  0x0004   /* Second builder pass needed */
 
 /* The WhereLoopBuilder.iPlanLimit is used to limit the number of
 ** index+constraint combinations the query planner will consider for a