Only choose to scan an IN operator rather than use an index if we have

real STAT1 data to suggest it is advantageous.

FossilOrigin-Name: 30e874661dcc1a2ecb40df2ef74582151d85bb36c754a38548829a3b6285f18d

manifest

@@ -1,5 +1,5 @@
-C Merge\sthe\sbtreeNext()\sassertion\sbug\sfix\sfrom\strunk.
+C Only\schoose\sto\sscan\san\sIN\soperator\srather\sthan\suse\san\sindex\sif\swe\shave\nreal\sSTAT1\sdata\sto\ssuggest\sit\sis\sadvantageous.
-D 2018-06-08T19:54:07.050
+D 2018-06-08T21:21:01.417
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in bfc40f350586923e0419d2ea4b559c37ec10ee4b6e210e08c14401f8e340f0da
@@ -579,7 +579,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c aa9cffc7a2bad6b826a86c8562dd4978398720ed41cb8ee7aa9d054eb8b456a0
 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a
 F src/walker.c da987a20d40145c0a03c07d8fefcb2ed363becc7680d0500d9c79915591f5b1f
-F src/where.c 8e95858b398f7451c30ed05dd372f39767f04f423fa1d15a19d568268ae9a557
+F src/where.c e4d48338ca4718c8034f313cca202cc5ca75639a24d27b959b65578198f63c81
 F src/whereInt.h b09753e74bf92a8b17cf0e41ca94c44432c454544be6699b5311dcc57bf229c6
 F src/wherecode.c 3317f2b083a66d3e65a03edf316ade4ccb0a99c9956273282ebb579b95d4ba96
 F src/whereexpr.c e90b2e76dcabc81edff56633bf281bc01d93b71e0c81482dc06925ce39f5844a
@@ -967,7 +967,7 @@ F test/in2.test 5d4c61d17493c832f7d2d32bef785119e87bde75
 F test/in3.test 3cbf58c87f4052cee3a58b37b6389777505aa0c0
 F test/in4.test d2b38cba404bc4320f4fe1b595b3d163f212c068
 F test/in5.test 7ae37fcd4a5e198291c6ab5f31a5bb3d15397efe8b75a6736d7a95a7b8dd9e08
-F test/in6.test 77c3e1d356d8aeb0864051f0677d3c0a032cf97b7f33a0ba8fa2b04a663f6b7b
+F test/in6.test 62d943a02f722948f4410ee0b53c3cb39acd7c41afb083df8d7004238fe90a20
 F test/incrblob.test c9b96afc292aeff43d6687bcb09b0280aa599822
 F test/incrblob2.test a494c9e848560039a23974b9119cfc2cf3ad3bd15cc2694ee6367ae537ef8f1f
 F test/incrblob3.test d8d036fde015d4a159cd3cbae9d29003b37227a4
@@ -1185,7 +1185,7 @@ F test/rowid.test 5b7509f384f4f6fae1af3c8c104c8ca299fea18d
 F test/rowvalue.test ef851a80f7e6de93b51caca9e4b6b7d2dcd540bbcca7d51860e80435b8b4c0de
 F test/rowvalue2.test 060d238b7e5639a7c5630cb5e63e311b44efef2b
 F test/rowvalue3.test 3068f508753af69884b12125995f023da0dbb256
-F test/rowvalue4.test cbd1cead27a797d11ec93301fd801c89e97eb1809b3d984b7f16a3876e362eac
+F test/rowvalue4.test 2b20468da3775aba971caf3158e9696a4d99c69a7623fb495f332a596daebbee
 F test/rowvalue5.test c81c7d8cf36711ab37675ad7376084ae2a359cb6
 F test/rowvalue6.test d19b54feb604d5601f8614b15e214e0774c01087
 F test/rowvalue7.test 5d06ff19d9e6969e574a2e662a531dd0c67801a8
@@ -1731,7 +1731,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 1fa40a78fef4516c39b217bff67efe7e7d2077cca00aae0ef5c2c9cff94f008b 99057383acc8f92093530e216c621d40386a06fe98131ff0af6df524d80a6410
+P 11bd66e09035039fbfc4c5631196962dd3f2e6634f5123a7b2fec17aeaaab07b
-R 56ae5d735c58aa67ec9684ecac6c800d
+R 54c7353f86aece043f7640340784a722
 U drh
-Z b50b0532bda90928b75cfc1466c6bb97
+Z b0c1273ce7490f506712c4b76e5917fd

manifest.uuid

@@ -1 +1 @@
-11bd66e09035039fbfc4c5631196962dd3f2e6634f5123a7b2fec17aeaaab07b
+30e874661dcc1a2ecb40df2ef74582151d85bb36c754a38548829a3b6285f18d

src/where.c

@@ -2471,34 +2471,38 @@ static int whereLoopAddBtreeIndex(
         assert( nIn>0 );  /* RHS always has 2 or more terms...  The parser
                           ** changes "x IN (?)" into "x=?". */
       }
-      /* Let:
+      if( pProbe->hasStat1 ){
-      **   N = the total number of rows in the table
+        /* Let:
-      **   K = the number of entries on the right-hand side of the IN operator
+        **   N = the total number of rows in the table
-      **   M = the number of rows in the table that match terms to the 
+        **   K = the number of entries on the RHS of the IN operator
-      **       to the left in the same index.  If the IN operator is on
+        **   M = the number of rows in the table that match terms to the 
-      **       the left-most index column, M==N.
+        **       to the left in the same index.  If the IN operator is on
-      **
+        **       the left-most index column, M==N.
-      ** Given the definitions above, it is better to omit the IN operator
+        **
-      ** from the index lookup and instead do a scan of the M elements,
+        ** Given the definitions above, it is better to omit the IN operator
-      ** testing each scanned row against the IN operator separately, if:
+        ** from the index lookup and instead do a scan of the M elements,
-      **
+        ** testing each scanned row against the IN operator separately, if:
-      **        M*log(K) < K*log(N)
+        **
-      **
+        **        M*log(K) < K*log(N)
-      ** Our estimates for M, K, and N might be inaccurate, so we build in
+        **
-      ** a safety margin of 2 (LogEst: 10) that favors using the IN operator
+        ** Our estimates for M, K, and N might be inaccurate, so we build in
-      ** with the index, as using an index has better worst-case behavior.
+        ** a safety margin of 2 (LogEst: 10) that favors using the IN operator
-      */
+        ** with the index, as using an index has better worst-case behavior.
-      M = pProbe->aiRowLogEst[saved_nEq+1];
+        ** If we do not have real sqlite_stat1 data, always prefer to use
-      logK = sqlite3LogEst(nIn);
+        ** the index.
-      if( M + logK + 10 < nIn + rLogSize ){
+        */
-        WHERETRACE(0x40,
+        M = pProbe->aiRowLogEst[saved_nEq];
-          ("IN operator costs more than scan on column %d of \"%s\" (%d<%d)\n",
+        logK = estLog(nIn);
-           saved_nEq, pProbe->zName, M+logK+10, nIn+rLogSize));
+        if( M + logK + 10 < nIn + rLogSize ){
-        continue;
+          WHERETRACE(0x40,
-      }else{
+            ("Scan preferred over IN operator on column %d of \"%s\" (%d<%d)\n",
-        WHERETRACE(0x40,
+             saved_nEq, pProbe->zName, M+logK+10, nIn+rLogSize));
-          ("IN operator cheaper than scan on column %d of \"%s\" (%d>=%d)\n",
+          continue;
-           saved_nEq, pProbe->zName, M+logK+10, nIn+rLogSize));
+        }else{
+          WHERETRACE(0x40,
+            ("IN operator preferred on column %d of \"%s\" (%d>=%d)\n",
+             saved_nEq, pProbe->zName, M+logK+10, nIn+rLogSize));
+        }
       }
       pNew->wsFlags |= WHERE_COLUMN_IN;
     }else if( eOp & (WO_EQ|WO_IS) ){

test/in6.test

@@ -28,6 +28,9 @@ do_test in6-1.1 {
       INSERT INTO t1(a,b,c,d)
         SELECT 100, 200+x/2, 300+x/5, x FROM c;
     CREATE INDEX t1abc ON t1(a,b,c);
+    ANALYZE;
+    UPDATE sqlite_stat1 SET stat='1000000 500000 500 50';
+    ANALYZE sqlite_master;
   }
   set ::sqlite_search_count 0
   db eval {

test/rowvalue4.test

@@ -224,7 +224,7 @@ do_execsql_test 5.0 {
   WITH i(i) AS (
     VALUES(1) UNION ALL SELECT i+1 FROM i WHERE i<1000
   )
-  INSERT INTO d2 SELECT i/3, i%3, i/3 FROM i;
+  INSERT INTO d2 SELECT i/100, i%100, i/100 FROM i;
   ANALYZE;
 }