Continuing refinements of the range-scan optimizations in where.c.

The range scores are changed from an integer 1..9 to 0..100.

FossilOrigin-Name: f0c24b5fb86940f1a88adfb39cc4b9cbfcc66f8a

manifest

@@ -1,5 +1,8 @@
-C Change\sthe\scode\sthat\scollects\ssamples\sfor\ssqlite_stat2\sso\sthat\sthe\sfirst\ssample\staken\sis\sthe\s(nRow/(2*SQLITE_INDEX_SAMPLES))th\sentry\sin\sthe\sindex,\swhere\snRow\sis\sthe\stotal\snumber\sof\sindex\sentries.
+-----BEGIN PGP SIGNED MESSAGE-----
-D 2009-08-20T09:11:06
+Hash: SHA1
+
+C Continuing\srefinements\sof\sthe\srange-scan\soptimizations\sin\swhere.c.\nThe\srange\sscores\sare\schanged\sfrom\san\sinteger\s1..9\sto\s0..100.
+D 2009-08-20T18:14:43
 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0
 F Makefile.in 0f7761c5d1c62ae7a841e3393ffaff1fa0f5c00a
 F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654
@@ -214,7 +217,7 @@ F src/vdbeblob.c a3f3e0e877fc64ea50165eec2855f5ada4477611
 F src/vdbemem.c c4a5188ff43692f2ca78d3539ad4877e14b70712
 F src/vtab.c aedd76e8670d5a5379f93804398d3ba960125547
 F src/walker.c 1edca756275f158b80f20eb6f104c8d3fcc96a04
-F src/where.c 02f2bb999fa80df9399b5a906d2ce988b2e85541
+F src/where.c 2d258a5698152e446f4bc4577b3bc7dd5ac67f79
 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2
 F test/alias.test 4529fbc152f190268a15f9384a5651bbbabc9d87
 F test/all.test 14165b3e32715b700b5f0cbf8f6e3833dda0be45
@@ -714,7 +717,7 @@ F test/where3.test 97d3936e6a443b968f1a61cdcc0f673252000e94
 F test/where4.test e9b9e2f2f98f00379e6031db6a6fca29bae782a2
 F test/where5.test fdf66f96d29a064b63eb543e28da4dfdccd81ad2
 F test/where6.test 42c4373595f4409d9c6a9987b4a60000ad664faf
-F test/where7.test b6e84b472a024e45c6dbdadc52bbcab3fcc8d0e1
+F test/where7.test fdd58ab9dec9f97679e65d4414bf5e07d725d79f
 F test/where8.test 8d3704d04a683e792d373005f2e4e13bfd7e2dd5
 F test/where8m.test da346596e19d54f0aba35ebade032a7c47d79739
 F test/where9.test be19e1a92f80985c1a121b4678bf7d2123eaa623
@@ -747,7 +750,14 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff
 F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e
 F tool/vdbe-compress.tcl 672f81d693a03f80f5ae60bfefacd8a349e76746
-P 4a5d9550bdc08633535a7869d7748f56ac3e9a36
+P cbfe6e9df39684607cbc9637e3fb3c5ee6af2515
-R a06a46e0e6092a62dbfee6601412e863
+R 43a18a46bd05bd23f1d5442f4fb0bca8
-U dan
+U drh
-Z 2c479b2a4ff638923d236b4fdc91cdb8
+Z 786f0f962425fe36a5d48fe3c8eb25c6
+-----BEGIN PGP SIGNATURE-----
+Version: GnuPG v1.4.6 (GNU/Linux)
+
+iD8DBQFKjZKWoxKgR168RlERAhTBAJ9Ykk0kCcM2iERfod39WLJXi/CtswCcDNpn
+cSayfJiAGIFOSYysKjP59ko=
+=+imO
+-----END PGP SIGNATURE-----

manifest.uuid

@@ -1 +1 @@
-cbfe6e9df39684607cbc9637e3fb3c5ee6af2515
+f0c24b5fb86940f1a88adfb39cc4b9cbfcc66f8a

src/where.c

@@ -1900,9 +1900,10 @@ static void bestVirtualIndex(
 ** but smaller than the value of the second. And so on.
 **
 ** If successful, this function determines which of the regions value 
-** pVal lies in, sets *piRegion to the region index and returns SQLITE_OK.
+** pVal lies in, sets *piRegion to the region index (a value between 0
+** and SQLITE_INDEX_SAMPLES+1, inclusive) and returns SQLITE_OK.
 ** Or, if an OOM occurs while converting text values between encodings,
-** SQLITE_NOMEM is returned.
+** SQLITE_NOMEM is returned and *piRegion is undefined.
 */
 #ifdef SQLITE_ENABLE_STAT2
 static int whereRangeRegion(
@@ -1991,7 +1992,7 @@ static int whereRangeRegion(
 **                       |         |
 **                     pLower    pUpper
 **
-** If the upper or lower bound is not present, then NULL should be passed in
+** If either of the upper or lower bound is not present, then NULL is passed in
 ** place of the corresponding WhereTerm.
 **
 ** The nEq parameter is passed the index of the index column subject to the
@@ -2008,12 +2009,17 @@ static int whereRangeRegion(
 **
 ** then nEq should be passed 0.
 **
-** The returned value is an integer between 1 and 9, inclusive. A return
+** The returned value is an integer between 1 and 100, inclusive. A return
 ** value of 1 indicates that the proposed range scan is expected to visit
-** approximately 1/9 (11%) of the rows selected by the nEq equality constraints
+** approximately 1/100th (1%) of the rows selected by the nEq equality
-** (if any). A return value of 9 indicates that it is expected that the
+** constraints (if any). A return value of 100 indicates that it is expected
-** range scan will visit 9/9 (100%) of the rows selected by the equality
+** that the range scan will visit every row (100%) selected by the equality
 ** constraints.
+**
+** In the absence of sqlite_stat2 ANALYZE data, each range inequality
+** reduces the search space by 2/3rds.  Hence a single constraint (x>?)
+** results in a return of 33 and a range constraint (x>? AND x<?) results
+** in a return of 11.
 */
 static int whereRangeScanEst(
   Parse *pParse,       /* Parsing & code generating context */
@@ -2032,41 +2038,59 @@ static int whereRangeScanEst(
 
   if( nEq==0 && p->aSample ){
     int iEst;
-    int iUpper = SQLITE_INDEX_SAMPLES;
+    int iUpper;
-    int iLower = 0;
+    int iLower;
     u8 aff = p->pTable->aCol[0].affinity;
+
     if( pLower ){
       Expr *pExpr = pLower->pExpr->pRight;
       rc = sqlite3ValueFromExpr(db, pExpr, SQLITE_UTF8, aff, &pLowerVal);
-      if( !pLowerVal ) goto fallback;
     }
-    if( pUpper ){
+    if( rc==SQLITE_OK && pUpper ){
       Expr *pExpr = pUpper->pExpr->pRight;
       rc = sqlite3ValueFromExpr(db, pExpr, SQLITE_UTF8, aff, &pUpperVal);
-      if( !pUpperVal ){
+    }
-        sqlite3ValueFree(pLowerVal);
+
-        goto fallback;
+    if( rc!=SQLITE_OK || (pLowerVal==0 && pUpperVal==0) ){
+      sqlite3ValueFree(pLowerVal);
+      sqlite3ValueFree(pUpperVal);
+      goto range_est_fallback;
+    }else if( pLowerVal==0 ){
+      rc = whereRangeRegion(pParse, p, pUpperVal, &iUpper);
+      iLower = pLower ? iUpper/2 : 0;
+    }else if( pUpperVal==0 ){
+      rc = whereRangeRegion(pParse, p, pLowerVal, &iLower);
+      iUpper = pUpper ? (iLower + SQLITE_INDEX_SAMPLES + 1)/2 
+                      : SQLITE_INDEX_SAMPLES;
+    }else{
+      rc = whereRangeRegion(pParse, p, pUpperVal, &iUpper);
+      if( rc==SQLITE_OK ){
+        rc = whereRangeRegion(pParse, p, pLowerVal, &iLower);
+      }else{
+        iLower = 0;
       }
     }
 
-    rc = whereRangeRegion(pParse, p, pUpperVal, &iUpper);
-    if( rc==SQLITE_OK ){
-      rc = whereRangeRegion(pParse, p, pLowerVal, &iLower);
-    }
-
     iEst = iUpper - iLower;
-    if( iEst>=SQLITE_INDEX_SAMPLES ) iEst = SQLITE_INDEX_SAMPLES-1;
+    if( iEst>SQLITE_INDEX_SAMPLES ){
-    else if( iEst<1 ) iEst = 1;
+      iEst = SQLITE_INDEX_SAMPLES;
+    }else if( iEst<1 ){
+      iEst = 1;
+    }
 
     sqlite3ValueFree(pLowerVal);
     sqlite3ValueFree(pUpperVal);
-    *piEst = iEst;
+    *piEst = (iEst * 100)/SQLITE_INDEX_SAMPLES;
     return rc;
   }
-fallback:
+range_est_fallback:
 #endif
   assert( pLower || pUpper );
-  *piEst = (SQLITE_INDEX_SAMPLES-1) / ((pLower&&pUpper)?9:3);
+  if( pLower && pUpper ){
+    *piEst = 11;
+  }else{
+    *piEst = 33;
+  }
   return rc;
 }
 
@@ -2212,10 +2236,13 @@ static void bestBtreeIndex(
     **    in determining the value of nInMul.
     **
     **  nBound:
-    **    An estimate on the amount of the table that must be searched due
+    **    An estimate on the amount of the table that must be searched.  A
-    **    to a range constraint.  The value is between 1 and 9 and indicates
+    **    value of 100 means the entire table is searched.  Range constraints
-    **    9ths of the table.  1 means that about 1/9th of the is searched.
+    **    might reduce this to a value less than 100 to indicate that only
-    **    9 indicates that the entire table is searched.
+    **    a fraction of the table needs searching.  In the absence of
+    **    sqlite_stat2 ANALYZE data, a single inequality reduces the search
+    **    space to 1/3rd its original size.  So an x>? constraint reduces
+    **    nBound to 33.  Two constraints (x>? AND x<?) reduce nBound to 11.
     **
     **  bSort:   
     **    Boolean. True if there is an ORDER BY clause that will require an 
@@ -2237,7 +2264,7 @@ static void bestBtreeIndex(
     int nEq;
     int bInEst = 0;
     int nInMul = 1;
-    int nBound = 9;
+    int nBound = 100;
     int bSort = 0;
     int bLookup = 0;
 
@@ -2344,8 +2371,8 @@ static void bestBtreeIndex(
     /* Adjust the number of rows and the cost downward to reflect rows
     ** that are excluded by range constraints.
     */
-    nRow = nRow * (double)nBound / (double)9;
+    nRow = (nRow * (double)nBound) / (double)100;
-    cost = cost * (double)nBound / (double)9;
+    cost = (cost * (double)nBound) / (double)100;
 
     /* Add in the estimated cost of sorting the result
     */

test/where7.test

@@ -90,10 +90,12 @@ do_test where7-1.9 {
   }
 } {2 4 5 scan 0 sort 0}
 do_test where7-1.10 {
+breakpoint
   count_steps {
     SELECT a FROM t1 WHERE (b=3 OR c>=10 OR c=4 OR b>10)
   }
 } {2 4 5 scan 0 sort 0}
+breakpoint
 do_test where7-1.11 {
   count_steps {
     SELECT a FROM t1 WHERE (d=5 AND b=3) OR c==100 ORDER BY a;
@@ -5412,7 +5414,7 @@ do_test where7-2.235.1 {
          OR (g='rqponml' AND f GLOB 'lmnop*')
          OR (f GLOB '?ijkl*' AND f GLOB 'hijk*')
   }
-} {7 14 18 31 33 37 40 51 53 59 66 85 92 94 98 100 scan 99 sort 0}
+} {7 14 18 31 33 37 40 51 53 59 66 85 92 94 98 100 scan 0 sort 0}
 do_test where7-2.235.2 {
   count_steps_sort {
      SELECT a FROM t3
@@ -5426,7 +5428,7 @@ do_test where7-2.235.2 {
          OR (g='rqponml' AND f GLOB 'lmnop*')
          OR (f GLOB '?ijkl*' AND f GLOB 'hijk*')
   }
-} {7 14 18 31 33 37 40 51 53 59 66 85 92 94 98 100 scan 99 sort 0}
+} {7 14 18 31 33 37 40 51 53 59 66 85 92 94 98 100 scan 0 sort 0}
 do_test where7-2.236.1 {
   count_steps_sort {
      SELECT a FROM t2
@@ -6854,7 +6856,7 @@ do_test where7-2.297.1 {
          OR (f GLOB '?ghij*' AND f GLOB 'fghi*')
          OR ((a BETWEEN 53 AND 55) AND a!=54)
   }
-} {5 7 18 20 23 25 31 33 37 39 45 53 54 55 56 57 58 59 72 74 83 85 95 scan 99 sort 0}
+} {5 7 18 20 23 25 31 33 37 39 45 53 54 55 56 57 58 59 72 74 83 85 95 scan 0 sort 0}
 do_test where7-2.297.2 {
   count_steps_sort {
      SELECT a FROM t3
@@ -8952,7 +8954,7 @@ do_test where7-2.385.1 {
          OR (d>=21.0 AND d<22.0 AND d NOT NULL)
          OR ((a BETWEEN 93 AND 95) AND a!=94)
   }
-} {1 11 13 21 22 24 26 27 32 34 39 41 53 61 74 76 79 93 95 scan 99 sort 0}
+} {1 11 13 21 22 24 26 27 32 34 39 41 53 61 74 76 79 93 95 scan 0 sort 0}
 do_test where7-2.385.2 {
   count_steps_sort {
      SELECT a FROM t3