When estimating the number of rows scanned using data from the sqlite_stat4 table, avoid allocating UnpackedRecord and KeyInfo structures until they are definitely required.

FossilOrigin-Name: 353950a5269fa439cc3e57b62e16558a84ea2557

manifest

@@ -1,5 +1,5 @@
-C Fixes\sfor\sbuilds\swithout\sSQLITE_ENABLE_STAT4.
+C When\sestimating\sthe\snumber\sof\srows\sscanned\susing\sdata\sfrom\sthe\ssqlite_stat4\stable,\savoid\sallocating\sUnpackedRecord\sand\sKeyInfo\sstructures\suntil\sthey\sare\sdefinitely\srequired.
-D 2013-08-06T20:15:06.692
+D 2013-08-07T15:52:41.738
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -221,7 +221,7 @@ F src/shell.c 52f975eae87c8338c4dfbf4c2842d2a0971f01fd
 F src/sqlite.h.in 442c109e0c3447c34b1794971ecdb673ce08a843
 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0
 F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc
-F src/sqliteInt.h 31057687e0ebc9e56a025bcebb252bbd84e5db13
+F src/sqliteInt.h f56ec862ae37cf1b6e699ab5a2b53d40c8ec5068
 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d
 F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158
 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e
@@ -283,14 +283,14 @@ F src/vdbeInt.h e9b7c6b165a31a4715c5aa97223d20d265515231
 F src/vdbeapi.c 4d13580bd058b39623e8fcfc233b7df4b8191e8b
 F src/vdbeaux.c 4389b3692969b4415fcfd00de36818a02f84df28
 F src/vdbeblob.c 5dc79627775bd9a9b494dd956e26297946417d69
-F src/vdbemem.c 83f9b6e68a421cfcdde125a66b5ebe8220c12de2
+F src/vdbemem.c f0512045147702adec3ca6388663e243c17d2ea4
 F src/vdbesort.c 3937e06b2a0e354500e17dc206ef4c35770a5017
 F src/vdbetrace.c e7ec40e1999ff3c6414424365d5941178966dcbc
 F src/vtab.c 2e8b489db47e20ae36cd247932dc671c9ded0624
 F src/wal.c 7dc3966ef98b74422267e7e6e46e07ff6c6eb1b4
 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4
 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73
-F src/where.c 142525786b7855a0997e113f3932833bf5c1f63c
+F src/where.c 4e188dc4a1f668d761750eb27e603616179806d0
 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2
 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6
@@ -1106,7 +1106,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381
 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae
 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac
-P 2973f5ca736c4a6f13c653d54b6a29d7cae8d0ed
+P 84999e27cc0d14b89d9fe024e29d287c69285369
-R 3fc350a341f72f5a912cf9e1ccf56bc1
+R b05f226dd58169f35c18db270697f169
 U dan
-Z 97c03093fe0c6a197e3d6120177c182a
+Z 7ab1efe6195a9307bc618357bb65cf2b

manifest.uuid

@@ -1 +1 @@
-84999e27cc0d14b89d9fe024e29d287c69285369
+353950a5269fa439cc3e57b62e16558a84ea2557

src/sqliteInt.h

@@ -1545,7 +1545,7 @@ struct Index {
   unsigned autoIndex:2;    /* 1==UNIQUE, 2==PRIMARY KEY, 0==CREATE INDEX */
   unsigned bUnordered:1;   /* Use this index for == or IN queries only */
   unsigned uniqNotNull:1;  /* True if UNIQUE and NOT NULL for all columns */
-#ifdef SQLITE_ENABLE_STAT3
+#ifdef SQLITE_ENABLE_STAT4
   int nSample;             /* Number of elements in aSample[] */
   tRowcnt avgEq;           /* Average nEq value for key values not in aSample */
   IndexSample *aSample;    /* Samples of the left-most key */
@@ -3101,9 +3101,8 @@ Expr *sqlite3CreateColumnExpr(sqlite3 *, SrcList *, int, int);
 void sqlite3BackupRestart(sqlite3_backup *);
 void sqlite3BackupUpdate(sqlite3_backup *, Pgno, const u8 *);
 
-int sqlite3Stat4ProbeSetValue(Parse*, UnpackedRecord*, Expr*, u8, int, int*);
+int sqlite3Stat4ProbeSetValue(Parse*,Index*,UnpackedRecord**,Expr*,u8,int,int*);
 void sqlite3Stat4ProbeFree(UnpackedRecord*);
-int sqlite3Stat4ProbeNew(Parse*, Index*, UnpackedRecord**);
 
 /*
 ** The interface to the LEMON-generated parser

src/vdbemem.c

@@ -1005,27 +1005,30 @@ sqlite3_value *sqlite3ValueNew(sqlite3 *db){
   return p;
 }
 
-static sqlite3_value *valueNew(sqlite3 *db, sqlite3_value *pOld){
+/*
-  if( pOld ) return pOld;
+** Argument pCtx is actually a pointer to a database handle. Allocate and
-  return sqlite3ValueNew(db);
+** return an sqlite3_value object associated with this database handle.
+**
+** This function used as the xAlloc callback for valueFromExpr() when
+** it is called by sqlite3ValueFromExpr().
+*/
+static sqlite3_value *valueNew(void *pCtx){
+  return sqlite3ValueNew((sqlite3*)pCtx);
 }
 
 /*
-** Create a new sqlite3_value object, containing the value of pExpr.
+** This function is the same as sqlite3ValueFromExpr(), except that instead
-**
+** of allocating any required sqlite3_value object by calling 
-** This only works for very simple expressions that consist of one constant
+** sqlite3ValueNew(), it does so by calling the supplied xAlloc hook. 
-** token (i.e. "5", "5.1", "'a string'"). If the expression can
-** be converted directly into a value, then the value is allocated and
-** a pointer written to *ppVal. The caller is responsible for deallocating
-** the value by passing it to sqlite3ValueFree() later on. If the expression
-** cannot be converted to a value, then *ppVal is set to NULL.
 */
-int sqlite3ValueFromExpr(
+int valueFromExpr(
-  sqlite3 *db,              /* The database connection */
+  sqlite3 *db,                         /* The database connection */
-  Expr *pExpr,              /* The expression to evaluate */
+  Expr *pExpr,                         /* The expression to evaluate */
-  u8 enc,                   /* Encoding to use */
+  u8 enc,                              /* Encoding to use */
-  u8 affinity,              /* Affinity to use */
+  u8 affinity,                         /* Affinity to use */
-  sqlite3_value **ppVal     /* Write the new value here */
+  sqlite3_value **ppVal,               /* Write the new value here */
+  sqlite3_value *(*xAlloc)(void*),     /* Used to allocate new sqlite3_value */
+  void *pAlloc                         /* Argument passed to xAlloc */
 ){
   int op;
   char *zVal = 0;
@@ -1061,7 +1064,7 @@ int sqlite3ValueFromExpr(
   }
 
   if( op==TK_STRING || op==TK_FLOAT || op==TK_INTEGER ){
-    pVal = valueNew(db, *ppVal);
+    pVal = xAlloc(pAlloc);
     if( pVal==0 ) goto no_mem;
     if( ExprHasProperty(pExpr, EP_IntValue) ){
       sqlite3VdbeMemSetInt64(pVal, (i64)pExpr->u.iValue*negInt);
@@ -1095,7 +1098,7 @@ int sqlite3ValueFromExpr(
       sqlite3ValueApplyAffinity(pVal, affinity, enc);
     }
   }else if( op==TK_NULL ){
-    pVal = valueNew(db, *ppVal);
+    pVal = xAlloc(pAlloc);
     if( pVal==0 ) goto no_mem;
   }
 #ifndef SQLITE_OMIT_BLOB_LITERAL
@@ -1103,7 +1106,7 @@ int sqlite3ValueFromExpr(
     int nVal;
     assert( pExpr->u.zToken[0]=='x' || pExpr->u.zToken[0]=='X' );
     assert( pExpr->u.zToken[1]=='\'' );
-    pVal = valueNew(db, *ppVal);
+    pVal = xAlloc(pAlloc);
     if( !pVal ) goto no_mem;
     zVal = &pExpr->u.zToken[2];
     nVal = sqlite3Strlen30(zVal)-1;
@@ -1127,51 +1130,178 @@ no_mem:
   return SQLITE_NOMEM;
 }
 
+/*
+** Create a new sqlite3_value object, containing the value of pExpr.
+**
+** This only works for very simple expressions that consist of one constant
+** token (i.e. "5", "5.1", "'a string'"). If the expression can
+** be converted directly into a value, then the value is allocated and
+** a pointer written to *ppVal. The caller is responsible for deallocating
+** the value by passing it to sqlite3ValueFree() later on. If the expression
+** cannot be converted to a value, then *ppVal is set to NULL.
+*/
+int sqlite3ValueFromExpr(
+  sqlite3 *db,              /* The database connection */
+  Expr *pExpr,              /* The expression to evaluate */
+  u8 enc,                   /* Encoding to use */
+  u8 affinity,              /* Affinity to use */
+  sqlite3_value **ppVal     /* Write the new value here */
+){
+  return valueFromExpr(db, pExpr, enc, affinity, ppVal, valueNew, (void*)db);
+}
+
 #ifdef SQLITE_ENABLE_STAT4
+/*
+** A pointer to an instance of this object is passed as the context 
+** pointer to valueNewStat4() (see below.
+*/
+struct ValueNewStat4Ctx {
+  Parse *pParse;
+  Index *pIdx;
+  UnpackedRecord **ppRec;
+  int iVal;
+};
+
+/*
+** This function is used as the xAlloc function with valueFromExpr() when
+** it is called by sqlite3Stat4ProbeSetValue(). The argument points to
+** an object of type ValueNewStat4Ctx (see above).
+**
+** If it has not already been allocated, this function allocates an 
+** UnpackedRecord structure and space for up to N values, where N is the
+** number of columns in the index being probed.
+*/
+static sqlite3_value *valueNewStat4(void *pCtx){
+  struct ValueNewStat4Ctx *p = (struct ValueNewStat4Ctx*)pCtx;
+  UnpackedRecord *pRec = p->ppRec[0];
+
+  if( pRec==0 ){
+    sqlite3 *db = p->pParse->db;  /* Database handle */
+    Index *pIdx = p->pIdx;        /* Index being probed */
+    int nByte;                    /* Bytes of space to allocate */
+    int i;                        /* Counter variable */
+
+    nByte = sizeof(Mem) * pIdx->nColumn + sizeof(UnpackedRecord);
+    pRec = (UnpackedRecord*)sqlite3DbMallocZero(db, nByte);
+    if( pRec ){
+      pRec->pKeyInfo = sqlite3IndexKeyinfo(p->pParse, pIdx);
+      if( pRec->pKeyInfo ){
+        pRec->pKeyInfo->enc = ENC(db);
+        pRec->flags = UNPACKED_PREFIX_MATCH;
+        pRec->aMem = (Mem *)&pRec[1];
+        for(i=0; i<pIdx->nColumn; i++){
+          pRec->aMem[i].flags = MEM_Null;
+          pRec->aMem[i].type = SQLITE_NULL;
+          pRec->aMem[i].db = db;
+        }
+      }else{
+        sqlite3DbFree(db, pRec);
+        pRec = 0;
+      }
+    }
+    if( pRec==0 ) return 0;
+    p->ppRec[0] = pRec;
+  }
+
+  pRec->nField = p->iVal+1;
+  return &pRec->aMem[p->iVal];
+}
+
+/*
+** This function is used to allocate and populate UnpackedRecord 
+** structures intended to be compared against sample index keys stored 
+** in the sqlite_stat4 table.
+**
+** A single call to this function attempts to populates field iVal (leftmost 
+** is 0 etc.) of the unpacked record with a value extracted from expression
+** pExpr. Extraction of values is possible if:
+**
+**  * (pExpr==0). In this case the value is assumed to be an SQL NULL,
+**
+**  * The expression is a bound variable, and this is a reprepare, or
+**
+**  * The sqlite3ValueFromExpr() function is able to extract a value 
+**    from the expression (i.e. the expression is a literal value).
+**
+** If a value can be extracted, the affinity passed as the 5th argument
+** is applied to it before it is copied into the UnpackedRecord. Output
+** parameter *pbOk is set to true if a value is extracted, or false 
+** otherwise.
+**
+** When this function is called, *ppRec must either point to an object
+** allocated by an earlier call to this function, or must be NULL. If it
+** is NULL and a value can be successfully extracted, a new UnpackedRecord
+** is allocated (and *ppRec set to point to it) before returning.
+**
+** Unless an error is encountered, SQLITE_OK is returned. It is not an
+** error if a value cannot be extracted from pExpr. If an error does
+** occur, an SQLite error code is returned.
+*/
 int sqlite3Stat4ProbeSetValue(
   Parse *pParse,                  /* Parse context */
-  UnpackedRecord *pRec,           /* Set field in this probe */
+  Index *pIdx,                    /* Index being probed */
+  UnpackedRecord **ppRec,         /* IN/OUT: Probe record */
   Expr *pExpr,                    /* The expression to extract a value from */
   u8 affinity,                    /* Affinity to use */
   int iVal,                       /* Array element to populate */
   int *pbOk                       /* OUT: True if value was extracted */
 ){
   int rc = SQLITE_OK;
-  sqlite3_value *pVal = &pRec->aMem[iVal];
+  sqlite3_value *pVal = 0;
+
+  struct ValueNewStat4Ctx alloc;
+  alloc.pParse = pParse;
+  alloc.pIdx = pIdx;
+  alloc.ppRec = ppRec;
+  alloc.iVal = iVal;
 
 #if 0
   if( iVal>0 ){ *pbOk = 0; return SQLITE_OK; }
 #endif
 
   if( !pExpr ){
-    sqlite3VdbeMemSetNull((Mem*)pVal);
+    pVal = valueNewStat4((void*)&alloc);
-    *pbOk = 1;
+    if( pVal ){
+      sqlite3VdbeMemSetNull((Mem*)pVal);
+      *pbOk = 1;
+    }
   }else if( pExpr->op==TK_VARIABLE
         || (pExpr->op==TK_REGISTER && pExpr->op2==TK_VARIABLE)
   ){
     Vdbe *v;
     int iVar = pExpr->iColumn;
     sqlite3VdbeSetVarmask(pParse->pVdbe, iVar);
-    if( v = pParse->pReprepare ){
+    if( (v = pParse->pReprepare) ){
-      rc = sqlite3VdbeMemCopy((Mem*)pVal, &v->aVar[iVal-1]);
+      pVal = valueNewStat4((void*)&alloc);
-      if( rc==SQLITE_OK ){
+      if( pVal ){
-        sqlite3ValueApplyAffinity(pVal, affinity, SQLITE_UTF8);
+        rc = sqlite3VdbeMemCopy((Mem*)pVal, &v->aVar[iVal-1]);
+        if( rc==SQLITE_OK ){
+          sqlite3ValueApplyAffinity(pVal, affinity, SQLITE_UTF8);
+        }
+        pVal->db = pParse->db;
+        *pbOk = 1;
+        sqlite3VdbeMemStoreType((Mem*)pVal);
       }
-      pVal->db = pParse->db;
-      *pbOk = 1;
-      sqlite3VdbeMemStoreType((Mem*)pVal);
     }else{
       *pbOk = 0;
     }
   }else{
-    sqlite3 *db = pRec->aMem[0].db;
+    sqlite3 *db = pParse->db;
-    rc = sqlite3ValueFromExpr(db, pExpr, ENC(db), affinity, &pVal);
+    rc = valueFromExpr(
+        db, pExpr, ENC(db), affinity, &pVal, valueNewStat4, (void*)&alloc
+    );
     *pbOk = (pVal!=0);
   }
+
   assert( pVal==0 || pVal->db==pParse->db );
   return rc;
 }
 
+/*
+** Unless it is NULL, the argument must be an UnpackedRecord object returned
+** by an earlier call to sqlite3Stat4ProbeSetValue(). This call deletes
+** the object.
+*/
 void sqlite3Stat4ProbeFree(UnpackedRecord *pRec){
   if( pRec ){
     int i;
@@ -1184,40 +1314,6 @@ void sqlite3Stat4ProbeFree(UnpackedRecord *pRec){
     sqlite3DbFree(db, pRec);
   }
 }
-
-int sqlite3Stat4ProbeNew(
-  Parse *pParse,                  /* Parse context */
-  Index *pIdx,                    /* Allocate record for this index */
-  UnpackedRecord **ppRec          /* OUT: Allocated record */
-){
-  sqlite3 *db = pParse->db;       /* Database handle */
-  UnpackedRecord *pRec;           /* Return value */
-  int nByte;                      /* Bytes of space to allocate */
-  int i;                          /* Counter variable */
-
-  assert( *ppRec==0 );
-  if( pIdx->nSample==0 ) return SQLITE_OK;
-
-  nByte = sizeof(Mem) * pIdx->nColumn + sizeof(UnpackedRecord);
-  *ppRec = pRec = (UnpackedRecord*)sqlite3DbMallocZero(db, nByte);
-  if( !pRec ) return SQLITE_NOMEM;
-  pRec->pKeyInfo = sqlite3IndexKeyinfo(pParse, pIdx);
-  if( !pRec->pKeyInfo ){
-    sqlite3DbFree(db, pRec);
-    *ppRec = 0;
-    return SQLITE_NOMEM;
-  }
-  pRec->pKeyInfo->enc = ENC(pParse->db);
-  pRec->flags = UNPACKED_PREFIX_MATCH;
-  pRec->aMem = (Mem *)&pRec[1];
-  for(i=0; i<pIdx->nColumn; i++){
-    pRec->aMem[i].flags = MEM_Null;
-    pRec->aMem[i].type = SQLITE_NULL;
-    pRec->aMem[i].db = db;
-  }
-
-  return SQLITE_OK;
-}
 #endif /* ifdef SQLITE_ENABLE_STAT4 */
 
 /*

src/where.c

@@ -2525,8 +2525,7 @@ static int whereRangeScanEst(
       int bOk;                    /* True if value is extracted from pExpr */
       Expr *pExpr = pLower->pExpr->pRight;
       assert( (pLower->eOperator & (WO_GT|WO_GE))!=0 );
-      rc = sqlite3Stat4ProbeSetValue(pParse, pRec, pExpr, aff, nEq, &bOk);
+      rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq, &bOk);
-      pRec->nField = nEq+1;
       if( rc==SQLITE_OK && bOk
        && whereKeyStats(pParse, p, pRec, 0, a)==SQLITE_OK
       ){
@@ -2538,8 +2537,7 @@ static int whereRangeScanEst(
       int bOk;                    /* True if value is extracted from pExpr */
       Expr *pExpr = pUpper->pExpr->pRight;
       assert( (pUpper->eOperator & (WO_LT|WO_LE))!=0 );
-      rc = sqlite3Stat4ProbeSetValue(pParse, pRec, pExpr, aff, nEq, &bOk);
+      rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq, &bOk);
-      pRec->nField = nEq+1;
       if( rc==SQLITE_OK && bOk
        && whereKeyStats(pParse, p, pRec, 1, a)==SQLITE_OK
       ){
@@ -2547,6 +2545,7 @@ static int whereRangeScanEst(
         if( (pUpper->eOperator & WO_LE)!=0 ) iUpper += a[1];
       }
     }
+    pBuilder->pRec = pRec;
     if( rc==SQLITE_OK ){
       WhereCost iBase = whereCost(p->aiRowEst[0]);
       if( iUpper>iLower ){
@@ -2629,12 +2628,11 @@ static int whereEqualScanEst(
   }
 
   aff = p->pTable->aCol[p->aiColumn[0]].affinity;
-  rc = sqlite3Stat4ProbeSetValue(pParse, pRec, pExpr, aff, nEq-1, &bOk);
+  rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq-1, &bOk);
+  pBuilder->pRec = pRec;
   if( rc!=SQLITE_OK ) return rc;
   if( bOk==0 ) return SQLITE_NOTFOUND;
-
   pBuilder->nRecValid = nEq;
-  pRec->nField = nEq;
 
   rc = whereKeyStats(pParse, p, pRec, 0, a);
   if( rc==SQLITE_OK ){
@@ -4575,18 +4573,11 @@ static int whereLoopAddBtree(
       }
     }
 
+    rc = whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, 0);
 #ifdef SQLITE_ENABLE_STAT4
-    assert( pBuilder->pRec==0 );
+    sqlite3Stat4ProbeFree(pBuilder->pRec);
-    rc = sqlite3Stat4ProbeNew(pWInfo->pParse, pProbe, &pBuilder->pRec);
+    pBuilder->nRecValid = 0;
-    if( rc==SQLITE_OK ){
+    pBuilder->pRec = 0;
-#endif
-      rc = whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, 0);
-#ifdef SQLITE_ENABLE_STAT4
-      sqlite3Stat4ProbeFree(pBuilder->pRec);
-      pBuilder->nRecValid = 0;
-      pBuilder->pRec = 0;
-    }
-    assert( pBuilder->pRec==0 );
 #endif
 
     /* If there was an INDEXED BY clause, then only that one index is