From 02fa469619720661e02b351d78b001d1fbc3451a Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 17 Aug 2009 17:06:58 +0000 Subject: [PATCH] First version of sqlite_stat2 (schema forces exactly 10 samples). FossilOrigin-Name: dd96bda2a85c1d94fb4a0bf5f27e2977f7f7e42e --- manifest | 42 ++++----- manifest.uuid | 2 +- src/analyze.c | 229 ++++++++++++++++++++++++++++++++++++------------ src/build.c | 1 + src/expr.c | 1 + src/sqliteInt.h | 19 ++++ src/test1.c | 22 +++-- src/utf.c | 24 +++++ src/vdbe.c | 43 +++++++++ src/vdbemem.c | 3 + src/where.c | 180 +++++++++++++++++++++++++++++++++++-- test/auth.test | 2 +- 12 files changed, 469 insertions(+), 99 deletions(-) diff --git a/manifest b/manifest index 63dec38152..ec6bc382cf 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,5 @@ ------BEGIN PGP SIGNED MESSAGE----- -Hash: SHA1 - -C Update\sthe\samalgamation\sbuilder\sso\sthat\sit\savoids\sputting\sredundant\nSQLITE_API\smacros\son\sdeclarations. -D 2009-08-14T18:18:04 +C First\sversion\sof\ssqlite_stat2\s(schema\sforces\sexactly\s10\ssamples). +D 2009-08-17T17:06:59 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in 0f7761c5d1c62ae7a841e3393ffaff1fa0f5c00a F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -103,7 +100,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 6be1ad09113570e1fc8dcaff84c9b0b337db5ffc F sqlite3.pc.in ae6f59a76e862f5c561eb32a380228a02afc3cad F src/alter.c 8b42cace4f8e312de596807ba2685179da64fec4 -F src/analyze.c e239496cfb5394ac8867f1c112905ddab8d01cd9 +F src/analyze.c c3f1ea347d5a2c90aec66a510e4c7b29d79fbca2 F src/attach.c 13995348fc5a26cdd136a50806faf292aabc173f F src/auth.c 802a9439dfa0b8c208b10055cba400e82ef18025 F src/backup.c 6f1c2d9862c8a3feb7739dfcca02c1f5352e37f3 @@ -112,12 +109,12 @@ F src/btmutex.c 0f43a75bb5b8147b386e8e1c3e71ba734e3863b7 F src/btree.c 49212ddaee8d7d12b4f1e17b9de62f7ea91ca59d F src/btree.h 577448a890c2ab9b21e6ab74f073526184bceebe F src/btreeInt.h 1c86297e69380f6577e7ae67452597dd8d5c2705 -F src/build.c a15de7c5d020a778b641fca0b2510126843f4b30 +F src/build.c 09389ab5d5a5997dbefa8f15fc4755f6c3243f4c F src/callback.c cb68b21b0d4ae7d11ae0e487933bce3323784dcf F src/complete.c 5ad5c6cd4548211867c204c41a126d73a9fbcea0 F src/date.c ab5f7137656652a48434d64f96bdcdc823bb23b3 F src/delete.c dcf07632d8ca3d4086df8b65ea907a47278e6382 -F src/expr.c d069ba1e060f296ea4f18fb85198fafefd00b22f +F src/expr.c ea04de0bf495eb899ba0c8c7af6561afb1dfac2d F src/fault.c dc88c821842157460750d2d61a8a8b4197d047ff F src/func.c e536218d193b8d326aab91120bc4c6f28aa2b606 F src/global.c 448419c44ce0701104c2121b0e06919b44514c0c @@ -166,12 +163,12 @@ F src/select.c 67b0778c9585905c8aa75aaa469e76ef3c1d315a F src/shell.c db2643650b9268df89a4bedca3f1c6d9e786f1bb F src/sqlite.h.in a6850e9034df1336e8139c4d6964d7d2f0f52337 F src/sqlite3ext.h 1db7d63ab5de4b3e6b83dd03d1a4e64fef6d2a17 -F src/sqliteInt.h 6a90791138ba3447572d184d0798c24f3cbbec98 +F src/sqliteInt.h 21722d546c8a93bf079564d49e628d5e66d3244a F src/sqliteLimit.h ffe93f5a0c4e7bd13e70cd7bf84cfb5c3465f45d F src/status.c 237b193efae0cf6ac3f0817a208de6c6c6ef6d76 F src/table.c cc86ad3d6ad54df7c63a3e807b5783c90411a08d F src/tclsqlite.c e18e5013dc6bca9f25e6022fbe17ba3ccb821f95 -F src/test1.c 0e882812c94cf35fce30fc25fbf952a33a86d70b +F src/test1.c eacb3456a9419191f42a0f601e12ca8a424a6de1 F src/test2.c 0de743ec8890ca4f09e0bce5d6d5a681f5957fec F src/test3.c 2445c2beb5e7a0c91fd8136dc1339ec369a24898 F src/test4.c f79ab52d27ff49b784b631a42e2ccd52cfd5c84c @@ -204,19 +201,19 @@ F src/test_wsd.c 3ae5101de6cbfda2720152ab659ea84079719241 F src/tokenize.c af8a56e6a50c5042fc305bfa796275e9bf26ff2b F src/trigger.c 9bc5278d509d81ff0f9b52f0ce7239563d188e32 F src/update.c 4da327f706c0d0dfedf4d92154b1b5688bdea0ac -F src/utf.c 9541d28f40441812c0b40f00334372a0542c00ff +F src/utf.c 9b022ac1c1f306733d57daa0df0b8beb7c17e95e F src/util.c c2416f60ae704a8c4990e4909aa810f90cbffa07 F src/vacuum.c 3fe0eebea6d2311c1c2ab2962887d11f7a4dcfb0 -F src/vdbe.c 0ce57f8211899b59d1d6f1642f79e75fc212d6d0 +F src/vdbe.c f2c07c6440826f69fc6d39083ac7fe5ba98fe3be F src/vdbe.h 457b6c70f02885cec1f5225b5e6441d067b55d3f F src/vdbeInt.h 831c254a6eef237ef4664c8381a0137586567007 F src/vdbeapi.c 0ab8ada7260b32031ca97f338caecf0812460624 F src/vdbeaux.c 4956536a636468fd07284028c39aab65ea99777e F src/vdbeblob.c a3f3e0e877fc64ea50165eec2855f5ada4477611 -F src/vdbemem.c ff40efaa2772e7aa66cf7501bf4142fd1a44bf51 +F src/vdbemem.c afd6ce02945e659f65642f290a37ccf4a88c4dcb F src/vtab.c aedd76e8670d5a5379f93804398d3ba960125547 F src/walker.c 1edca756275f158b80f20eb6f104c8d3fcc96a04 -F src/where.c 7573120c1f2fe6d4c246f138f1e30fbcda3db241 +F src/where.c 33a3aa8bef9594002300b4bc9aa2a7b37c71345c F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/alias.test 4529fbc152f190268a15f9384a5651bbbabc9d87 F test/all.test 14165b3e32715b700b5f0cbf8f6e3833dda0be45 @@ -235,7 +232,7 @@ F test/attach.test 1d1be27b9e4c654f9bb14d011a4a87753c0b197a F test/attach2.test a295d2d7061adcee5884ef4a93c7c96a82765437 F test/attach3.test 7b92dc8e40c1ebca9732ca6f2d3fefbd46f196df F test/attachmalloc.test cf8cf17d183de357b1147a9baacbdfc85b940b61 -F test/auth.test b2813abf4ae55f179fbd6db486ed8a6599de0b73 +F test/auth.test 393be593c72bc452cd2fe6e026682752aa258559 F test/auth2.test ee3ba272e2b975e913afc9b041ee75706e190005 F test/auth3.test a4755e6a2a2fea547ffe63c874eb569e60a28eb5 F test/autoinc.test 71bc5183c93ed5e2b8b3a71c218d777b55e4fffc @@ -746,14 +743,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl 672f81d693a03f80f5ae60bfefacd8a349e76746 -P 9cbe3654055a78c09ea1ecd5dc599bcd888b57e3 -R 9142cd5088584d6322dd160e9df9a386 -U drh -Z 2a7604fcf5cc328cdfa6d300890f778c ------BEGIN PGP SIGNATURE----- -Version: GnuPG v1.4.6 (GNU/Linux) - -iD8DBQFKhapgoxKgR168RlERAkH2AJoC/gxQtUL74VspfOFK2+qp6iCVFQCfRUDA -GwXZvHleDts/BZKq+4hsve0= -=Wsj5 ------END PGP SIGNATURE----- +P 0d5b058717858c9cda8ca120a3d814453a94a0e6 +R 942eb5447bdff248ec9f0cbf9f54bb64 +U dan +Z d16a7c84025a55a7e2426c9a05cd868d diff --git a/manifest.uuid b/manifest.uuid index 82fd998483..5099e8f445 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0d5b058717858c9cda8ca120a3d814453a94a0e6 \ No newline at end of file +dd96bda2a85c1d94fb4a0bf5f27e2977f7f7e42e \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index fd7803b783..e8c7c37171 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -17,8 +17,9 @@ #include "sqliteInt.h" /* -** This routine generates code that opens the sqlite_stat1 table on cursor -** iStatCur. +** This routine generates code that opens the sqlite_stat1 table for +** writing with cursor iStatCur. The sqlite_stat2 table is opened +** for writing using cursor (iStatCur+1). ** ** If the sqlite_stat1 tables does not previously exist, it is created. ** If it does previously exist, all entires associated with table zWhere @@ -30,53 +31,55 @@ static void openStatTable( int iStatCur, /* Open the sqlite_stat1 table on this cursor */ const char *zWhere /* Delete entries associated with this table */ ){ + const char *aName[] = { "sqlite_stat1", "sqlite_stat2" }; + const char *aCols[] = { "tbl,idx,stat", "tbl,idx," SQLITE_INDEX_SAMPLE_COLS }; + int aRoot[] = {0, 0}; + int aCreateTbl[] = {0, 0}; + + int i; sqlite3 *db = pParse->db; Db *pDb; - int iRootPage; - u8 createStat1 = 0; - Table *pStat; Vdbe *v = sqlite3GetVdbe(pParse); - if( v==0 ) return; assert( sqlite3BtreeHoldsAllMutexes(db) ); assert( sqlite3VdbeDb(v)==db ); pDb = &db->aDb[iDb]; - if( (pStat = sqlite3FindTable(db, "sqlite_stat1", pDb->zName))==0 ){ - /* The sqlite_stat1 tables does not exist. Create it. - ** Note that a side-effect of the CREATE TABLE statement is to leave - ** the rootpage of the new table in register pParse->regRoot. This is - ** important because the OpenWrite opcode below will be needing it. */ - sqlite3NestedParse(pParse, - "CREATE TABLE %Q.sqlite_stat1(tbl,idx,stat)", - pDb->zName - ); - iRootPage = pParse->regRoot; - createStat1 = 1; /* Cause rootpage to be taken from top of stack */ - }else if( zWhere ){ - /* The sqlite_stat1 table exists. Delete all entries associated with - ** the table zWhere. */ - sqlite3NestedParse(pParse, - "DELETE FROM %Q.sqlite_stat1 WHERE tbl=%Q", - pDb->zName, zWhere - ); - iRootPage = pStat->tnum; - }else{ - /* The sqlite_stat1 table already exists. Delete all rows. */ - iRootPage = pStat->tnum; - sqlite3VdbeAddOp2(v, OP_Clear, pStat->tnum, iDb); + + for(i=0; izName))==0 ){ + /* The sqlite_stat[12] table does not exist. Create it. Note that a + ** side-effect of the CREATE TABLE statement is to leave the rootpage + ** of the new table in register pParse->regRoot. This is important + ** because the OpenWrite opcode below will be needing it. */ + sqlite3NestedParse(pParse, + "CREATE TABLE %Q.%s(%s)", pDb->zName, aName[i], aCols[i] + ); + aRoot[i] = pParse->regRoot; + aCreateTbl[i] = 1; + }else{ + /* The table already exists. If zWhere is not NULL, delete all entries + ** associated with the table zWhere. If zWhere is NULL, delete the + ** entire contents of the table. */ + aRoot[i] = pStat->tnum; + sqlite3TableLock(pParse, iDb, aRoot[i], 1, aName[i]); + if( zWhere ){ + sqlite3NestedParse(pParse, + "DELETE FROM %Q.%s WHERE tbl=%Q", pDb->zName, aName[i], zWhere + ); + }else{ + /* The sqlite_stat[12] table already exists. Delete all rows. */ + sqlite3VdbeAddOp2(v, OP_Clear, aRoot[i], iDb); + } + } } - /* Open the sqlite_stat1 table for writing. Unless it was created - ** by this vdbe program, lock it for writing at the shared-cache level. - ** If this vdbe did create the sqlite_stat1 table, then it must have - ** already obtained a schema-lock, making the write-lock redundant. - */ - if( !createStat1 ){ - sqlite3TableLock(pParse, iDb, iRootPage, 1, "sqlite_stat1"); + /* Open the sqlite_stat[12] tables for writing. */ + for(i=0; itnum, 0, pTab->zName); + iMem += 3; iIdxCur = pParse->nTab++; for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ KeyInfo *pKey = sqlite3IndexKeyinfo(pParse, pIdx); @@ -126,6 +130,7 @@ static void analyzeOneTable( int regCol; /* Content of a column from the table being analyzed */ int regRowid; /* Rowid for the inserted record */ int regF2; + int regStat2; /* Open a cursor to the index to be analyzed */ @@ -134,22 +139,35 @@ static void analyzeOneTable( sqlite3VdbeAddOp4(v, OP_OpenRead, iIdxCur, pIdx->tnum, iDb, (char *)pKey, P4_KEYINFO_HANDOFF); VdbeComment((v, "%s", pIdx->zName)); - regFields = iMem+nCol*2; + regStat2 = iMem+nCol*2+1; + regFields = regStat2+2+SQLITE_INDEX_SAMPLES; regTemp = regRowid = regCol = regFields+3; regRec = regCol+1; if( regRec>pParse->nMem ){ pParse->nMem = regRec; } - /* Memory cells are used as follows: + /* Fill in the register with the total number of rows. */ + if( pTab->pIndex==pIdx ){ + sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, iMem-3); + } + sqlite3VdbeAddOp2(v, OP_Integer, 0, iMem-2); + sqlite3VdbeAddOp2(v, OP_Integer, 1, iMem-1); + + /* Memory cells are used as follows. All memory cell addresses are + ** offset by iMem. That is, cell 0 below is actually cell iMem, cell + ** 1 is cell 1+iMem, etc. ** - ** mem[iMem]: The total number of rows in the table. - ** mem[iMem+1]: Number of distinct values in column 1 - ** ... - ** mem[iMem+nCol]: Number of distinct values in column N - ** mem[iMem+nCol+1] Last observed value of column 1 - ** ... - ** mem[iMem+nCol+nCol]: Last observed value of column N + ** 0: The total number of rows in the table. + ** + ** 1..nCol: Number of distinct entries in index considering the + ** left-most N columns, where N is the same as the + ** memory cell number. + ** + ** nCol+1..2*nCol: Previous value of indexed columns, from left to + ** right. + ** + ** 2*nCol+1..2*nCol+10: 10 evenly spaced samples. ** ** Cells iMem through iMem+nCol are initialized to 0. The others ** are initialized to NULL. @@ -161,29 +179,35 @@ static void analyzeOneTable( sqlite3VdbeAddOp2(v, OP_Null, 0, iMem+nCol+i+1); } - /* Do the analysis. - */ + /* Start the analysis loop. This loop runs through all the entries inof + ** the index b-tree. */ endOfLoop = sqlite3VdbeMakeLabel(v); sqlite3VdbeAddOp2(v, OP_Rewind, iIdxCur, endOfLoop); topOfLoop = sqlite3VdbeCurrentAddr(v); sqlite3VdbeAddOp2(v, OP_AddImm, iMem, 1); + for(i=0; izName, 0); + sqlite3VdbeAddOp4(v, OP_String8, 0, regStat2+1, 0, pIdx->zName, 0); + sqlite3VdbeAddOp4(v, OP_MakeRecord, regStat2, SQLITE_INDEX_SAMPLES+2, + regRec, "aabbbbbbbbbb", 0 + ); + sqlite3VdbeAddOp2(v, OP_NewRowid, iStatCur+1, regRowid); + sqlite3VdbeAddOp3(v, OP_Insert, iStatCur+1, regRec, regRowid); + sqlite3VdbeJumpHere(v, addr); } } @@ -245,7 +279,8 @@ static void analyzeDatabase(Parse *pParse, int iDb){ int iMem; sqlite3BeginWriteOperation(pParse, 0, iDb); - iStatCur = pParse->nTab++; + iStatCur = pParse->nTab; + pParse->nTab += 2; openStatTable(pParse, iDb, iStatCur, 0); iMem = pParse->nMem+1; for(k=sqliteHashFirst(&pSchema->tblHash); k; k=sqliteHashNext(k)){ @@ -267,7 +302,8 @@ static void analyzeTable(Parse *pParse, Table *pTab){ assert( sqlite3BtreeHoldsAllMutexes(pParse->db) ); iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); sqlite3BeginWriteOperation(pParse, 0, iDb); - iStatCur = pParse->nTab++; + iStatCur = pParse->nTab; + pParse->nTab += 2; openStatTable(pParse, iDb, iStatCur, pTab->zName); analyzeOneTable(pParse, pTab, iStatCur, pParse->nMem+1); loadAnalysis(pParse, iDb); @@ -387,7 +423,8 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ } /* -** Load the content of the sqlite_stat1 table into the index hash tables. +** Load the content of the sqlite_stat1 and sqlite_stat2 tables into the +** index hash tables. */ int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ analysisInfo sInfo; @@ -412,7 +449,6 @@ int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ return SQLITE_ERROR; } - /* Load new statistics out of the sqlite_stat1 table */ zSql = sqlite3MPrintf(db, "SELECT idx, stat FROM %Q.sqlite_stat1", sInfo.zDatabase); @@ -423,8 +459,87 @@ int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ rc = sqlite3_exec(db, zSql, analysisLoader, &sInfo, 0); (void)sqlite3SafetyOn(db); sqlite3DbFree(db, zSql); - if( rc==SQLITE_NOMEM ) db->mallocFailed = 1; } + + /* Load the statistics from the sqlite_stat2 table */ + if( rc==SQLITE_OK ){ + zSql = sqlite3MPrintf(db, + "SELECT idx," SQLITE_INDEX_SAMPLE_COLS " FROM %Q.sqlite_stat2", + sInfo.zDatabase + ); + if( zSql ){ + sqlite3_stmt *pStmt = 0; + (void)sqlite3SafetyOff(db); + rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); + if( rc==SQLITE_OK ){ + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + char *zIndex = (char *)sqlite3_column_text(pStmt, 0); + Index *pIdx; + pIdx = sqlite3FindIndex(db, zIndex, sInfo.zDatabase); + if( pIdx ){ + char *pSpace; + IndexSample *pSample; + int iCol; + int nAlloc = SQLITE_INDEX_SAMPLES * sizeof(IndexSample); + for(iCol=1; iCol<=SQLITE_INDEX_SAMPLES; iCol++){ + int eType = sqlite3_column_type(pStmt, iCol); + if( eType==SQLITE_TEXT || eType==SQLITE_BLOB ){ + nAlloc += sqlite3_column_bytes(pStmt, iCol); + } + } + pSample = sqlite3DbMallocRaw(db, nAlloc); + if( !pSample ){ + rc = SQLITE_NOMEM; + break; + } + sqlite3DbFree(db, pIdx->aSample); + pIdx->aSample = pSample; + pSpace = (char *)&pSample[SQLITE_INDEX_SAMPLES]; + for(iCol=1; iCol<=SQLITE_INDEX_SAMPLES; iCol++){ + int eType = sqlite3_column_type(pStmt, iCol); + pSample[iCol-1].eType = eType; + switch( eType ){ + case SQLITE_BLOB: + case SQLITE_TEXT: { + const char *z = (const char *)( + (eType==SQLITE_BLOB) ? + sqlite3_column_blob(pStmt, iCol): + sqlite3_column_text(pStmt, iCol) + ); + int n = sqlite3_column_bytes(pStmt, iCol); + if( n>24 ){ + n = 24; + } + pSample[iCol-1].nByte = n; + pSample[iCol-1].u.z = pSpace; + memcpy(pSpace, z, n); + pSpace += n; + break; + } + case SQLITE_INTEGER: + case SQLITE_FLOAT: + pSample[iCol-1].u.r = sqlite3_column_double(pStmt, iCol); + break; + case SQLITE_NULL: + break; + } + } + } + } + if( rc==SQLITE_NOMEM ){ + sqlite3_finalize(pStmt); + }else{ + rc = sqlite3_finalize(pStmt); + } + } + (void)sqlite3SafetyOn(db); + sqlite3DbFree(db, zSql); + }else{ + rc = SQLITE_NOMEM; + } + } + + if( rc==SQLITE_NOMEM ) db->mallocFailed = 1; return rc; } diff --git a/src/build.c b/src/build.c index 7136cbc1b3..8cf9ec7530 100644 --- a/src/build.c +++ b/src/build.c @@ -343,6 +343,7 @@ Index *sqlite3FindIndex(sqlite3 *db, const char *zName, const char *zDb){ static void freeIndex(Index *p){ sqlite3 *db = p->pTable->dbMem; /* testcase( db==0 ); */ + sqlite3DbFree(db, p->aSample); sqlite3DbFree(db, p->zColAff); sqlite3DbFree(db, p); } diff --git a/src/expr.c b/src/expr.c index 9460163bf2..05b8ecd660 100644 --- a/src/expr.c +++ b/src/expr.c @@ -2835,6 +2835,7 @@ static int evalConstExpr(Walker *pWalker, Expr *pExpr){ int r2; r2 = sqlite3ExprCodeTarget(pParse, pExpr, r1); if( NEVER(r1!=r2) ) sqlite3ReleaseTempReg(pParse, r1); + pExpr->iColumn = pExpr->op; pExpr->op = TK_REGISTER; pExpr->iTable = r2; return WRC_Prune; diff --git a/src/sqliteInt.h b/src/sqliteInt.h index a737a12c73..3912715e52 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -77,6 +77,9 @@ #include #endif +#define SQLITE_INDEX_SAMPLES 10 +#define SQLITE_INDEX_SAMPLE_COLS "s1,s2,s3,s4,s5,s6,s7,s8,s9,s10" + /* ** This macro is used to "hide" some ugliness in casting an int ** value to a ptr value under the MSVC 64-bit compiler. Casting @@ -595,6 +598,7 @@ typedef struct FuncDef FuncDef; typedef struct FuncDefHash FuncDefHash; typedef struct IdList IdList; typedef struct Index Index; +typedef struct IndexSample IndexSample; typedef struct KeyClass KeyClass; typedef struct KeyInfo KeyInfo; typedef struct Lookaside Lookaside; @@ -1410,6 +1414,20 @@ struct Index { Schema *pSchema; /* Schema containing this index */ u8 *aSortOrder; /* Array of size Index.nColumn. True==DESC, False==ASC */ char **azColl; /* Array of collation sequence names for index */ + IndexSample *aSample; /* Array of SQLITE_INDEX_SAMPLES samples */ +}; + +/* +** Each sample stored in the sqlite_stat2 table is represented in memory +** using a structure of this type. +*/ +struct IndexSample { + union { + char *z; /* Value if eType is SQLITE_TEXT or SQLITE_BLOB */ + double r; /* Value if eType is SQLITE_FLOAT or SQLITE_INTEGER */ + } u; + u8 eType; /* SQLITE_NULL, SQLITE_INTEGER ... etc. */ + u8 nByte; /* Size in byte of text or blob. */ }; /* @@ -2781,6 +2799,7 @@ void sqlite3ValueSetStr(sqlite3_value*, int, const void *,u8, void sqlite3ValueFree(sqlite3_value*); sqlite3_value *sqlite3ValueNew(sqlite3 *); char *sqlite3Utf16to8(sqlite3 *, const void*, int); +char *sqlite3Utf8to16(sqlite3 *, int, char *, int, int *); int sqlite3ValueFromExpr(sqlite3 *, Expr *, u8, u8, sqlite3_value **); void sqlite3ValueApplyAffinity(sqlite3_value *, u8, u8); #ifndef SQLITE_AMALGAMATION diff --git a/src/test1.c b/src/test1.c index 5976cab1ac..95e00a9de4 100644 --- a/src/test1.c +++ b/src/test1.c @@ -2309,16 +2309,20 @@ static int test_collate_func( assert(0); } + sqlite3BeginBenignMalloc(); pVal = sqlite3ValueNew(0); - sqlite3ValueSetStr(pVal, nA, zA, encin, SQLITE_STATIC); - n = sqlite3_value_bytes(pVal); - Tcl_ListObjAppendElement(i,pX, - Tcl_NewStringObj((char*)sqlite3_value_text(pVal),n)); - sqlite3ValueSetStr(pVal, nB, zB, encin, SQLITE_STATIC); - n = sqlite3_value_bytes(pVal); - Tcl_ListObjAppendElement(i,pX, - Tcl_NewStringObj((char*)sqlite3_value_text(pVal),n)); - sqlite3ValueFree(pVal); + if( pVal ){ + sqlite3ValueSetStr(pVal, nA, zA, encin, SQLITE_STATIC); + n = sqlite3_value_bytes(pVal); + Tcl_ListObjAppendElement(i,pX, + Tcl_NewStringObj((char*)sqlite3_value_text(pVal),n)); + sqlite3ValueSetStr(pVal, nB, zB, encin, SQLITE_STATIC); + n = sqlite3_value_bytes(pVal); + Tcl_ListObjAppendElement(i,pX, + Tcl_NewStringObj((char*)sqlite3_value_text(pVal),n)); + sqlite3ValueFree(pVal); + } + sqlite3EndBenignMalloc(); Tcl_EvalObjEx(i, pX, 0); Tcl_DecrRefCount(pX); diff --git a/src/utf.c b/src/utf.c index ceb0ab641d..89eb83f79a 100644 --- a/src/utf.c +++ b/src/utf.c @@ -454,6 +454,30 @@ char *sqlite3Utf16to8(sqlite3 *db, const void *z, int nByte){ return (m.flags & MEM_Dyn)!=0 ? m.z : sqlite3DbStrDup(db, m.z); } +/* +** Convert a UTF-8 string to the UTF-16 encoding specified by parameter +** enc. A pointer to the new string is returned, and the value of *pnOut +** is set to the length of the returned string in bytes. The call should +** arrange to call sqlite3DbFree() on the returned pointer when it is +** no longer required. +** +** If a malloc failure occurs, NULL is returned and the db.mallocFailed +** flag set. +*/ +char *sqlite3Utf8to16(sqlite3 *db, int enc, char *z, int n, int *pnOut){ + Mem m; + memset(&m, 0, sizeof(m)); + m.db = db; + sqlite3VdbeMemSetStr(&m, z, n, SQLITE_UTF8, SQLITE_STATIC); + if( sqlite3VdbeMemTranslate(&m, enc) ){ + assert( db->mallocFailed ); + return 0; + } + assert( m.z==m.zMalloc ); + *pnOut = m.n; + return m.z; +} + /* ** pZ is a UTF-16 encoded unicode string at least nChar characters long. ** Return the number of bytes in the first nChar unicode characters diff --git a/src/vdbe.c b/src/vdbe.c index e27fbeb48e..9bae940846 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -4974,6 +4974,49 @@ case OP_Expire: { break; } + +/* Opcode: Sample P1 P2 P3 * * +** +** Register P1 contains the total number of rows in the index being +** analyzed. Register P1+1 contains an integer between 0 and 9, the +** index of the next sample required. Register P1+2 contains an index +** between 1 and *P1, the number of the next sample required. Register +** P1+3 contains the current row index. +** +** If the integer in register P1+3 is the same as the integer in register +** P1+1, then the following takes place: +** +** (a) the contents of register P1+1 is incremented. +** +** (b) the contents of the register identified by parameter P2 is +** copied to register number (P3 + X), where X is the newly +** incremented value of register P1+1. +** +** (c) register P1+2 is set to the index of the next sample required. +*/ +case OP_Sample: { + int p1 = pOp->p1; + i64 iReq = p->aMem[p1+2].u.i; + i64 iRow = p->aMem[p1+3].u.i; + + while( iReq==iRow ){ + i64 nRow = p->aMem[p1].u.i; + int iSample = ++p->aMem[p1+1].u.i; + Mem *pReg = &p->aMem[pOp->p3 + iSample - 1]; + + assert( pReg<&p->aMem[p->nMem] ); + sqlite3VdbeMemShallowCopy(pReg, &p->aMem[pOp->p2], MEM_Ephem); + Deephemeralize(pReg); + if( iSample==SQLITE_INDEX_SAMPLES ){ + iReq = 0; + }else{ + iReq = iRow + (nRow-iRow)/(SQLITE_INDEX_SAMPLES - iSample); + p->aMem[p1+2].u.i = iReq; + } + } + break; +} + #ifndef SQLITE_OMIT_SHARED_CACHE /* Opcode: TableLock P1 P2 P3 P4 * ** diff --git a/src/vdbemem.c b/src/vdbemem.c index ef6ed8f9de..1a7108f8f6 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -999,6 +999,9 @@ int sqlite3ValueFromExpr( return SQLITE_OK; } op = pExpr->op; + if( op==TK_REGISTER ){ + op = pExpr->iColumn; + } if( op==TK_STRING || op==TK_FLOAT || op==TK_INTEGER ){ pVal = sqlite3ValueNew(db); diff --git a/src/where.c b/src/where.c index 50a08c952a..d96578f540 100644 --- a/src/where.c +++ b/src/where.c @@ -1890,6 +1890,177 @@ static void bestVirtualIndex( } #endif /* SQLITE_OMIT_VIRTUALTABLE */ +/* +** Argument pIdx is a pointer to an index structure that has an array of +** SQLITE_INDEX_SAMPLES evenly spaced samples of the first indexed column +** stored in Index.aSample. The domain of values stored in said column +** may be thought of as divided into (SQLITE_INDEX_SAMPLES+1) regions. +** Region 0 contains all values smaller than the first sample value. Region +** 1 contains values larger than or equal to the value of the first sample, +** but smaller than the value of the second. And so on. +** +** If successful, this function determines which of the regions value +** pVal lies in, sets *piRegion to the region index and returns SQLITE_OK. +** Or, if an OOM occurs while converting text values between encodings, +** SQLITE_NOMEM is returned. +*/ +static int whereRangeRegion( + Parse *pParse, /* Database connection */ + Index *pIdx, /* Index to consider domain of */ + sqlite3_value *pVal, /* Value to consider */ + int *piRegion /* OUT: Region of domain in which value lies */ +){ + if( pVal ){ + IndexSample *aSample = pIdx->aSample; + int i = 0; + int eType = sqlite3_value_type(pVal); + + if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){ + double r = sqlite3_value_double(pVal); + for(i=0; i=SQLITE_TEXT || aSample[i].u.r>r ) break; + } + }else if( eType==SQLITE_TEXT || eType==SQLITE_BLOB ){ + sqlite3 *db = pParse->db; + CollSeq *pColl; + const u8 *z; + int n; + if( eType==SQLITE_BLOB ){ + z = (const u8 *)sqlite3_value_blob(pVal); + pColl = db->pDfltColl; + assert( pColl->enc==SQLITE_UTF8 ); + }else{ + pColl = sqlite3FindCollSeq(db, SQLITE_UTF8, *pIdx->azColl, 0); + if( sqlite3CheckCollSeq(pParse, pColl) ){ + return SQLITE_ERROR; + } + z = (const u8 *)sqlite3ValueText(pVal, pColl->enc); + if( !z ){ + return SQLITE_NOMEM; + } + assert( z && pColl && pColl->xCmp ); + } + n = sqlite3ValueBytes(pVal, pColl->enc); + + for(i=0; ienc==SQLITE_UTF8 ){ + r = pColl->xCmp(pColl->pUser, aSample[i].nByte, aSample[i].u.z, n, z); + }else{ + int nSample; + char *zSample = sqlite3Utf8to16( + db, pColl->enc, aSample[i].u.z, aSample[i].nByte, &nSample + ); + if( !zSample ){ + assert( db->mallocFailed ); + return SQLITE_NOMEM; + } + r = pColl->xCmp(pColl->pUser, nSample, zSample, n, z); + sqlite3DbFree(db, zSample); + } + if( r>0 ) break; + } + } + + *piRegion = i; + } + return SQLITE_OK; +} + +/* +** This function is used to estimate the number of rows that will be visited +** by scanning an index for a range of values. The range may have an upper +** bound, a lower bound, or both. The WHERE clause terms that set the upper +** and lower bounds are represented by pLower and pUpper respectively. For +** example, assuming that index p is on t1(a): +** +** ... FROM t1 WHERE a > ? AND a < ? ... +** |_____| |_____| +** | | +** pLower pUpper +** +** If the upper or lower bound is not present, then NULL should be passed in +** place of a WhereTerm. +** +** The nEq parameter is passed the index of the index column subject to the +** range constraint. Or, equivalently, the number of equality constraints +** optimized by the proposed index scan. For example, assuming index p is +** on t1(a, b), and the SQL query is: +** +** ... FROM t1 WHERE a = ? AND b > ? AND b < ? ... +** +** then nEq should be passed the value 1 (as the range restricted column, +** b, is the second left-most column of the index). Or, if the query is: +** +** ... FROM t1 WHERE a > ? AND a < ? ... +** +** then nEq should be passed 0. +** +** The returned value is an integer between 1 and 9, inclusive. A return +** value of 1 indicates that the proposed range scan is expected to visit +** approximately 1/9 (11%) of the rows selected by the nEq equality constraints +** (if any). A return value of 9 indicates that it is expected that the +** range scan will visit 9/9 (100%) of the rows selected by the equality +** constraints. +*/ +static int whereRangeScanEst( + Parse *pParse, + Index *p, + int nEq, + WhereTerm *pLower, + WhereTerm *pUpper, + int *piEst /* OUT: Return value */ +){ + sqlite3 *db = pParse->db; + sqlite3_value *pLowerVal = 0; + sqlite3_value *pUpperVal = 0; + int rc = SQLITE_OK; + + if( nEq==0 && p->aSample ){ + int iEst; + int iUpper = SQLITE_INDEX_SAMPLES; + int iLower = 0; + u8 aff = p->pTable->aCol[0].affinity; + if( pLower ){ + Expr *pExpr = pLower->pExpr->pRight; + rc = sqlite3ValueFromExpr(db, pExpr, SQLITE_UTF8, aff, &pLowerVal); + if( !pLowerVal ) goto fallback; + } + if( pUpper ){ + Expr *pExpr = pUpper->pExpr->pRight; + rc = sqlite3ValueFromExpr(db, pExpr, SQLITE_UTF8, aff, &pUpperVal); + if( !pUpperVal ){ + sqlite3ValueFree(pLowerVal); + goto fallback; + } + } + + rc = whereRangeRegion(pParse, p, pUpperVal, &iUpper); + if( rc==SQLITE_OK ){ + rc = whereRangeRegion(pParse, p, pLowerVal, &iLower); + } + + iEst = iUpper - iLower; + if( iEst>=SQLITE_INDEX_SAMPLES ) iEst = SQLITE_INDEX_SAMPLES-1; + else if( iEst<1 ) iEst = 1; + + sqlite3ValueFree(pLowerVal); + sqlite3ValueFree(pUpperVal); + *piEst = iEst; + return rc; + } + +fallback: + assert( pLower || pUpper ); + *piEst = (SQLITE_INDEX_SAMPLES-1) / ((pLower&&pUpper)?9:3); + return rc; +} + + /* ** Find the query plan for accessing a particular table. Write the ** best query plan and its cost into the WhereCost object supplied as the @@ -2043,7 +2214,7 @@ static void bestBtreeIndex( int nEq; int bInEst = 0; int nInMul = 1; - int nBound = 1; + int nBound = 9; int bSort = 0; int bLookup = 0; @@ -2075,14 +2246,13 @@ static void bestBtreeIndex( if( findTerm(pWC, iCur, j, notReady, WO_LT|WO_LE|WO_GT|WO_GE, pIdx) ){ WhereTerm *pTop = findTerm(pWC, iCur, j, notReady, WO_LT|WO_LE, pIdx); WhereTerm *pBtm = findTerm(pWC, iCur, j, notReady, WO_GT|WO_GE, pIdx); + whereRangeScanEst(pParse, pProbe, nEq, pBtm, pTop, &nBound); if( pTop ){ wsFlags |= WHERE_TOP_LIMIT; - nBound *= 3; used |= pTop->prereqRight; } if( pBtm ){ wsFlags |= WHERE_BTM_LIMIT; - nBound *= 3; used |= pBtm->prereqRight; } wsFlags |= (WHERE_COLUMN_RANGE|WHERE_ROWID_RANGE); @@ -2152,8 +2322,8 @@ static void bestBtreeIndex( nInMul = nRow / aiRowEst[nEq]; } cost = nRow + nInMul*estLog(aiRowEst[0]); - nRow /= nBound; - cost /= nBound; + nRow = nRow * (double)nBound / 9.0; + cost = cost * (double)nBound / 9.0; if( bSort ){ cost += cost*estLog(cost); } diff --git a/test/auth.test b/test/auth.test index 7ac031d539..600a552755 100644 --- a/test/auth.test +++ b/test/auth.test @@ -2316,7 +2316,7 @@ ifcapable compound&&subquery { WHERE type='table' ORDER BY name } - } {sqlite_stat1 t1 t2 t3 t4} + } {sqlite_stat1 sqlite_stat2 t1 t2 t3 t4} } # Ticket #3944