database/sqlite
1
0
Fork 0
mirror of https://github.com/sqlite/sqlite.git synced 2025-08-10 01:02:56 +03:00
Code Activity

Consolidate memory allocations made while loading stat4 data in a way that may be more efficient on systems under load.

Browse Source 
FossilOrigin-Name: af65a902d10e50d827ff31f9ded7d05bc7ab0956a767366e1321b28fcb60b0bd
This commit is contained in:
dan
2025-01-29 17:26:44 +00:00
parent 2a8768ad66
commit bbda3d68da
5 changed files with 156 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
manifest
View File

@@ -1,5 +1,5 @@
C If\sSQLITE_ENABLE_WAL2NOCKSUM\sis\sdefined,\sthen\sSQLite\scalculates\sthe\sframe\schecksums\sused\sin\swal2\smode\sbased\son\sthe\sprevious\schecksum\sand\sthe\sframe\sheader\sonly,\snot\sthe\sframe\sbody\sonly.\sThis\srisks\scorruption\sfollowing\sa\sOS\scrash\sor\spower\sfailure,\sbut\salso\sspeeds\sup\swrites\sin\swal2\smode.
D 2025-01-29T15:11:07.802
C Consolidate\smemory\sallocations\smade\swhile\sloading\sstat4\sdata\sin\sa\sway\sthat\smay\sbe\smore\sefficient\son\ssystems\sunder\sload.
D 2025-01-29T17:26:44.487
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md e108e1e69ae8e8a59e93c455654b8ac9356a11720d3345df2a4743e9590fb20d
@@ -719,7 +719,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b
F sqlite3.1 acdff36db796e2d00225b911d3047d580cd136547298435426ce9d40347973cc
F sqlite3.pc.in 0977c03a4da7c4204bd60e784a0efb8d51a190448aba78a4e973fe7192bdaf03
F src/alter.c aa93e37e4a36a0525bbb2a2aeda20d2018f0aa995542c7dc658e031375e3f532
F src/analyze.c 9a8b67239d899ac12289db5db3f5bfe7f7a0ad1277f80f87ead1d048085876eb
F src/analyze.c c5573e4c5442bdcebd38692319fb4636c4b8c17de28be10e9ee9c2b57f7f8794
F src/attach.c f35bb8cc1fcdde8f6815a7ef09ae413bcac71821d530796800ba24b3c7da1e80
F src/auth.c 54ab9c6c5803b47c0d45b76ce27eff22a03b4b1f767c5945a3a4eb13aa4c78dc
F src/backup.c 5c97e8023aab1ce14a42387eb3ae00ba5a0644569e3476f38661fa6f824c3523
@@ -729,7 +729,7 @@ F src/btree.c 9145be06761b9947d621e575d3837f9cf265242a013c86b23f28ef4be66b3f3e
F src/btree.h df26089b055c4cffe243e5bc98edc729c4ad880bfeb8f229fd16248e4cec10ff
F src/btreeInt.h bb28bf05e6206befd5f5fd2ed3825fc6382979fa4a83bf50f1875a0d3404111b
F src/build.c 5a14f6593b9b0002f54d4215febceeaa1354c12f8eca53f8135d3fdf8a93f4d7
F src/callback.c db3a45e376deff6a16c0058163fe0ae2b73a2945f3f408ca32cf74960b28d490
F src/callback.c 43c8ca52b1ecbdec43522f121126fd4e3ee10bc9ca01cdd3ae207cfa419780b6
F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e
F src/ctime.c dbd2d51fa2874ad54c673cb71b1f5bbd0df1b8b47b34f31d105f42500680a89a
F src/date.c 89ce1ff20512a7fa5070ba6e7dd5c171148ca7d580955795bf97c79c2456144a
@@ -791,7 +791,7 @@ F src/shell.c.in beb370609906092a6810fcd9ea76737be2c91694445061c2eb05c4c0a3753de
F src/sqlite.h.in 278a2679a4e50ebe965c5f8e3ec78d52541aa5eeb2632b6eb0a0a395f63b1cea
F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54
F src/sqliteInt.h d711bbe8b255024336fb84b090fbc4a6e506f2768925da5426222594537b90c8
F src/sqliteInt.h 1fc598d06c6c7d09fbb68791f8e876ba5a72a42421f55c3f14e066f155bee1d2
F src/sqliteLimit.h 1bbdbf72bd0411d003267ffebc59a262f061df5653027a75627d03f48ca30523
F src/status.c cb11f8589a6912af2da3bb1ec509a94dd8ef27df4d4c1a97e0bcf2309ece972b
F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1
@@ -2242,9 +2242,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350
F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7
F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139
F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
P fab341c8295545739cdce8b71e38ead68cb80a6f836f7ec0540b387f17b6cbe2
Q +4d3706727005397c3c9006b9ad9a2645b09533b02735ea68974c4f2df4c2e853
R 474ce037ad8c9757d8c8e66f75356bf2
P dc74bd8915a9e1a915fb4ff3229a7b5e8f89486fe1df812a7738f6627d379648
R e6eba16885e8253a0c8420a72196b2b2
U dan
Z 363c756be7599a910bd16bdd6751a132
Z f498dbc7b0d6c6dc50ccede72f5ffcd3
# Remove this line to create a well-formed Fossil manifest.

2
manifest.uuid
View File

@@ -1 +1 @@
dc74bd8915a9e1a915fb4ff3229a7b5e8f89486fe1df812a7738f6627d379648
af65a902d10e50d827ff31f9ded7d05bc7ab0956a767366e1321b28fcb60b0bd

207
src/analyze.c
View File

@@ -256,6 +256,13 @@ static void openStatTable(
# define SQLITE_STAT4_SAMPLES 24
#endif
/*
** Assumed number of of samples when loading sqlite_stat4 data. It doesn't
** matter if there are more or fewer samples than this, but is more efficient
** if this estimate turns out to be true.
*/
#define SQLITE_STAT4_EST_SAMPLES SQLITE_STAT4_SAMPLES
/*
** Three SQL functions - stat_init(), stat_push(), and stat_get() -
** share an instance of the following structure to hold their state
@@ -1550,6 +1557,9 @@ static void decodeIntArray(
#endif
if( *z==' ' ) z++;
}
if( aOut ){
for(/* no-op */; i<nOut; i++){ aOut[i] = 0; }
}
#ifndef SQLITE_ENABLE_STAT4
assert( pIndex!=0 ); {
#else
@@ -1672,11 +1682,14 @@ void sqlite3DeleteIndexSamples(sqlite3 *db, Index *pIdx){
IndexSample *p = &pIdx->aSample[j];
sqlite3DbFree(db, p->p);
}
if( pIdx->nSampleAlloc!=SQLITE_STAT4_EST_SAMPLES ){
sqlite3DbFree(db, pIdx->aSample);
}
}
if( db->pnBytesFreed==0 ){
pIdx->nSample = 0;
pIdx->aSample = 0;
pIdx->nSampleAlloc = 0;
}
#else
UNUSED_PARAMETER(db);
@@ -1762,8 +1775,110 @@ static Index *findIndexOrPrimaryKey(
}
/*
** Load the content from either the sqlite_stat4
** into the relevant Index.aSample[] arrays.
** Grow the pIdx->aSample[] array. Return SQLITE_OK if successful, or
** SQLITE_NOMEM otherwise.
*/
static int growSampleArray(sqlite3 *db, Index *pIdx, int *piOff){
int nIdxCol = pIdx->nSampleCol;
int nNew = 0;
IndexSample *aNew = 0;
int nByte = 0;
tRowcnt *pSpace; /* Available allocated memory space */
u8 *pPtr; /* Available memory as a u8 for easier manipulation */
int i;
u64 t;
assert( pIdx->nSample==pIdx->nSampleAlloc );
nNew = SQLITE_STAT4_EST_SAMPLES;
if( pIdx->nSample ){
nNew = pIdx->nSample*2;
}
/* Set nByte to the required amount of space */
nByte = ROUND8(sizeof(IndexSample) * nNew);
nByte += sizeof(tRowcnt) * nIdxCol * 3 * nNew;
nByte += nIdxCol * sizeof(tRowcnt); /* Space for Index.aAvgEq[] */
if( nNew==SQLITE_STAT4_EST_SAMPLES ){
aNew = (IndexSample*)&((u8*)pIdx->pSchema->pStat4Space)[*piOff];
*piOff += nByte;
assert( *piOff<=sqlite3_msize(pIdx->pSchema->pStat4Space) );
}else{
aNew = (IndexSample*)sqlite3DbMallocRaw(db, nByte);
if( aNew==0 ) return SQLITE_NOMEM_BKPT;
}
pPtr = (u8*)aNew;
pPtr += ROUND8(nNew*sizeof(pIdx->aSample[0]));
pSpace = (tRowcnt*)pPtr;
pIdx->aAvgEq = pSpace; pSpace += nIdxCol;
assert( EIGHT_BYTE_ALIGNMENT( pSpace ) );
if( pIdx->nSample ){
/* Copy the contents of the anEq[], anLt[], anDLt[] arrays for all
** extant samples to the new location. */
int nByte = nIdxCol * 3 * sizeof(tRowcnt) * pIdx->nSample;
memcpy(pSpace, pIdx->aSample[0].anEq, nByte);
}
for(i=0; i<nNew; i++){
aNew[i].anEq = pSpace; pSpace += nIdxCol;
aNew[i].anLt = pSpace; pSpace += nIdxCol;
aNew[i].anDLt = pSpace; pSpace += nIdxCol;
if( i<pIdx->nSample ){
aNew[i].p = pIdx->aSample[i].p;
aNew[i].n = pIdx->aSample[i].n;
}
}
assert( ((u8*)pSpace)-nByte==(u8*)aNew );
if( pIdx->nSample!=SQLITE_STAT4_EST_SAMPLES ){
sqlite3DbFree(db, pIdx->aSample);
}
pIdx->aSample = aNew;
pIdx->nSampleAlloc = nNew;
return SQLITE_OK;
}
/*
** Allocate the space that will likely be required for the Index.aSample[]
** arrays populated by loading data from the sqlite_stat4 table. Return
** SQLITE_OK if successful, or SQLITE_NOMEM otherwise.
*/
static int stat4AllocSpace(sqlite3 *db, const char *zDb){
int iDb = sqlite3FindDbName(db, zDb);
Schema *pSchema = db->aDb[iDb].pSchema;
int nByte = 0;
HashElem *k;
assert( iDb>=0 );
assert( pSchema->pStat4Space==0 );
for(k=sqliteHashFirst(&pSchema->idxHash); k; k=sqliteHashNext(k)){
Index *pIdx = sqliteHashData(k);
int nIdxCol;
if( !HasRowid(pIdx->pTable) && IsPrimaryKeyIndex(pIdx) ){
nIdxCol = pIdx->nKeyCol;
}else{
nIdxCol = pIdx->nColumn;
}
nByte += ROUND8(sizeof(IndexSample) * SQLITE_STAT4_EST_SAMPLES);
nByte += sizeof(tRowcnt) * nIdxCol * 3 * SQLITE_STAT4_EST_SAMPLES;
nByte += nIdxCol * sizeof(tRowcnt); /* Space for Index.aAvgEq[] */
}
if( nByte>0 ){
pSchema->pStat4Space = sqlite3_malloc(nByte);
if( pSchema->pStat4Space==0 ){
return SQLITE_NOMEM_BKPT;
}
}
return SQLITE_OK;
}
/*
** Load the content from the sqlite_stat4 into the relevant Index.aSample[]
** arrays.
**
** Arguments zSql1 and zSql2 must point to SQL statements that return
** data equivalent to the following:
@@ -1784,69 +1899,14 @@ static int loadStatTbl(
char *zSql; /* Text of the SQL statement */
Index *pPrevIdx = 0; /* Previous index in the loop */
IndexSample *pSample; /* A slot in pIdx->aSample[] */
int iBlockOff = 0; /* Offset into Schema.pStat4Space */
assert( db->lookaside.bDisable );
zSql = sqlite3MPrintf(db, zSql1, zDb);
if( !zSql ){
return SQLITE_NOMEM_BKPT;
}
rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0);
sqlite3DbFree(db, zSql);
if( rc ) return rc;
while( sqlite3_step(pStmt)==SQLITE_ROW ){
int nIdxCol = 1; /* Number of columns in stat4 records */
char *zIndex; /* Index name */
Index *pIdx; /* Pointer to the index object */
int nSample; /* Number of samples */
i64 nByte; /* Bytes of space required */
i64 i; /* Bytes of space required */
tRowcnt *pSpace; /* Available allocated memory space */
u8 *pPtr; /* Available memory as a u8 for easier manipulation */
zIndex = (char *)sqlite3_column_text(pStmt, 0);
if( zIndex==0 ) continue;
nSample = sqlite3_column_int(pStmt, 1);
pIdx = findIndexOrPrimaryKey(db, zIndex, zDb);
assert( pIdx==0 || pIdx->nSample==0 );
if( pIdx==0 ) continue;
if( pIdx->aSample!=0 ){
/* The same index appears in sqlite_stat4 under multiple names */
continue;
}
assert( !HasRowid(pIdx->pTable) || pIdx->nColumn==pIdx->nKeyCol+1 );
if( !HasRowid(pIdx->pTable) && IsPrimaryKeyIndex(pIdx) ){
nIdxCol = pIdx->nKeyCol;
}else{
nIdxCol = pIdx->nColumn;
}
pIdx->nSampleCol = nIdxCol;
pIdx->mxSample = nSample;
nByte = ROUND8(sizeof(IndexSample) * nSample);
nByte += sizeof(tRowcnt) * nIdxCol * 3 * nSample;
nByte += nIdxCol * sizeof(tRowcnt); /* Space for Index.aAvgEq[] */
pIdx->aSample = sqlite3DbMallocZero(db, nByte);
if( pIdx->aSample==0 ){
sqlite3_finalize(pStmt);
return SQLITE_NOMEM_BKPT;
}
pPtr = (u8*)pIdx->aSample;
pPtr += ROUND8(nSample*sizeof(pIdx->aSample[0]));
pSpace = (tRowcnt*)pPtr;
assert( EIGHT_BYTE_ALIGNMENT( pSpace ) );
pIdx->aAvgEq = pSpace; pSpace += nIdxCol;
pIdx->pTable->tabFlags |= TF_HasStat4;
for(i=0; i<nSample; i++){
pIdx->aSample[i].anEq = pSpace; pSpace += nIdxCol;
pIdx->aSample[i].anLt = pSpace; pSpace += nIdxCol;
pIdx->aSample[i].anDLt = pSpace; pSpace += nIdxCol;
}
assert( ((u8*)pSpace)-nByte==(u8*)(pIdx->aSample) );
}
rc = sqlite3_finalize(pStmt);
if( rc ) return rc;
/* Allocate the Schema.pStat4Space block that will be used for the
** Index.aSample[] arrays populated by this call. */
rc = stat4AllocSpace(db, zDb);
if( rc!=SQLITE_OK ) return rc;
zSql = sqlite3MPrintf(db, zSql2, zDb);
if( !zSql ){
@@ -1865,18 +1925,23 @@ static int loadStatTbl(
if( zIndex==0 ) continue;
pIdx = findIndexOrPrimaryKey(db, zIndex, zDb);
if( pIdx==0 ) continue;
if( pIdx->nSample>=pIdx->mxSample ){
/* Too many slots used because the same index appears in
** sqlite_stat4 using multiple names */
continue;
if( pIdx->nSample==pIdx->nSampleAlloc ){
pIdx->pTable->tabFlags |= TF_HasStat4;
assert( !HasRowid(pIdx->pTable) || pIdx->nColumn==pIdx->nKeyCol+1 );
if( !HasRowid(pIdx->pTable) && IsPrimaryKeyIndex(pIdx) ){
pIdx->nSampleCol = pIdx->nKeyCol;
}else{
pIdx->nSampleCol = pIdx->nColumn;
}
/* This next condition is true if data has already been loaded from
** the sqlite_stat4 table. */
nCol = pIdx->nSampleCol;
if( growSampleArray(db, pIdx, &iBlockOff) ) break;
}
if( pIdx!=pPrevIdx ){
initAvgEq(pPrevIdx);
pPrevIdx = pIdx;
}
nCol = pIdx->nSampleCol;
pSample = &pIdx->aSample[pIdx->nSample];
decodeIntArray((char*)sqlite3_column_text(pStmt,1),nCol,pSample->anEq,0,0);
decodeIntArray((char*)sqlite3_column_text(pStmt,2),nCol,pSample->anLt,0,0);
@@ -1973,6 +2038,10 @@ int sqlite3AnalysisLoad(sqlite3 *db, int iDb){
pIdx->aSample = 0;
#endif
}
#ifdef SQLITE_ENABLE_STAT4
sqlite3_free(pSchema->pStat4Space);
pSchema->pStat4Space = 0;
#endif
/* Load new statistics out of the sqlite_stat1 table */
sInfo.db = db;

4
src/callback.c
View File

@@ -514,6 +514,10 @@ void sqlite3SchemaClear(void *p){
pSchema->iGeneration++;
}
pSchema->schemaFlags &= ~(DB_SchemaLoaded|DB_ResetWanted);
#ifdef SQLITE_ENABLE_STAT4
sqlite3_free(pSchema->pStat4Space);
pSchema->pStat4Space = 0;
#endif
}
/*

5
src/sqliteInt.h
View File

@@ -1487,6 +1487,9 @@ struct Schema {
u8 enc; /* Text encoding used by this database */
u16 schemaFlags; /* Flags associated with this schema */
int cache_size; /* Number of pages to use in the cache */
#ifdef SQLITE_ENABLE_STAT4
void *pStat4Space; /* Memory for stat4 Index.aSample[] arrays */
#endif
};
/*
@@ -2779,7 +2782,7 @@ struct Index {
** expression, or a reference to a VIRTUAL column */
#ifdef SQLITE_ENABLE_STAT4
int nSample; /* Number of elements in aSample[] */
int mxSample; /* Number of slots allocated to aSample[] */
int nSampleAlloc; /* Allocated size of aSample[] */
int nSampleCol; /* Size of IndexSample.anEq[] and so on */
tRowcnt *aAvgEq; /* Average nEq values for keys not in aSample */
IndexSample *aSample; /* Samples of the left-most key */