database/sqlite
1
0
Fork 0
mirror of https://github.com/sqlite/sqlite.git synced 2026-10-23 21:49:02 +03:00
Code Activity

Enhance the WAL checkpoint algorithm to run faster and use less memory

Browse Source 
if the WAL file is larger than the database file.

FossilOrigin-Name: 630ab4b6bd4497d4d09e3910f8a95908d19b0a0ab0da4bfe9c3f22e613411523
This commit is contained in:
drh
2026-09-03 18:37:25 +00:00
parent 42c45bd235 dd986a5c51
commit 7740d55183
3 changed files with 116 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
manifest
View File

@@ -1,5 +1,5 @@
C Fix\sa\sbug\sin\sfts5\stest\scode\sfunction\sfts5TestUtf8().\sReport\s[bugs:/info/2026-09-03T14:51:50Z\s|\s2026-09-03T14:51:50Z].
D 2026-09-03T16:37:55.476
C Enhance\sthe\sWAL\scheckpoint\salgorithm\sto\srun\sfaster\sand\suse\sless\smemory\nif\sthe\sWAL\sfile\sis\slarger\sthan\sthe\sdatabase\sfile.
D 2026-09-03T18:37:25.313
F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
@@ -822,7 +822,7 @@ F src/vdbetrace.c 49e689f751505839742f4a243a1a566e57d5c9eaf0d33bbaa26e2de3febf7b
F src/vdbevtab.c fc46b9cbd759dc013f0b3724549cc0d71379183c667df3a5988f7e2f1bd485f3
F src/vtab.c 1fa9e01d3da114a634a6ba1f36de8a3af6381b675cbbca399f0de64934e50ac6
F src/vxworks.h 9d18819c5235b49c2340a8a4d48195ec5d5afb637b152406de95a9436beeaeab
F src/wal.c 8124330a8702418202a3653712e03702ab0562e9bd9aa19ffe77067565ea8f5d
F src/wal.c 41438914e455bbc42ccff8545eab4c711f91233e480337e44a3d5fdf121fce74
F src/wal.h c6251cfc1950643a976625ca452e46dc9b466b4f5dd2e85f244f29f79bf68ace
F src/walker.c d5006d6b005e4ea7302ad390957a8d41ed83faa177e412f89bc5600a7462a014
F src/where.c 717331794a92c00151b080c2c44de592720d26d03ea50de34e3fab05ab09ba61
@@ -2220,8 +2220,9 @@ F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee
F tool/warnings.sh a554d13f6e5cf3760f041b87939e3d616ec6961859c3245e8ef701d1eafc2ca2
F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
F tool/winmain.c 00c8fb88e365c9017db14c73d3c78af62194d9644feaf60e220ab0f411f3604c
P 5d9e5cddec144492df866bf0ea1808b1c099be92551cbb2433a0f3a0ff119023
R 502bd5d161c1680eba5cceafd538703f
U dan
Z 8a496f1624cfd8f1dfe4db595dbd951e
P ac094ec69b9b726985d2fdb82062f098e77c427539cdf5b07ee15dd8427cba66 f690f5b1487083fde158a9dcd1f61c67821ed163726e07f1f42585e979b0462d
R e75fbab533c8c8b3fa3266c0c9e694ba
T +closed f690f5b1487083fde158a9dcd1f61c67821ed163726e07f1f42585e979b0462d
U drh
Z 7f35250b36cc57f279af568bdc0046b3
# Remove this line to create a well-formed Fossil manifest.

2
manifest.uuid
View File

@@ -1 +1 @@
ac094ec69b9b726985d2fdb82062f098e77c427539cdf5b07ee15dd8427cba66
630ab4b6bd4497d4d09e3910f8a95908d19b0a0ab0da4bfe9c3f22e613411523

138
src/wal.c
View File

@@ -587,10 +587,41 @@ typedef u16 ht_slot;
** walIteratorFree() - Free an iterator.
**
** This functionality is used by the checkpoint code (see walCheckpoint()).
**
** There are two possible algorithms:
**
** WALITER-1 Load all pages mentioned in the WAL file and sort them,
** then hand them back out in sorted order as requested by
** walIteratorNext(). nPagemap==0 for this algorithm.
**
** WALITER-2 Create a map of all database file pages and record the
** last WAL file frame that updates that page. Then walk
** through the map in acsending order and return all
** non-zero entries. nPagemap is positive for this algorithm.
**
** Either algorithm should work for any database and WAL file, however
** algorithm WALITER-1 is faster and uses less memory when the database is
** larger than the WAL file and algorithm WALITER-2 is faster and uses
** less memory when the WAL file is larger than the database, modulo a
** constant of proportionality. The walIteratorInit() routine selects
** the best algorithm for each situation. WALITER-2 was added for
** SQLite 3.54.0. Prior to that, WALITER-1 was used always.
**
** For algorithm WALITER-2, nPagemap stores the number of pages in the
** database file, nSegment==1, and aSegment[0].aPgno[] is the page map array
** that is nPagemap+1 entries in length. The value of aSegment[0].aPgno[PGNO]
** is the frame number of the last (newest) entry for page PGNO in the WAL
** file, or 0 if PGNO is not in the WAL file. (Aside: There is no page
** number 0 in a well-formed database, so the memory allocation is arranged
** such that aSegment[0].aPgno[0] and aSegment[0].iZero are aliases for the
** same four bits of memory. This saves us from having to use -1 and +1
** offsets to translate between 1-based SQLite page numbers and 0-based
** C-language index numbers. tag-20260903-1)
*/
struct WalIterator {
u32 iPrior; /* Last result returned from the iterator */
int nSegment; /* Number of entries in aSegment[] */
u32 nPagemap; /* Non-zero in "page-map" mode */
struct WalSegment {
int iNext; /* Next slot in aIndex[] not yet returned */
ht_slot *aIndex; /* i0, i1, i2... such that aPgno[iN] ascend */
@@ -604,6 +635,7 @@ struct WalIterator {
#define SZ_WALITERATOR(N) \
(offsetof(WalIterator,aSegment)+(N)*sizeof(struct WalSegment))
/*
** Define the parameters of the hash tables in the wal-index file. There
** is a hash-table following every HASHTABLE_NPAGE page numbers in the
@@ -1766,24 +1798,38 @@ static int walIteratorNext(
u32 *piPage, /* OUT: The page number of the next page */
u32 *piFrame /* OUT: Wal frame index of next page */
){
u32 iMin; /* Result pgno must be greater than iMin */
u32 iRet = 0xFFFFFFFF; /* 0xffffffff is never a valid page number */
int i; /* For looping through segments */
iMin = p->iPrior;
assert( iMin<0xffffffff );
for(i=p->nSegment-1; i>=0; i--){
struct WalSegment *pSegment = &p->aSegment[i];
while( pSegment->iNext<pSegment->nEntry ){
u32 iPg = pSegment->aPgno[pSegment->aIndex[pSegment->iNext]];
if( iPg>iMin ){
if( iPg<iRet ){
iRet = iPg;
*piFrame = pSegment->iZero + pSegment->aIndex[pSegment->iNext];
}
if( p->nPagemap ){
/* Algorithm WALITER-2 */
while( p->iPrior<p->nPagemap ){
p->iPrior++;
if( p->aSegment[0].aPgno[p->iPrior] ){
iRet = p->iPrior;
*piFrame = p->aSegment[0].aPgno[p->iPrior];
break;
}
pSegment->iNext++;
}
}else{
/* Algorithm WALITER-1 */
u32 iMin; /* Result pgno must be greater than iMin */
int i; /* For looping through segments */
iMin = p->iPrior;
assert( iMin<0xffffffff );
for(i=p->nSegment-1; i>=0; i--){
struct WalSegment *pSegment = &p->aSegment[i];
while( pSegment->iNext<pSegment->nEntry ){
u32 iPg = pSegment->aPgno[pSegment->aIndex[pSegment->iNext]];
if( iPg>iMin ){
if( iPg<iRet ){
iRet = iPg;
*piFrame = pSegment->iZero + pSegment->aIndex[pSegment->iNext];
}
break;
}
pSegment->iNext++;
}
}
}
@@ -1959,9 +2005,11 @@ static int walIteratorInit(Wal *pWal, u32 nBackfill, WalIterator **pp){
int nSegment; /* Number of segments to merge */
u32 iLast; /* Last frame in log */
sqlite3_int64 nByte; /* Number of bytes to allocate */
sqlite3_int64 nExtra; /* Number of bytes to allocate */
int i; /* Iterator variable */
ht_slot *aTmp; /* Temp space used by merge-sort */
ht_slot *aTmp = 0; /* Temp space used by merge-sort */
int rc = SQLITE_OK; /* Return Code */
int eAlg; /* Which algorithm to use. 1 or 2 */
/* This routine only runs while holding the checkpoint lock. And
** it only runs if there is actually content in the log (mxFrame>0). */
@@ -1972,18 +2020,35 @@ static int walIteratorInit(Wal *pWal, u32 nBackfill, WalIterator **pp){
iLastPage = walFramePage(iLast);
iZero = iFirstPage ? (HASHTABLE_NPAGE_ONE+(iFirstPage-1)*HASHTABLE_NPAGE) : 0;
if( (iLast-iZero)*sizeof(ht_slot)>pWal->hdr.nPage*sizeof(u32) ){
/* Algorithm WALITER-2 */
nSegment = 1;
eAlg = 2;
nByte = SZ_WALITERATOR(nSegment) + pWal->hdr.nPage * sizeof(u32);
nExtra = 0;
}else{
/* Algorithm WALITER-1 */
nSegment = iLastPage - iFirstPage + 1;
eAlg = 1;
nByte = SZ_WALITERATOR(nSegment) + (iLast-iZero)*sizeof(ht_slot);
nExtra = sizeof(ht_slot) * (iLast>HASHTABLE_NPAGE?HASHTABLE_NPAGE:iLast);
}
/* Allocate space for the WalIterator object. */
nSegment = iLastPage - iFirstPage + 1;
nByte = SZ_WALITERATOR(nSegment) + (iLast-iZero)*sizeof(ht_slot);
p = (WalIterator *)sqlite3_malloc64(nByte
+ sizeof(ht_slot) * (iLast>HASHTABLE_NPAGE?HASHTABLE_NPAGE:iLast)
);
p = (WalIterator *)sqlite3_malloc64(nByte + nExtra);
if( !p ){
return SQLITE_NOMEM_BKPT;
}
memset(p, 0, nByte);
p->nSegment = nSegment;
aTmp = (ht_slot*)&(((u8*)p)[nByte]);
if( eAlg==2 ){
p->nPagemap = pWal->hdr.nPage;
p->aSegment[0].aPgno = ((u32*)&p->aSegment[1]) - 1;
assert( (u8*)&(p->aSegment[0].aPgno[0]) == (u8*)&(p->aSegment[0].iZero) );
/* ^-- verifies tag-20260903-1 */
}else{
aTmp = (ht_slot*)&(((u8*)p)[nByte]);
}
SEH_FREE_ON_ERROR(0, p);
for(i=iFirstPage; rc==SQLITE_OK && i<=iLastPage; i++){
WalHashLoc sLoc;
@@ -1999,18 +2064,29 @@ static int walIteratorInit(Wal *pWal, u32 nBackfill, WalIterator **pp){
}else{
nEntry = (int)((u32*)sLoc.aHash - (u32*)sLoc.aPgno);
}
assert( i!=iFirstPage || iZero==sLoc.iZero );
aIndex = &((ht_slot *)&p->aSegment[p->nSegment])[sLoc.iZero - iZero];
sLoc.iZero++;
for(j=0; j<nEntry; j++){
aIndex[j] = (ht_slot)j;
if( eAlg==2 ){
u32 *aPagemap = p->aSegment[0].aPgno;
for(j=0; j<nEntry; j++){
u32 pgno = sLoc.aPgno[ j ];
if( pgno<=p->nPagemap ){
aPagemap[pgno] = sLoc.iZero + j + 1;
}
}
}else{
assert( i!=iFirstPage || iZero==sLoc.iZero );
aIndex = &((ht_slot *)&p->aSegment[p->nSegment])[sLoc.iZero - iZero];
sLoc.iZero++;
for(j=0; j<nEntry; j++){
aIndex[j] = (ht_slot)j;
}
walMergesort((u32 *)sLoc.aPgno, aTmp, aIndex, &nEntry);
p->aSegment[i-iFirstPage].iZero = sLoc.iZero;
p->aSegment[i-iFirstPage].nEntry = nEntry;
p->aSegment[i-iFirstPage].aIndex = aIndex;
p->aSegment[i-iFirstPage].aPgno = (u32 *)sLoc.aPgno;
}
walMergesort((u32 *)sLoc.aPgno, aTmp, aIndex, &nEntry);
p->aSegment[i-iFirstPage].iZero = sLoc.iZero;
p->aSegment[i-iFirstPage].nEntry = nEntry;
p->aSegment[i-iFirstPage].aIndex = aIndex;
p->aSegment[i-iFirstPage].aPgno = (u32 *)sLoc.aPgno;
}
}
if( rc!=SQLITE_OK ){