From c1d5978a4fc792ae0ce730a4d334e0b9f66f4efd Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 3 Sep 2026 14:45:41 +0000 Subject: [PATCH 1/2] When checkpointing, use a mapping from pgno to frame instead of the sorted list of frame/pgno pairs in cases where this will use less memory. FossilOrigin-Name: ec3d33931be807ba77296397775f2d88228d4200ed23e54962d2592ddff37979 --- manifest | 17 ++++---- manifest.tags | 4 +- manifest.uuid | 2 +- src/wal.c | 114 ++++++++++++++++++++++++++++++++++++-------------- 4 files changed, 96 insertions(+), 41 deletions(-) diff --git a/manifest b/manifest index e488aa12c7..013faac15e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\s-DSQLITE_DEBUG\sor\s-DSQLITE_ENABLE_WALSTAT\sis\sused,\sprovide\sthe\nsqlite_walstat()\sSQL\sfunction\sthat\sprovides\sstate\sinformation\sabout\sthe\nWAL\ssubsystem.\s\sUsed\sfor\sdebugging\sonly.\s\sFunction\ssubject\sto\schange\sor\nremoval. -D 2026-09-02T12:39:06.762 +C When\scheckpointing,\suse\sa\smapping\sfrom\spgno\sto\sframe\sinstead\sof\sthe\ssorted\slist\sof\sframe/pgno\spairs\sin\scases\swhere\sthis\swill\suse\sless\smemory. +D 2026-09-03T14:45:41.450 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -822,7 +822,7 @@ F src/vdbetrace.c 49e689f751505839742f4a243a1a566e57d5c9eaf0d33bbaa26e2de3febf7b F src/vdbevtab.c fc46b9cbd759dc013f0b3724549cc0d71379183c667df3a5988f7e2f1bd485f3 F src/vtab.c 1fa9e01d3da114a634a6ba1f36de8a3af6381b675cbbca399f0de64934e50ac6 F src/vxworks.h 9d18819c5235b49c2340a8a4d48195ec5d5afb637b152406de95a9436beeaeab -F src/wal.c 8124330a8702418202a3653712e03702ab0562e9bd9aa19ffe77067565ea8f5d +F src/wal.c 35cf86afedeff7e093358ca8fdf31e23f747d155eb8d65be065a4e7c43701ae1 F src/wal.h c6251cfc1950643a976625ca452e46dc9b466b4f5dd2e85f244f29f79bf68ace F src/walker.c d5006d6b005e4ea7302ad390957a8d41ed83faa177e412f89bc5600a7462a014 F src/where.c 717331794a92c00151b080c2c44de592720d26d03ea50de34e3fab05ab09ba61 @@ -2220,8 +2220,11 @@ F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee F tool/warnings.sh a554d13f6e5cf3760f041b87939e3d616ec6961859c3245e8ef701d1eafc2ca2 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f F tool/winmain.c 00c8fb88e365c9017db14c73d3c78af62194d9644feaf60e220ab0f411f3604c -P c76e64373cca884ab47ea24f8df3cedcdc823ccf3484e9b78715154f6d1ea66f -R 2d1ce33cb2d779c5d5c57e14c35ffcbc -U drh -Z 9a2c2b5126acb895a729e0270509945a +P 789d84e69171ef05d36486c38fbb0db10f8d827d9941472a2809f3113503e6c4 +R 0b558490a350d1a2751554bd844ca2c3 +T *branch * pagemap-checkpoint +T *sym-pagemap-checkpoint * +T -sym-trunk * +U dan +Z 9188279066c8596e85d377f21a6c9942 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.tags b/manifest.tags index bec971799f..1a32ad5a61 100644 --- a/manifest.tags +++ b/manifest.tags @@ -1,2 +1,2 @@ -branch trunk -tag trunk +branch pagemap-checkpoint +tag pagemap-checkpoint diff --git a/manifest.uuid b/manifest.uuid index be98efc619..74f8f84027 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -789d84e69171ef05d36486c38fbb0db10f8d827d9941472a2809f3113503e6c4 +ec3d33931be807ba77296397775f2d88228d4200ed23e54962d2592ddff37979 diff --git a/src/wal.c b/src/wal.c index e0ea8fdef1..2f8b9b2866 100644 --- a/src/wal.c +++ b/src/wal.c @@ -587,10 +587,20 @@ typedef u16 ht_slot; ** walIteratorFree() - Free an iterator. ** ** This functionality is used by the checkpoint code (see walCheckpoint()). +** +** nPagemap: +** If this is non-zero, then the iterator struct is used in a different +** way. In this case, it is always true that nSegment==1 and that +** aSegment[0].aPgno points to an array containing one entry for each +** page in the database file. If the entry is 0, then the corresponding +** page is not present in the wal file. Or, if the entry is non-zero, +** then it is the frame number of the last (newest) frame in the wal file +** that contains a version of the corresponding page. */ struct WalIterator { u32 iPrior; /* Last result returned from the iterator */ int nSegment; /* Number of entries in aSegment[] */ + u32 nPagemap; /* Non-zero in "page-map" mode */ struct WalSegment { int iNext; /* Next slot in aIndex[] not yet returned */ ht_slot *aIndex; /* i0, i1, i2... such that aPgno[iN] ascend */ @@ -604,6 +614,7 @@ struct WalIterator { #define SZ_WALITERATOR(N) \ (offsetof(WalIterator,aSegment)+(N)*sizeof(struct WalSegment)) + /* ** Define the parameters of the hash tables in the wal-index file. There ** is a hash-table following every HASHTABLE_NPAGE page numbers in the @@ -1766,24 +1777,36 @@ static int walIteratorNext( u32 *piPage, /* OUT: The page number of the next page */ u32 *piFrame /* OUT: Wal frame index of next page */ ){ - u32 iMin; /* Result pgno must be greater than iMin */ u32 iRet = 0xFFFFFFFF; /* 0xffffffff is never a valid page number */ - int i; /* For looping through segments */ - iMin = p->iPrior; - assert( iMin<0xffffffff ); - for(i=p->nSegment-1; i>=0; i--){ - struct WalSegment *pSegment = &p->aSegment[i]; - while( pSegment->iNextnEntry ){ - u32 iPg = pSegment->aPgno[pSegment->aIndex[pSegment->iNext]]; - if( iPg>iMin ){ - if( iPgiZero + pSegment->aIndex[pSegment->iNext]; - } + if( p->nPagemap ){ + while( p->iPriornPagemap ){ + p->iPrior++; + if( p->aSegment[0].aPgno[p->iPrior] ){ + iRet = p->iPrior; + *piFrame = p->aSegment[0].aPgno[p->iPrior]; break; } - pSegment->iNext++; + } + }else{ + u32 iMin; /* Result pgno must be greater than iMin */ + int i; /* For looping through segments */ + + iMin = p->iPrior; + assert( iMin<0xffffffff ); + for(i=p->nSegment-1; i>=0; i--){ + struct WalSegment *pSegment = &p->aSegment[i]; + while( pSegment->iNextnEntry ){ + u32 iPg = pSegment->aPgno[pSegment->aIndex[pSegment->iNext]]; + if( iPg>iMin ){ + if( iPgiZero + pSegment->aIndex[pSegment->iNext]; + } + break; + } + pSegment->iNext++; + } } } @@ -1959,9 +1982,11 @@ static int walIteratorInit(Wal *pWal, u32 nBackfill, WalIterator **pp){ int nSegment; /* Number of segments to merge */ u32 iLast; /* Last frame in log */ sqlite3_int64 nByte; /* Number of bytes to allocate */ + sqlite3_int64 nExtra; /* Number of bytes to allocate */ int i; /* Iterator variable */ - ht_slot *aTmp; /* Temp space used by merge-sort */ + ht_slot *aTmp = 0; /* Temp space used by merge-sort */ int rc = SQLITE_OK; /* Return Code */ + int bPagemap = 0; /* This routine only runs while holding the checkpoint lock. And ** it only runs if there is actually content in the log (mxFrame>0). */ @@ -1972,18 +1997,34 @@ static int walIteratorInit(Wal *pWal, u32 nBackfill, WalIterator **pp){ iLastPage = walFramePage(iLast); iZero = iFirstPage ? (HASHTABLE_NPAGE_ONE+(iFirstPage-1)*HASHTABLE_NPAGE) : 0; + if( (iLast-iZero)*sizeof(ht_slot)>pWal->hdr.nPage*sizeof(u32) ){ + /* In this case it would require more memory to use the normal + ** WalIterator scheme than it would to simply create a map from + ** page-number -> frame-number with an entry for each page in + ** the database. So do that instead. */ + nSegment = 1; + bPagemap = 1; + nByte = SZ_WALITERATOR(nSegment) + pWal->hdr.nPage * sizeof(u32); + nExtra = 0; + }else{ + nSegment = iLastPage - iFirstPage + 1; + nByte = SZ_WALITERATOR(nSegment) + (iLast-iZero)*sizeof(ht_slot); + nExtra = sizeof(ht_slot) * (iLast>HASHTABLE_NPAGE?HASHTABLE_NPAGE:iLast); + } + /* Allocate space for the WalIterator object. */ - nSegment = iLastPage - iFirstPage + 1; - nByte = SZ_WALITERATOR(nSegment) + (iLast-iZero)*sizeof(ht_slot); - p = (WalIterator *)sqlite3_malloc64(nByte - + sizeof(ht_slot) * (iLast>HASHTABLE_NPAGE?HASHTABLE_NPAGE:iLast) - ); + p = (WalIterator *)sqlite3_malloc64(nByte + nExtra); if( !p ){ return SQLITE_NOMEM_BKPT; } memset(p, 0, nByte); p->nSegment = nSegment; - aTmp = (ht_slot*)&(((u8*)p)[nByte]); + if( bPagemap ){ + p->nPagemap = pWal->hdr.nPage; + p->aSegment[0].aPgno = ((u32*)&p->aSegment[1]) - 1; + }else{ + aTmp = (ht_slot*)&(((u8*)p)[nByte]); + } SEH_FREE_ON_ERROR(0, p); for(i=iFirstPage; rc==SQLITE_OK && i<=iLastPage; i++){ WalHashLoc sLoc; @@ -1999,18 +2040,29 @@ static int walIteratorInit(Wal *pWal, u32 nBackfill, WalIterator **pp){ }else{ nEntry = (int)((u32*)sLoc.aHash - (u32*)sLoc.aPgno); } - assert( i!=iFirstPage || iZero==sLoc.iZero ); - aIndex = &((ht_slot *)&p->aSegment[p->nSegment])[sLoc.iZero - iZero]; - sLoc.iZero++; - for(j=0; jaSegment[0].aPgno; + for(j=0; jnPagemap ){ + aPagemap[pgno] = sLoc.iZero + j + 1; + } + } + }else{ + assert( i!=iFirstPage || iZero==sLoc.iZero ); + aIndex = &((ht_slot *)&p->aSegment[p->nSegment])[sLoc.iZero - iZero]; + sLoc.iZero++; + + for(j=0; jaSegment[i-iFirstPage].iZero = sLoc.iZero; + p->aSegment[i-iFirstPage].nEntry = nEntry; + p->aSegment[i-iFirstPage].aIndex = aIndex; + p->aSegment[i-iFirstPage].aPgno = (u32 *)sLoc.aPgno; } - walMergesort((u32 *)sLoc.aPgno, aTmp, aIndex, &nEntry); - p->aSegment[i-iFirstPage].iZero = sLoc.iZero; - p->aSegment[i-iFirstPage].nEntry = nEntry; - p->aSegment[i-iFirstPage].aIndex = aIndex; - p->aSegment[i-iFirstPage].aPgno = (u32 *)sLoc.aPgno; } } if( rc!=SQLITE_OK ){ From dd986a5c51cd38167a99c5a5eec90ade6e84c3c0 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 3 Sep 2026 18:26:53 +0000 Subject: [PATCH 2/2] Improve the documentation of the WalIterator object and the new pagemap algorithm. Remove one unnecessary conditional. FossilOrigin-Name: f690f5b1487083fde158a9dcd1f61c67821ed163726e07f1f42585e979b0462d --- manifest | 17 ++++++--------- manifest.uuid | 2 +- src/wal.c | 60 +++++++++++++++++++++++++++++++++++---------------- 3 files changed, 50 insertions(+), 29 deletions(-) diff --git a/manifest b/manifest index 013faac15e..38a15dc643 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\scheckpointing,\suse\sa\smapping\sfrom\spgno\sto\sframe\sinstead\sof\sthe\ssorted\slist\sof\sframe/pgno\spairs\sin\scases\swhere\sthis\swill\suse\sless\smemory. -D 2026-09-03T14:45:41.450 +C Improve\sthe\sdocumentation\sof\sthe\sWalIterator\sobject\sand\sthe\snew\spagemap\nalgorithm.\s\sRemove\sone\sunnecessary\sconditional. +D 2026-09-03T18:26:53.404 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -822,7 +822,7 @@ F src/vdbetrace.c 49e689f751505839742f4a243a1a566e57d5c9eaf0d33bbaa26e2de3febf7b F src/vdbevtab.c fc46b9cbd759dc013f0b3724549cc0d71379183c667df3a5988f7e2f1bd485f3 F src/vtab.c 1fa9e01d3da114a634a6ba1f36de8a3af6381b675cbbca399f0de64934e50ac6 F src/vxworks.h 9d18819c5235b49c2340a8a4d48195ec5d5afb637b152406de95a9436beeaeab -F src/wal.c 35cf86afedeff7e093358ca8fdf31e23f747d155eb8d65be065a4e7c43701ae1 +F src/wal.c 41438914e455bbc42ccff8545eab4c711f91233e480337e44a3d5fdf121fce74 F src/wal.h c6251cfc1950643a976625ca452e46dc9b466b4f5dd2e85f244f29f79bf68ace F src/walker.c d5006d6b005e4ea7302ad390957a8d41ed83faa177e412f89bc5600a7462a014 F src/where.c 717331794a92c00151b080c2c44de592720d26d03ea50de34e3fab05ab09ba61 @@ -2220,11 +2220,8 @@ F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee F tool/warnings.sh a554d13f6e5cf3760f041b87939e3d616ec6961859c3245e8ef701d1eafc2ca2 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f F tool/winmain.c 00c8fb88e365c9017db14c73d3c78af62194d9644feaf60e220ab0f411f3604c -P 789d84e69171ef05d36486c38fbb0db10f8d827d9941472a2809f3113503e6c4 -R 0b558490a350d1a2751554bd844ca2c3 -T *branch * pagemap-checkpoint -T *sym-pagemap-checkpoint * -T -sym-trunk * -U dan -Z 9188279066c8596e85d377f21a6c9942 +P ec3d33931be807ba77296397775f2d88228d4200ed23e54962d2592ddff37979 +R f7d01afe083cba83456db34833f65bcd +U drh +Z d6d208cd5260b0abc28c325ce271d5be # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 74f8f84027..0a61a1a3d8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ec3d33931be807ba77296397775f2d88228d4200ed23e54962d2592ddff37979 +f690f5b1487083fde158a9dcd1f61c67821ed163726e07f1f42585e979b0462d diff --git a/src/wal.c b/src/wal.c index 2f8b9b2866..77161f3df9 100644 --- a/src/wal.c +++ b/src/wal.c @@ -588,14 +588,35 @@ typedef u16 ht_slot; ** ** This functionality is used by the checkpoint code (see walCheckpoint()). ** -** nPagemap: -** If this is non-zero, then the iterator struct is used in a different -** way. In this case, it is always true that nSegment==1 and that -** aSegment[0].aPgno points to an array containing one entry for each -** page in the database file. If the entry is 0, then the corresponding -** page is not present in the wal file. Or, if the entry is non-zero, -** then it is the frame number of the last (newest) frame in the wal file -** that contains a version of the corresponding page. +** There are two possible algorithms: +** +** WALITER-1 Load all pages mentioned in the WAL file and sort them, +** then hand them back out in sorted order as requested by +** walIteratorNext(). nPagemap==0 for this algorithm. +** +** WALITER-2 Create a map of all database file pages and record the +** last WAL file frame that updates that page. Then walk +** through the map in acsending order and return all +** non-zero entries. nPagemap is positive for this algorithm. +** +** Either algorithm should work for any database and WAL file, however +** algorithm WALITER-1 is faster and uses less memory when the database is +** larger than the WAL file and algorithm WALITER-2 is faster and uses +** less memory when the WAL file is larger than the database, modulo a +** constant of proportionality. The walIteratorInit() routine selects +** the best algorithm for each situation. WALITER-2 was added for +** SQLite 3.54.0. Prior to that, WALITER-1 was used always. +** +** For algorithm WALITER-2, nPagemap stores the number of pages in the +** database file, nSegment==1, and aSegment[0].aPgno[] is the page map array +** that is nPagemap+1 entries in length. The value of aSegment[0].aPgno[PGNO] +** is the frame number of the last (newest) entry for page PGNO in the WAL +** file, or 0 if PGNO is not in the WAL file. (Aside: There is no page +** number 0 in a well-formed database, so the memory allocation is arranged +** such that aSegment[0].aPgno[0] and aSegment[0].iZero are aliases for the +** same four bits of memory. This saves us from having to use -1 and +1 +** offsets to translate between 1-based SQLite page numbers and 0-based +** C-language index numbers. tag-20260903-1) */ struct WalIterator { u32 iPrior; /* Last result returned from the iterator */ @@ -1780,6 +1801,7 @@ static int walIteratorNext( u32 iRet = 0xFFFFFFFF; /* 0xffffffff is never a valid page number */ if( p->nPagemap ){ + /* Algorithm WALITER-2 */ while( p->iPriornPagemap ){ p->iPrior++; if( p->aSegment[0].aPgno[p->iPrior] ){ @@ -1789,6 +1811,7 @@ static int walIteratorNext( } } }else{ + /* Algorithm WALITER-1 */ u32 iMin; /* Result pgno must be greater than iMin */ int i; /* For looping through segments */ @@ -1982,11 +2005,11 @@ static int walIteratorInit(Wal *pWal, u32 nBackfill, WalIterator **pp){ int nSegment; /* Number of segments to merge */ u32 iLast; /* Last frame in log */ sqlite3_int64 nByte; /* Number of bytes to allocate */ - sqlite3_int64 nExtra; /* Number of bytes to allocate */ + sqlite3_int64 nExtra; /* Number of bytes to allocate */ int i; /* Iterator variable */ ht_slot *aTmp = 0; /* Temp space used by merge-sort */ int rc = SQLITE_OK; /* Return Code */ - int bPagemap = 0; + int eAlg; /* Which algorithm to use. 1 or 2 */ /* This routine only runs while holding the checkpoint lock. And ** it only runs if there is actually content in the log (mxFrame>0). */ @@ -1998,16 +2021,15 @@ static int walIteratorInit(Wal *pWal, u32 nBackfill, WalIterator **pp){ iZero = iFirstPage ? (HASHTABLE_NPAGE_ONE+(iFirstPage-1)*HASHTABLE_NPAGE) : 0; if( (iLast-iZero)*sizeof(ht_slot)>pWal->hdr.nPage*sizeof(u32) ){ - /* In this case it would require more memory to use the normal - ** WalIterator scheme than it would to simply create a map from - ** page-number -> frame-number with an entry for each page in - ** the database. So do that instead. */ + /* Algorithm WALITER-2 */ nSegment = 1; - bPagemap = 1; + eAlg = 2; nByte = SZ_WALITERATOR(nSegment) + pWal->hdr.nPage * sizeof(u32); nExtra = 0; }else{ + /* Algorithm WALITER-1 */ nSegment = iLastPage - iFirstPage + 1; + eAlg = 1; nByte = SZ_WALITERATOR(nSegment) + (iLast-iZero)*sizeof(ht_slot); nExtra = sizeof(ht_slot) * (iLast>HASHTABLE_NPAGE?HASHTABLE_NPAGE:iLast); } @@ -2019,9 +2041,11 @@ static int walIteratorInit(Wal *pWal, u32 nBackfill, WalIterator **pp){ } memset(p, 0, nByte); p->nSegment = nSegment; - if( bPagemap ){ + if( eAlg==2 ){ p->nPagemap = pWal->hdr.nPage; p->aSegment[0].aPgno = ((u32*)&p->aSegment[1]) - 1; + assert( (u8*)&(p->aSegment[0].aPgno[0]) == (u8*)&(p->aSegment[0].iZero) ); + /* ^-- verifies tag-20260903-1 */ }else{ aTmp = (ht_slot*)&(((u8*)p)[nByte]); } @@ -2041,11 +2065,11 @@ static int walIteratorInit(Wal *pWal, u32 nBackfill, WalIterator **pp){ nEntry = (int)((u32*)sLoc.aHash - (u32*)sLoc.aPgno); } - if( bPagemap ){ + if( eAlg==2 ){ u32 *aPagemap = p->aSegment[0].aPgno; for(j=0; jnPagemap ){ + if( pgno<=p->nPagemap ){ aPagemap[pgno] = sLoc.iZero + j + 1; } }