diff --git a/manifest b/manifest index 9bbf85e9e6..8eb0b6bfb3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sbug\sin\sfts5\stest\scode\sfunction\sfts5TestUtf8().\sReport\s[bugs:/info/2026-09-03T14:51:50Z\s|\s2026-09-03T14:51:50Z]. -D 2026-09-03T16:37:55.476 +C Enhance\sthe\sWAL\scheckpoint\salgorithm\sto\srun\sfaster\sand\suse\sless\smemory\nif\sthe\sWAL\sfile\sis\slarger\sthan\sthe\sdatabase\sfile. +D 2026-09-03T18:37:25.313 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -822,7 +822,7 @@ F src/vdbetrace.c 49e689f751505839742f4a243a1a566e57d5c9eaf0d33bbaa26e2de3febf7b F src/vdbevtab.c fc46b9cbd759dc013f0b3724549cc0d71379183c667df3a5988f7e2f1bd485f3 F src/vtab.c 1fa9e01d3da114a634a6ba1f36de8a3af6381b675cbbca399f0de64934e50ac6 F src/vxworks.h 9d18819c5235b49c2340a8a4d48195ec5d5afb637b152406de95a9436beeaeab -F src/wal.c 8124330a8702418202a3653712e03702ab0562e9bd9aa19ffe77067565ea8f5d +F src/wal.c 41438914e455bbc42ccff8545eab4c711f91233e480337e44a3d5fdf121fce74 F src/wal.h c6251cfc1950643a976625ca452e46dc9b466b4f5dd2e85f244f29f79bf68ace F src/walker.c d5006d6b005e4ea7302ad390957a8d41ed83faa177e412f89bc5600a7462a014 F src/where.c 717331794a92c00151b080c2c44de592720d26d03ea50de34e3fab05ab09ba61 @@ -2220,8 +2220,9 @@ F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee F tool/warnings.sh a554d13f6e5cf3760f041b87939e3d616ec6961859c3245e8ef701d1eafc2ca2 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f F tool/winmain.c 00c8fb88e365c9017db14c73d3c78af62194d9644feaf60e220ab0f411f3604c -P 5d9e5cddec144492df866bf0ea1808b1c099be92551cbb2433a0f3a0ff119023 -R 502bd5d161c1680eba5cceafd538703f -U dan -Z 8a496f1624cfd8f1dfe4db595dbd951e +P ac094ec69b9b726985d2fdb82062f098e77c427539cdf5b07ee15dd8427cba66 f690f5b1487083fde158a9dcd1f61c67821ed163726e07f1f42585e979b0462d +R e75fbab533c8c8b3fa3266c0c9e694ba +T +closed f690f5b1487083fde158a9dcd1f61c67821ed163726e07f1f42585e979b0462d +U drh +Z 7f35250b36cc57f279af568bdc0046b3 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 8bf85215d6..06b08905c7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ac094ec69b9b726985d2fdb82062f098e77c427539cdf5b07ee15dd8427cba66 +630ab4b6bd4497d4d09e3910f8a95908d19b0a0ab0da4bfe9c3f22e613411523 diff --git a/src/wal.c b/src/wal.c index e0ea8fdef1..77161f3df9 100644 --- a/src/wal.c +++ b/src/wal.c @@ -587,10 +587,41 @@ typedef u16 ht_slot; ** walIteratorFree() - Free an iterator. ** ** This functionality is used by the checkpoint code (see walCheckpoint()). +** +** There are two possible algorithms: +** +** WALITER-1 Load all pages mentioned in the WAL file and sort them, +** then hand them back out in sorted order as requested by +** walIteratorNext(). nPagemap==0 for this algorithm. +** +** WALITER-2 Create a map of all database file pages and record the +** last WAL file frame that updates that page. Then walk +** through the map in acsending order and return all +** non-zero entries. nPagemap is positive for this algorithm. +** +** Either algorithm should work for any database and WAL file, however +** algorithm WALITER-1 is faster and uses less memory when the database is +** larger than the WAL file and algorithm WALITER-2 is faster and uses +** less memory when the WAL file is larger than the database, modulo a +** constant of proportionality. The walIteratorInit() routine selects +** the best algorithm for each situation. WALITER-2 was added for +** SQLite 3.54.0. Prior to that, WALITER-1 was used always. +** +** For algorithm WALITER-2, nPagemap stores the number of pages in the +** database file, nSegment==1, and aSegment[0].aPgno[] is the page map array +** that is nPagemap+1 entries in length. The value of aSegment[0].aPgno[PGNO] +** is the frame number of the last (newest) entry for page PGNO in the WAL +** file, or 0 if PGNO is not in the WAL file. (Aside: There is no page +** number 0 in a well-formed database, so the memory allocation is arranged +** such that aSegment[0].aPgno[0] and aSegment[0].iZero are aliases for the +** same four bits of memory. This saves us from having to use -1 and +1 +** offsets to translate between 1-based SQLite page numbers and 0-based +** C-language index numbers. tag-20260903-1) */ struct WalIterator { u32 iPrior; /* Last result returned from the iterator */ int nSegment; /* Number of entries in aSegment[] */ + u32 nPagemap; /* Non-zero in "page-map" mode */ struct WalSegment { int iNext; /* Next slot in aIndex[] not yet returned */ ht_slot *aIndex; /* i0, i1, i2... such that aPgno[iN] ascend */ @@ -604,6 +635,7 @@ struct WalIterator { #define SZ_WALITERATOR(N) \ (offsetof(WalIterator,aSegment)+(N)*sizeof(struct WalSegment)) + /* ** Define the parameters of the hash tables in the wal-index file. There ** is a hash-table following every HASHTABLE_NPAGE page numbers in the @@ -1766,24 +1798,38 @@ static int walIteratorNext( u32 *piPage, /* OUT: The page number of the next page */ u32 *piFrame /* OUT: Wal frame index of next page */ ){ - u32 iMin; /* Result pgno must be greater than iMin */ u32 iRet = 0xFFFFFFFF; /* 0xffffffff is never a valid page number */ - int i; /* For looping through segments */ - iMin = p->iPrior; - assert( iMin<0xffffffff ); - for(i=p->nSegment-1; i>=0; i--){ - struct WalSegment *pSegment = &p->aSegment[i]; - while( pSegment->iNextnEntry ){ - u32 iPg = pSegment->aPgno[pSegment->aIndex[pSegment->iNext]]; - if( iPg>iMin ){ - if( iPgiZero + pSegment->aIndex[pSegment->iNext]; - } + if( p->nPagemap ){ + /* Algorithm WALITER-2 */ + while( p->iPriornPagemap ){ + p->iPrior++; + if( p->aSegment[0].aPgno[p->iPrior] ){ + iRet = p->iPrior; + *piFrame = p->aSegment[0].aPgno[p->iPrior]; break; } - pSegment->iNext++; + } + }else{ + /* Algorithm WALITER-1 */ + u32 iMin; /* Result pgno must be greater than iMin */ + int i; /* For looping through segments */ + + iMin = p->iPrior; + assert( iMin<0xffffffff ); + for(i=p->nSegment-1; i>=0; i--){ + struct WalSegment *pSegment = &p->aSegment[i]; + while( pSegment->iNextnEntry ){ + u32 iPg = pSegment->aPgno[pSegment->aIndex[pSegment->iNext]]; + if( iPg>iMin ){ + if( iPgiZero + pSegment->aIndex[pSegment->iNext]; + } + break; + } + pSegment->iNext++; + } } } @@ -1959,9 +2005,11 @@ static int walIteratorInit(Wal *pWal, u32 nBackfill, WalIterator **pp){ int nSegment; /* Number of segments to merge */ u32 iLast; /* Last frame in log */ sqlite3_int64 nByte; /* Number of bytes to allocate */ + sqlite3_int64 nExtra; /* Number of bytes to allocate */ int i; /* Iterator variable */ - ht_slot *aTmp; /* Temp space used by merge-sort */ + ht_slot *aTmp = 0; /* Temp space used by merge-sort */ int rc = SQLITE_OK; /* Return Code */ + int eAlg; /* Which algorithm to use. 1 or 2 */ /* This routine only runs while holding the checkpoint lock. And ** it only runs if there is actually content in the log (mxFrame>0). */ @@ -1972,18 +2020,35 @@ static int walIteratorInit(Wal *pWal, u32 nBackfill, WalIterator **pp){ iLastPage = walFramePage(iLast); iZero = iFirstPage ? (HASHTABLE_NPAGE_ONE+(iFirstPage-1)*HASHTABLE_NPAGE) : 0; + if( (iLast-iZero)*sizeof(ht_slot)>pWal->hdr.nPage*sizeof(u32) ){ + /* Algorithm WALITER-2 */ + nSegment = 1; + eAlg = 2; + nByte = SZ_WALITERATOR(nSegment) + pWal->hdr.nPage * sizeof(u32); + nExtra = 0; + }else{ + /* Algorithm WALITER-1 */ + nSegment = iLastPage - iFirstPage + 1; + eAlg = 1; + nByte = SZ_WALITERATOR(nSegment) + (iLast-iZero)*sizeof(ht_slot); + nExtra = sizeof(ht_slot) * (iLast>HASHTABLE_NPAGE?HASHTABLE_NPAGE:iLast); + } + /* Allocate space for the WalIterator object. */ - nSegment = iLastPage - iFirstPage + 1; - nByte = SZ_WALITERATOR(nSegment) + (iLast-iZero)*sizeof(ht_slot); - p = (WalIterator *)sqlite3_malloc64(nByte - + sizeof(ht_slot) * (iLast>HASHTABLE_NPAGE?HASHTABLE_NPAGE:iLast) - ); + p = (WalIterator *)sqlite3_malloc64(nByte + nExtra); if( !p ){ return SQLITE_NOMEM_BKPT; } memset(p, 0, nByte); p->nSegment = nSegment; - aTmp = (ht_slot*)&(((u8*)p)[nByte]); + if( eAlg==2 ){ + p->nPagemap = pWal->hdr.nPage; + p->aSegment[0].aPgno = ((u32*)&p->aSegment[1]) - 1; + assert( (u8*)&(p->aSegment[0].aPgno[0]) == (u8*)&(p->aSegment[0].iZero) ); + /* ^-- verifies tag-20260903-1 */ + }else{ + aTmp = (ht_slot*)&(((u8*)p)[nByte]); + } SEH_FREE_ON_ERROR(0, p); for(i=iFirstPage; rc==SQLITE_OK && i<=iLastPage; i++){ WalHashLoc sLoc; @@ -1999,18 +2064,29 @@ static int walIteratorInit(Wal *pWal, u32 nBackfill, WalIterator **pp){ }else{ nEntry = (int)((u32*)sLoc.aHash - (u32*)sLoc.aPgno); } - assert( i!=iFirstPage || iZero==sLoc.iZero ); - aIndex = &((ht_slot *)&p->aSegment[p->nSegment])[sLoc.iZero - iZero]; - sLoc.iZero++; - for(j=0; jaSegment[0].aPgno; + for(j=0; jnPagemap ){ + aPagemap[pgno] = sLoc.iZero + j + 1; + } + } + }else{ + assert( i!=iFirstPage || iZero==sLoc.iZero ); + aIndex = &((ht_slot *)&p->aSegment[p->nSegment])[sLoc.iZero - iZero]; + sLoc.iZero++; + + for(j=0; jaSegment[i-iFirstPage].iZero = sLoc.iZero; + p->aSegment[i-iFirstPage].nEntry = nEntry; + p->aSegment[i-iFirstPage].aIndex = aIndex; + p->aSegment[i-iFirstPage].aPgno = (u32 *)sLoc.aPgno; } - walMergesort((u32 *)sLoc.aPgno, aTmp, aIndex, &nEntry); - p->aSegment[i-iFirstPage].iZero = sLoc.iZero; - p->aSegment[i-iFirstPage].nEntry = nEntry; - p->aSegment[i-iFirstPage].aIndex = aIndex; - p->aSegment[i-iFirstPage].aPgno = (u32 *)sLoc.aPgno; } } if( rc!=SQLITE_OK ){