From 588400b861b2fa82a591b78a85cf3f460435ba8a Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 27 Sep 2014 05:00:25 +0000 Subject: [PATCH 001/133] Reduce the amount of memcpy() required by defragmentPage(). FossilOrigin-Name: 3edab9957cc7bb90b52fd40b02613c2cb03fc166 --- manifest | 15 +++++++++------ manifest.uuid | 2 +- src/btree.c | 25 ++++++++++++++++--------- 3 files changed, 26 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 131d77b3a9..574bfe2c8d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\s"PRAGMA\sintegrity_check"\scommand\sso\sthat\sit\savoids\sformatting\serror\nmessage\scontext\smessages\suntil\sit\sactually\sneeds\sto\sgenerate\san\serror\smessage.\nThis\savoids\smuch\sformatting,\sand\shence\sgreatly\simproves\sthe\sperformance\sof\n"PRAGMA\sintegrity_check"\sin\sthe\scommon\scase\swhen\sthere\sare\sno\serrors.\s\sIt\salso\nmakes\sthe\scode\sa\slittle\ssmaller. -D 2014-09-26T02:41:05.726 +C Reduce\sthe\samount\sof\smemcpy()\srequired\sby\sdefragmentPage(). +D 2014-09-27T05:00:25.096 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 59f03e421dad3cb6e27cc7d2393d3a7459be4b5e +F src/btree.c 95a942a6ebdb23eb2a5d925526d35169aa6742f6 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 1bd7957161a1346a914f1f09231610e777a8e58d F src/build.c bde83dd5cf812e310a7e5ad2846790a14745bef4 @@ -1200,7 +1200,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P e93aecc090c2a1d3c231bb2bde044886eff0bdf7 -R 32be19747fff5e8f2465eed0f224b45d +P 83913515830aa850f9e38406f9422d7e88dcab66 +R 66a1e1f00a844450677737824735607d +T *branch * defrag-opt +T *sym-defrag-opt * +T -sym-trunk * U drh -Z f56b9000203c19d0f3a8172e8374b279 +Z f60a1f2e4650c574e91ad245c0c67dee diff --git a/manifest.uuid b/manifest.uuid index b182c8c316..bf8d17f152 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -83913515830aa850f9e38406f9422d7e88dcab66 \ No newline at end of file +3edab9957cc7bb90b52fd40b02613c2cb03fc166 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 135b40139f..c4832b4ee9 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1151,6 +1151,7 @@ static int defragmentPage(MemPage *pPage){ int nCell; /* Number of cells on the page */ unsigned char *data; /* The page data */ unsigned char *temp; /* Temp area for cell content */ + unsigned char *src; /* Source of content */ int iCellFirst; /* First allowable cell index */ int iCellLast; /* Last possible cell index */ @@ -1160,15 +1161,13 @@ static int defragmentPage(MemPage *pPage){ assert( pPage->pBt->usableSize <= SQLITE_MAX_PAGE_SIZE ); assert( pPage->nOverflow==0 ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - temp = sqlite3PagerTempSpace(pPage->pBt->pPager); - data = pPage->aData; + temp = 0; + src = data = pPage->aData; hdr = pPage->hdrOffset; cellOffset = pPage->cellOffset; nCell = pPage->nCell; assert( nCell==get2byte(&data[hdr+3]) ); usableSize = pPage->pBt->usableSize; - cbrk = get2byte(&data[hdr+5]); - memcpy(&temp[cbrk], &data[cbrk], usableSize - cbrk); cbrk = usableSize; iCellFirst = cellOffset + 2*nCell; iCellLast = usableSize - 4; @@ -1187,7 +1186,7 @@ static int defragmentPage(MemPage *pPage){ } #endif assert( pc>=iCellFirst && pc<=iCellLast ); - size = cellSizePtr(pPage, &temp[pc]); + size = cellSizePtr(pPage, &src[pc]); cbrk -= size; #if defined(SQLITE_ENABLE_OVERSIZE_CELL_CHECK) if( cbrk=iCellFirst ); testcase( cbrk+size==usableSize ); testcase( pc+size==usableSize ); - memcpy(&data[cbrk], &temp[pc], size); put2byte(pAddr, cbrk); + if( temp==0 ){ + int x; + if( cbrk==pc ) continue; + temp = sqlite3PagerTempSpace(pPage->pBt->pPager); + x = get2byte(&data[hdr+5]); + memcpy(&temp[x], &data[x], (cbrk+size) - x); + src = temp; + } + memcpy(&data[cbrk], &src[pc], size); } assert( cbrk>=iCellFirst ); put2byte(&data[hdr+5], cbrk); @@ -5955,13 +5962,13 @@ static void assemblePage( assert( pPage->nCell==0 ); assert( get2byteNotZero(&data[hdr+5])==nUsable ); - pCellptr = &pPage->aCellIdx[nCell*2]; + pCellptr = pPage->aCellIdx; cellbody = nUsable; - for(i=nCell-1; i>=0; i--){ + for(i=0; i Date: Thu, 9 Oct 2014 19:35:37 +0000 Subject: [PATCH 002/133] Change the balance_nonroot() routine to reduce the amount of memcpy work that takes place. This is a work in progress. FossilOrigin-Name: 29304499ea4b72dbb6701e10cc19b5d41f7e5ac9 --- manifest | 21 +-- manifest.uuid | 2 +- src/btree.c | 494 +++++++++++++++++++++++++++----------------------- src/pager.c | 9 + src/pager.h | 2 + 5 files changed, 292 insertions(+), 236 deletions(-) diff --git a/manifest b/manifest index 574bfe2c8d..aeefa53238 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Reduce\sthe\samount\sof\smemcpy()\srequired\sby\sdefragmentPage(). -D 2014-09-27T05:00:25.096 +C Change\sthe\sbalance_nonroot()\sroutine\sto\sreduce\sthe\samount\sof\smemcpy\swork\sthat\stakes\splace.\sThis\sis\sa\swork\sin\sprogress. +D 2014-10-09T19:35:37.452 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 95a942a6ebdb23eb2a5d925526d35169aa6742f6 +F src/btree.c 7b89fde3bffa5b7300e94c4aeb69ccff926ef513 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 1bd7957161a1346a914f1f09231610e777a8e58d F src/build.c bde83dd5cf812e310a7e5ad2846790a14745bef4 @@ -215,8 +215,8 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c fb587121840f690101336879adfa6d0b2cd0e8c7 F src/os_win.c 0a4042ef35f322e86fa01f6c8884c5e645b911e7 F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 -F src/pager.c caab007743821d96752597c9cfd7351654697b06 -F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 +F src/pager.c 0abcb0904a78d68b96357f360c6b160bcfc2a3e0 +F src/pager.h 8b6707cb32c788cf36bfc3d63f6d4b4fa689e7c2 F src/parse.y b98772da2bb5415970085b707203f92569400aa8 F src/pcache.c 4121a0571c18581ee9f82f086d5e2030051ebd6a F src/pcache.h 9b559127b83f84ff76d735c8262f04853be0c59a @@ -1200,10 +1200,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 83913515830aa850f9e38406f9422d7e88dcab66 -R 66a1e1f00a844450677737824735607d -T *branch * defrag-opt -T *sym-defrag-opt * -T -sym-trunk * -U drh -Z f60a1f2e4650c574e91ad245c0c67dee +P 3edab9957cc7bb90b52fd40b02613c2cb03fc166 +R 5fca1836b5a4d862df24682ecb47d048 +U dan +Z 8527330c8f275358262176fc962502e2 diff --git a/manifest.uuid b/manifest.uuid index bf8d17f152..bb422fd7dd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3edab9957cc7bb90b52fd40b02613c2cb03fc166 \ No newline at end of file +29304499ea4b72dbb6701e10cc19b5d41f7e5ac9 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index c4832b4ee9..dbf002ecdb 100644 --- a/src/btree.c +++ b/src/btree.c @@ -5977,6 +5977,49 @@ static void assemblePage( pPage->nCell = (u16)nCell; } + +static void rebuildPage( + MemPage *pPg, /* Edit this page */ + int nRemove, /* Cells to remove from start of page */ + int nCell, /* Final number of cells on page */ + u8 **apCell, /* Array of nCell final cells */ + u16 *szCell /* Array of nCell cell sizes */ +){ + const int hdr = pPg->hdrOffset; /* Offset of header on pPg */ + u8 * const aData = pPg->aData; /* Pointer to data for pPg */ + const int usableSize = pPg->pBt->usableSize; + u8 * const pEnd = &aData[usableSize]; + int i; + u8 *pCellptr = pPg->aCellIdx; + u8 *pTmp = sqlite3PagerTempSpace(pPg->pBt->pPager); + u8 *pData; + + i = get2byte(&aData[hdr+5]); + memcpy(&pTmp[i], &aData[i], usableSize - i); + pData = &aData[usableSize]; + + for(i=0; iaData && pCellnFree = (pData - pCellptr); + pPg->nCell = nCell; + pPg->nOverflow = 0; + + put2byte(&aData[hdr+1], 0); + put2byte(&aData[hdr+3], pPg->nCell); + put2byte(&aData[hdr+5], pData - aData); + aData[hdr+7] = 0x00; +} + /* ** The following parameters determine how many adjacent pages get involved ** in a balancing operation. NN is the number of neighbors on either side @@ -6098,7 +6141,7 @@ static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){ } #endif /* SQLITE_OMIT_QUICKBALANCE */ -#if 0 +#if 1 /* ** This function does not contribute anything to the operation of SQLite. ** it is sometimes activated temporarily while debugging code responsible @@ -6265,7 +6308,6 @@ static int balance_nonroot( int iOvflSpace = 0; /* First unused byte of aOvflSpace[] */ int szScratch; /* Size of scratch memory requested */ MemPage *apOld[NB]; /* pPage and up to two siblings */ - MemPage *apCopy[NB]; /* Private copies of apOld[] pages */ MemPage *apNew[NB+2]; /* pPage and up to NB siblings after balancing */ u8 *pRight; /* Location in parent of right-sibling pointer */ u8 *apDiv[NB-1]; /* Divider cells in pParent */ @@ -6276,6 +6318,13 @@ static int balance_nonroot( u8 *aSpace1; /* Space for copies of dividers cells */ Pgno pgno; /* Temp var to store a page number in */ + int aShiftLeft[NB+2]; + int aShiftRight[NB+2]; + u8 abDone[NB+2]; + Pgno aPgno[NB+2]; + u16 aPgFlags[NB+2]; + + memset(abDone, 0, sizeof(abDone)); pBt = pParent->pBt; assert( sqlite3_mutex_held(pBt->mutex) ); assert( sqlite3PagerIswriteable(pParent->pDbPage) ); @@ -6384,12 +6433,10 @@ static int balance_nonroot( /* ** Allocate space for memory structures */ - k = pBt->pageSize + ROUND8(sizeof(MemPage)); szScratch = nMaxCells*sizeof(u8*) /* apCell */ + nMaxCells*sizeof(u16) /* szCell */ - + pBt->pageSize /* aSpace1 */ - + k*nOld; /* Page copies (apCopy) */ + + pBt->pageSize; /* aSpace1 */ apCell = sqlite3ScratchMalloc( szScratch ); if( apCell==0 ){ rc = SQLITE_NOMEM; @@ -6402,8 +6449,8 @@ static int balance_nonroot( /* ** Load pointers to all cells on sibling pages and the divider cells ** into the local apCell[] array. Make copies of the divider cells - ** into space obtained from aSpace1[] and remove the divider cells - ** from pParent. + ** into space obtained from aSpace1[]. The divider cells have already + ** been removed from pParent. ** ** If the siblings are on leaf pages, then the child pointers of the ** divider cells are stripped from the cells before they are copied @@ -6419,15 +6466,7 @@ static int balance_nonroot( leafData = apOld[0]->intKeyLeaf; for(i=0; ipageSize + k*i]; - memcpy(pOld, apOld[i], sizeof(MemPage)); - pOld->aData = (void*)&pOld[1]; - memcpy(pOld->aData, apOld[i]->aData, pBt->pageSize); + MemPage *pOld = apOld[i]; limit = pOld->nCell+pOld->nOverflow; if( pOld->nOverflow>0 ){ @@ -6556,10 +6595,10 @@ static int balance_nonroot( assert( cntNew[0]>0 || (pParent->pgno==1 && pParent->nCell==0) ); #endif - TRACE(("BALANCE: old: %d %d %d ", - apOld[0]->pgno, - nOld>=2 ? apOld[1]->pgno : 0, - nOld>=3 ? apOld[2]->pgno : 0 + TRACE(("BALANCE: old: %d(nc=%d) %d(nc=%d) %d(nc=%d)\n", + apOld[0]->pgno, apOld[0]->nCell, + nOld>=2 ? apOld[1]->pgno : 0, nOld>=2 ? apOld[1]->nCell : 0, + nOld>=3 ? apOld[2]->pgno : 0, nOld>=3 ? apOld[2]->nCell : 0 )); /* @@ -6582,6 +6621,7 @@ static int balance_nonroot( assert( i>0 ); rc = allocateBtreePage(pBt, &pNew, &pgno, (bBulk ? 1 : pgno), 0); if( rc ) goto balance_cleanup; + zeroPage(pNew, pageFlags); apNew[i] = pNew; nNew++; @@ -6595,135 +6635,223 @@ static int balance_nonroot( } } - /* Free any old pages that were not reused as new pages. - */ - while( ipgno; - int minI = i; - for(j=i+1; jpgno<(unsigned)minV ){ - minI = j; - minV = apNew[j]->pgno; + for(i=0; ipgno; + aPgFlags[i] = apNew[i]->pDbPage->flags; + } + for(i=0; ipgno); + Pgno iMin = 0; + u16 flags = 0; + for(j=0; jiGt && (iMin==0 || iPgnoi ){ - MemPage *pT; - pT = apNew[i]; - apNew[i] = apNew[minI]; - apNew[minI] = pT; + if( apNew[i]->pgno!=iMin ){ + apNew[i]->pDbPage->flags = flags; + sqlite3PagerRekey(apNew[i]->pDbPage, iMin); + apNew[i]->pgno = iMin; } } - TRACE(("new: %d(%d) %d(%d) %d(%d) %d(%d) %d(%d)\n", - apNew[0]->pgno, szNew[0], + + TRACE(("BALANCE: new: %d(%d nc=%d) %d(%d nc=%d) %d(%d nc=%d) " + "%d(%d nc=%d) %d(%d nc=%d)\n", + apNew[0]->pgno, szNew[0], cntNew[0], nNew>=2 ? apNew[1]->pgno : 0, nNew>=2 ? szNew[1] : 0, + nNew>=2 ? cntNew[1] - cntNew[0] - !leafData : 0, nNew>=3 ? apNew[2]->pgno : 0, nNew>=3 ? szNew[2] : 0, + nNew>=3 ? cntNew[2] - cntNew[1] - !leafData : 0, nNew>=4 ? apNew[3]->pgno : 0, nNew>=4 ? szNew[3] : 0, - nNew>=5 ? apNew[4]->pgno : 0, nNew>=5 ? szNew[4] : 0)); + nNew>=4 ? cntNew[3] - cntNew[2] - !leafData : 0, + nNew>=5 ? apNew[4]->pgno : 0, nNew>=5 ? szNew[4] : 0, + nNew>=5 ? cntNew[4] - cntNew[3] - !leafData : 0 + )); assert( sqlite3PagerIswriteable(pParent->pDbPage) ); put4byte(pRight, apNew[nNew-1]->pgno); - /* - ** Evenly distribute the data in apCell[] across the new pages. - ** Insert divider cells into pParent as necessary. - */ j = 0; for(i=0; inCell>0 || (nNew==1 && cntNew[0]==0) ); - assert( pNew->nOverflow==0 ); + /* At this point, "j" is the apCell[] index of the first cell currently + ** stored on page apNew[i]. Or, if apNew[i] was not one of the original + ** sibling pages, "j" should be set to nCell. Variable iFirst is set + ** to the apCell[] index of the first cell that will appear on the + ** page following this balancing operation. */ + int iFirst = (i==0 ? 0 : cntNew[i-1] + !leafData); /* new first cell */ + assert( inCell + apNew[i]->nOverflow; + aShiftRight[i] = cntNew[i] - j; + assert( i!=nOld-1 || j==nCell ); + if( jnOld ? apNew : apOld)[nOld-1]; + memcpy(&apNew[nNew-1]->aData[8], &pOld->aData[8], 4); + } - /* If the sibling page assembled above was not the right-most sibling, - ** insert a divider cell into the parent page. - */ - assert( iaData; + int cntOldNext = pNew->nCell + pNew->nOverflow; + int usableSize = pBt->usableSize; + int iNew = 0; + int iOld = 0; - assert( jleaf ){ - memcpy(&pNew->aData[8], pCell, 4); - }else if( leafData ){ - /* If the tree is a leaf-data tree, and the siblings are leaves, - ** then there is no divider cell in apCell[]. Instead, the divider - ** cell consists of the integer key for the right-most cell of - ** the sibling-page assembled above only. - */ - CellInfo info; - j--; - btreeParseCellPtr(pNew, apCell[j], &info); - pCell = pTemp; - sz = 4 + putVarint(&pCell[4], info.nKey); - pTemp = 0; - }else{ - pCell -= 4; - /* Obscure case for non-leaf-data trees: If the cell at pCell was - ** previously stored on a leaf node, and its reported size was 4 - ** bytes, then it may actually be smaller than this - ** (see btreeParseCellPtr(), 4 bytes is the minimum size of - ** any cell). But it is important to pass the correct size to - ** insertCell(), so reparse the cell now. - ** - ** Note that this can never happen in an SQLite data file, as all - ** cells are at least 4 bytes. It only happens in b-trees used - ** to evaluate "IN (SELECT ...)" and similar clauses. - */ - if( szCell[j]==4 ){ - assert(leafCorrection==4); - sz = cellSizePtr(pParent, pCell); + for(i=0; inCell + pOld->nOverflow + !leafData; + aOld = pOld->aData; + } + if( i==cntNew[iNew] ){ + pNew = apNew[++iNew]; + if( !leafData ) continue; + } + + /* Cell pCell is destined for new sibling page pNew. Originally, it + ** was either part of sibling page iOld (possibly an overflow page), + ** or else the divider cell to the left of sibling page iOld. So, + ** if sibling page iOld had the same page number as pNew, and if + ** pCell really was a part of sibling page iOld (not a divider or + ** overflow cell), we can skip updating the pointer map entries. */ + if( pNew->pgno!=aPgno[iOld] || pCell=&aOld[usableSize] ){ + if( !leafCorrection ){ + ptrmapPut(pBt, get4byte(pCell), PTRMAP_BTREE, pNew->pgno, &rc); + } + if( szCell[i]>pNew->minLocal ){ + ptrmapPutOvflPtr(pNew, pCell, &rc); } } - iOvflSpace += sz; - assert( sz<=pBt->maxLocal+23 ); - assert( iOvflSpace <= (int)pBt->pageSize ); - insertCell(pParent, nxDiv, pCell, sz, pTemp, pNew->pgno, &rc); - if( rc!=SQLITE_OK ) goto balance_cleanup; - assert( sqlite3PagerIswriteable(pParent->pDbPage) ); - - j++; - nxDiv++; } } - assert( j==nCell ); + + /* Insert new divider cells into pParent. */ + for(i=0; ileaf ){ + memcpy(&pNew->aData[8], pCell, 4); + }else if( leafData ){ + /* If the tree is a leaf-data tree, and the siblings are leaves, + ** then there is no divider cell in apCell[]. Instead, the divider + ** cell consists of the integer key for the right-most cell of + ** the sibling-page assembled above only. + */ + CellInfo info; + j--; + btreeParseCellPtr(pNew, apCell[j], &info); + pCell = pTemp; + sz = 4 + putVarint(&pCell[4], info.nKey); + pTemp = 0; + }else{ + pCell -= 4; + /* Obscure case for non-leaf-data trees: If the cell at pCell was + ** previously stored on a leaf node, and its reported size was 4 + ** bytes, then it may actually be smaller than this + ** (see btreeParseCellPtr(), 4 bytes is the minimum size of + ** any cell). But it is important to pass the correct size to + ** insertCell(), so reparse the cell now. + ** + ** Note that this can never happen in an SQLite data file, as all + ** cells are at least 4 bytes. It only happens in b-trees used + ** to evaluate "IN (SELECT ...)" and similar clauses. + */ + if( szCell[j]==4 ){ + assert(leafCorrection==4); + sz = cellSizePtr(pParent, pCell); + } + } + iOvflSpace += sz; + assert( sz<=pBt->maxLocal+23 ); + assert( iOvflSpace <= (int)pBt->pageSize ); + insertCell(pParent, nxDiv+i, pCell, sz, pTemp, pNew->pgno, &rc); + if( rc!=SQLITE_OK ) goto balance_cleanup; + assert( sqlite3PagerIswriteable(pParent->pDbPage) ); + } + + /* Now update the actual sibling pages. The order in which they are updated + ** is important, as this code needs to avoid disrupting any page from which + ** cells may still to be read. In practice, this means: + ** + ** 1) If the aShiftLeft[] entry is less than 0, it is not safe to + ** update the page until the page to the left of the current page + ** (apNew[i-1]) has already been updated. + ** + ** 2) If the aShiftRight[] entry is less than 0, it is not safe to + ** update the page until the page to the right of the current page + ** (apNew[i+1]) has already been updated. + ** + ** If neither of the above apply, the page is safe to update. + */ + assert( aShiftRight[nNew-1]>=0 && aShiftLeft[0]==0 ); + for(i=0; i=nNew ? i-nNew : nNew-1-i); + if( abDone[iPg]==0 + && (aShiftLeft[iPg]>=0 || abDone[iPg-1]) + && (aShiftRight[iPg]>=0 || abDone[iPg+1]) + ){ + MemPage *pNew = apNew[iPg]; + int iLeft = ((iPg==0) ? 0 : cntNew[iPg-1] + !leafData); + rebuildPage(pNew, + aShiftLeft[iPg] < 0 ? (aShiftLeft[iPg]*-1) : 0, + cntNew[iPg] - iLeft, + &apCell[iLeft], + &szCell[iLeft] + ); + abDone[iPg] = 1; + assert( pNew->nOverflow==0 ); + assert( pNew->nCell==(cntNew[iPg] - (iPg==0?0:cntNew[iPg-1]+!leafData)) ); + } + } + assert( memcmp(abDone, "\01\01\01\01\01", nNew)==0 ); + assert( nOld>0 ); assert( nNew>0 ); - if( (pageFlags & PTF_LEAF)==0 ){ - u8 *zChild = &apCopy[nOld-1]->aData[8]; - memcpy(&apNew[nNew-1]->aData[8], zChild, 4); - } if( isRoot && pParent->nCell==0 && pParent->hdrOffset<=apNew[0]->nFree ){ /* The root page of the b-tree now contains no cells. The only sibling @@ -6746,116 +6874,36 @@ static int balance_nonroot( ); copyNodeContent(apNew[0], pParent, &rc); freePage(apNew[0], &rc); - }else if( ISAUTOVACUUM ){ - /* Fix the pointer-map entries for all the cells that were shifted around. - ** There are several different types of pointer-map entries that need to - ** be dealt with by this routine. Some of these have been set already, but - ** many have not. The following is a summary: - ** - ** 1) The entries associated with new sibling pages that were not - ** siblings when this function was called. These have already - ** been set. We don't need to worry about old siblings that were - ** moved to the free-list - the freePage() code has taken care - ** of those. - ** - ** 2) The pointer-map entries associated with the first overflow - ** page in any overflow chains used by new divider cells. These - ** have also already been taken care of by the insertCell() code. - ** - ** 3) If the sibling pages are not leaves, then the child pages of - ** cells stored on the sibling pages may need to be updated. - ** - ** 4) If the sibling pages are not internal intkey nodes, then any - ** overflow pages used by these cells may need to be updated - ** (internal intkey nodes never contain pointers to overflow pages). - ** - ** 5) If the sibling pages are not leaves, then the pointer-map - ** entries for the right-child pages of each sibling may need - ** to be updated. - ** - ** Cases 1 and 2 are dealt with above by other code. The next - ** block deals with cases 3 and 4 and the one after that, case 5. Since - ** setting a pointer map entry is a relatively expensive operation, this - ** code only sets pointer map entries for child or overflow pages that have - ** actually moved between pages. */ - MemPage *pNew = apNew[0]; - MemPage *pOld = apCopy[0]; - int nOverflow = pOld->nOverflow; - int iNextOld = pOld->nCell + nOverflow; - int iOverflow = (nOverflow ? pOld->aiOvfl[0] : -1); - j = 0; /* Current 'old' sibling page */ - k = 0; /* Current 'new' sibling page */ - for(i=0; inCell + pOld->nOverflow; - if( pOld->nOverflow ){ - nOverflow = pOld->nOverflow; - iOverflow = i + !leafData + pOld->aiOvfl[0]; - } - isDivider = !leafData; - } - - assert(nOverflow>0 || iOverflowaiOvfl[0]==pOld->aiOvfl[1]-1); - assert(nOverflow<3 || pOld->aiOvfl[1]==pOld->aiOvfl[2]-1); - if( i==iOverflow ){ - isDivider = 1; - if( (--nOverflow)>0 ){ - iOverflow++; - } - } - - if( i==cntNew[k] ){ - /* Cell i is the cell immediately following the last cell on new - ** sibling page k. If the siblings are not leaf pages of an - ** intkey b-tree, then cell i is a divider cell. */ - pNew = apNew[++k]; - if( !leafData ) continue; - } - assert( jpgno!=pNew->pgno ){ - if( !leafCorrection ){ - ptrmapPut(pBt, get4byte(apCell[i]), PTRMAP_BTREE, pNew->pgno, &rc); - } - if( szCell[i]>pNew->minLocal ){ - ptrmapPutOvflPtr(pNew, apCell[i], &rc); - } - } + }else if( ISAUTOVACUUM && !leafCorrection ){ + /* Fix the pointer map entries associated with the right-child of each + ** sibling page. All other pointer map entries have already been taken + ** care of. */ + for(i=0; iaData[8]); + ptrmapPut(pBt, key, PTRMAP_BTREE, apNew[i]->pgno, &rc); } + } - if( !leafCorrection ){ - for(i=0; iaData[8]); - ptrmapPut(pBt, key, PTRMAP_BTREE, apNew[i]->pgno, &rc); - } - } + assert( pParent->isInit ); + TRACE(("BALANCE: finished: old=%d new=%d cells=%d\n", + nOld, nNew, nCell)); -#if 0 + /* Free any old pages that were not reused as new pages. + */ + for(i=nNew; iisInit ){ /* The ptrmapCheckPages() contains assert() statements that verify that ** all pointer map pages are set correctly. This is helpful while ** debugging. This is usually disabled because a corrupt database may ** cause an assert() statement to fail. */ ptrmapCheckPages(apNew, nNew); ptrmapCheckPages(&pParent, 1); -#endif } - - assert( pParent->isInit ); - TRACE(("BALANCE: finished: old=%d new=%d cells=%d\n", - nOld, nNew, nCell)); +#endif /* ** Cleanup before returning. diff --git a/src/pager.c b/src/pager.c index 79bfe15f10..e68d147de5 100644 --- a/src/pager.c +++ b/src/pager.c @@ -6835,6 +6835,14 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ return SQLITE_OK; } + +void sqlite3PagerRekey(DbPage *pPage, Pgno iNew){ + PgHdr *pPg = (PgHdr*)pPage; + assert( pPg->flags & PGHDR_DIRTY ); + assert( !subjRequiresPage(pPg) ); + sqlite3PcacheMove(pPg, iNew); +} + #endif /* @@ -7235,4 +7243,5 @@ int sqlite3PagerWalFramesize(Pager *pPager){ } #endif + #endif /* SQLITE_OMIT_DISKIO */ diff --git a/src/pager.h b/src/pager.h index c9ca8553b9..f3a04bbca6 100644 --- a/src/pager.h +++ b/src/pager.h @@ -188,6 +188,8 @@ int sqlite3SectorSize(sqlite3_file *); /* Functions used to truncate the database file. */ void sqlite3PagerTruncateImage(Pager*,Pgno); +void sqlite3PagerRekey(DbPage*, Pgno); + #if defined(SQLITE_HAS_CODEC) && !defined(SQLITE_OMIT_WAL) void *sqlite3PagerCodec(DbPage *); #endif From 09c6840153697ea6679650b65dfa50bc15cef536 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 11 Oct 2014 20:00:24 +0000 Subject: [PATCH 003/133] Attempt to further reduce memcpy() in balance_nonroot(). FossilOrigin-Name: fec849dcca3aead2bc2d4ecffeda750684d32fb0 --- manifest | 12 ++-- manifest.uuid | 2 +- src/btree.c | 185 ++++++++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 179 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index aeefa53238..7e782aa82c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\sbalance_nonroot()\sroutine\sto\sreduce\sthe\samount\sof\smemcpy\swork\sthat\stakes\splace.\sThis\sis\sa\swork\sin\sprogress. -D 2014-10-09T19:35:37.452 +C Attempt\sto\sfurther\sreduce\smemcpy()\sin\sbalance_nonroot(). +D 2014-10-11T20:00:24.552 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 7b89fde3bffa5b7300e94c4aeb69ccff926ef513 +F src/btree.c d5f4f74e309f79ace4b4025c433874ead635bed2 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 1bd7957161a1346a914f1f09231610e777a8e58d F src/build.c bde83dd5cf812e310a7e5ad2846790a14745bef4 @@ -1200,7 +1200,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 3edab9957cc7bb90b52fd40b02613c2cb03fc166 -R 5fca1836b5a4d862df24682ecb47d048 +P 29304499ea4b72dbb6701e10cc19b5d41f7e5ac9 +R 3235ac37769ff059d7c714947e8a11dd U dan -Z 8527330c8f275358262176fc962502e2 +Z 17fdfd7faf788bbb79d2ec9941c17025 diff --git a/manifest.uuid b/manifest.uuid index bb422fd7dd..006b3c0008 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -29304499ea4b72dbb6701e10cc19b5d41f7e5ac9 \ No newline at end of file +fec849dcca3aead2bc2d4ecffeda750684d32fb0 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index dbf002ecdb..13ec912748 100644 --- a/src/btree.c +++ b/src/btree.c @@ -5980,10 +5980,9 @@ static void assemblePage( static void rebuildPage( MemPage *pPg, /* Edit this page */ - int nRemove, /* Cells to remove from start of page */ int nCell, /* Final number of cells on page */ - u8 **apCell, /* Array of nCell final cells */ - u16 *szCell /* Array of nCell cell sizes */ + u8 **apCell, /* Array of cells */ + u16 *szCell /* Array of cell sizes */ ){ const int hdr = pPg->hdrOffset; /* Offset of header on pPg */ u8 * const aData = pPg->aData; /* Pointer to data for pPg */ @@ -6020,6 +6019,129 @@ static void rebuildPage( aData[hdr+7] = 0x00; } +static void editPage( + MemPage *pPg, /* Edit this page */ + int iOld, /* Index of first cell currently on page */ + int iNew, /* Index of new first cell on page */ + int nNew, /* Final number of cells on page */ + u8 **apCell, /* Array of cells */ + u16 *szCell /* Array of cell sizes */ +){ + + if( 1 ){ + u8 * const aData = pPg->aData; + const int hdr = pPg->hdrOffset; + u8 *pBegin = &pPg->aCellIdx[nNew * 2]; + int nFree = pPg->nFree; /* Free bytes on pPg */ + int nCell = pPg->nCell; /* Cells stored on pPg */ + u8 *pData; + u8 *pCellptr; + int i; + int iOldEnd = iOld + pPg->nCell + pPg->nOverflow; + +#ifdef SQLITE_DEBUG + u8 *pTmp = sqlite3PagerTempSpace(pPg->pBt->pPager); + memcpy(pTmp, aData, pPg->pBt->usableSize); +#endif + + /* Remove cells from the start and end of the page */ + if( iOldaData && apCell[i]<&aData[pPg->pBt->usableSize] ){ + freeSpace(pPg, apCell[i] - aData, szCell[i]); + nFree += szCell[i] + 2; + nShift++; + } + } + nCell -= nShift; + memmove(pPg->aCellIdx, &pPg->aCellIdx[nShift*2], nCell*2); + } + for(i=iNew+nNew; iaData && apCell[i]<&aData[pPg->pBt->usableSize] ){ + freeSpace(pPg, apCell[i] - aData, szCell[i]); + nFree += szCell[i] + 2; + nCell--; + } + } + pData = &aData[get2byte(&aData[hdr+5])]; + if( pDataaCellIdx; + memmove(&pCellptr[(iOld-iNew)*2], pCellptr, nCell*2); + for(i=iNew; inOverflow; i++){ + int iCell = (iOld + pPg->aiOvfl[i]) - iNew; + if( iCell>=0 && iCellaCellIdx[iCell * 2]; + int sz = szCell[iCell+iNew]; + memmove(&pCellptr[2], pCellptr, (nCell - iCell) * 2); + pData -= sz; + if( pDataaCellIdx[nCell*2]; + for(i=iNew+nCell; i<(iNew+nNew); i++){ + pData -= szCell[i]; + if( pDatanFree = nFree; + pPg->nCell = nNew; + pPg->nOverflow = 0; + + put2byte(&aData[hdr+3], pPg->nCell); + put2byte(&aData[hdr+5], pData - aData); + +#ifdef SQLITE_DEBUG + for(i=0; iaCellIdx[i*2]); + if( pCell>=aData && pCell<&aData[pPg->pBt->usableSize] ){ + pCell = &pTmp[pCell - aData]; + } + assert( 0==memcmp(pCell, &aData[iOff], szCell[i+iNew]) ); + } +#endif + +#if 0 + printf("EDIT\n"); +#endif + + return; + } + + editpage_fail: +#if 0 + printf("REBUILD\n"); +#endif + /* Unable to edit this page. Rebuild it from scratch instead. */ + rebuildPage(pPg, nNew, &apCell[iNew], &szCell[iNew]); +} + /* ** The following parameters determine how many adjacent pages get involved ** in a balancing operation. NN is the number of neighbors on either side @@ -6312,6 +6434,7 @@ static int balance_nonroot( u8 *pRight; /* Location in parent of right-sibling pointer */ u8 *apDiv[NB-1]; /* Divider cells in pParent */ int cntNew[NB+2]; /* Index in aCell[] of cell after i-th page */ + int cntOld[NB+2]; /* Old index in aCell[] after i-th page */ int szNew[NB+2]; /* Combined size of cells place on i-th page */ u8 **apCell = 0; /* All cells begin balanced */ u16 *szCell; /* Local size of all cells in apCell[] */ @@ -6487,6 +6610,7 @@ static int balance_nonroot( nCell++; } } + cntOld[i] = nCell; if( i %d@%d", apNew[i]->pgno, + pNew->nCell+pNew->nOverflow, j, + cntNew[i] - iFirst, iFirst + ); + for(iCell=iFirst; iCellaData + || apCell[iCell]>=&pNew->aData[pBt->usableSize] + ){ + nCta += szCell[iCell]; + } + } + nFree = get2byte(&pNew->aData[pNew->hdrOffset+5]); + nFree -= (pNew->cellOffset + (cntNew[i] - iFirst) * 2); + printf(" cta=%d free=%d\n", nCta, nFree); + if( i==(nNew-1) ){ + printf("-----\n"); + fflush(stdout); + } +#endif + assert( inCell + apNew[i]->nOverflow; @@ -6835,17 +6987,23 @@ static int balance_nonroot( && (aShiftLeft[iPg]>=0 || abDone[iPg-1]) && (aShiftRight[iPg]>=0 || abDone[iPg+1]) ){ - MemPage *pNew = apNew[iPg]; - int iLeft = ((iPg==0) ? 0 : cntNew[iPg-1] + !leafData); - rebuildPage(pNew, - aShiftLeft[iPg] < 0 ? (aShiftLeft[iPg]*-1) : 0, - cntNew[iPg] - iLeft, - &apCell[iLeft], - &szCell[iLeft] - ); + int iNew; + int iOld; + int nNewCell; + + if( iPg==0 ){ + iNew = iOld = 0; + nNewCell = cntNew[0]; + }else{ + iOld = iPgnOverflow==0 ); - assert( pNew->nCell==(cntNew[iPg] - (iPg==0?0:cntNew[iPg-1]+!leafData)) ); + assert( apNew[iPg]->nOverflow==0 ); + assert( apNew[iPg]->nCell==nNewCell ); } } assert( memcmp(abDone, "\01\01\01\01\01", nNew)==0 ); @@ -6869,6 +7027,7 @@ static int balance_nonroot( ** is important if the parent page happens to be page 1 of the database ** image. */ assert( nNew==1 ); + defragmentPage(apNew[0]); assert( apNew[0]->nFree == (get2byte(&apNew[0]->aData[5])-apNew[0]->cellOffset-apNew[0]->nCell*2) ); From d7b545bbcb2225a33e8c8c5192ab11416a855999 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 13 Oct 2014 18:03:27 +0000 Subject: [PATCH 004/133] Further work on balance_nonroot(). FossilOrigin-Name: 6594f9b420e2fa642737722ff8521f756ecef227 --- manifest | 12 +-- manifest.uuid | 2 +- src/btree.c | 265 ++++++++++++++++++++++++++++++++------------------ 3 files changed, 180 insertions(+), 99 deletions(-) diff --git a/manifest b/manifest index 7e782aa82c..9f25f390d9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Attempt\sto\sfurther\sreduce\smemcpy()\sin\sbalance_nonroot(). -D 2014-10-11T20:00:24.552 +C Further\swork\son\sbalance_nonroot(). +D 2014-10-13T18:03:27.743 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c d5f4f74e309f79ace4b4025c433874ead635bed2 +F src/btree.c 63211ca1d4ae867eede39a37901aec4746d904a7 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 1bd7957161a1346a914f1f09231610e777a8e58d F src/build.c bde83dd5cf812e310a7e5ad2846790a14745bef4 @@ -1200,7 +1200,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 29304499ea4b72dbb6701e10cc19b5d41f7e5ac9 -R 3235ac37769ff059d7c714947e8a11dd +P fec849dcca3aead2bc2d4ecffeda750684d32fb0 +R 79e5b5564c934fc68257d82dc897cefd U dan -Z 17fdfd7faf788bbb79d2ec9941c17025 +Z ae5b74eee28666050af6baaa639de56c diff --git a/manifest.uuid b/manifest.uuid index 006b3c0008..fcba485229 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fec849dcca3aead2bc2d4ecffeda750684d32fb0 \ No newline at end of file +6594f9b420e2fa642737722ff8521f756ecef227 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 13ec912748..7c9a7228de 100644 --- a/src/btree.c +++ b/src/btree.c @@ -6009,7 +6009,7 @@ static void rebuildPage( assert( szCell[i]==cellSizePtr(pPg, pCell) ); } - pPg->nFree = (pData - pCellptr); + /* The pPg->nFree field is now set incorrectly. The caller will fix it. */ pPg->nCell = nCell; pPg->nOverflow = 0; @@ -6019,6 +6019,106 @@ static void rebuildPage( aData[hdr+7] = 0x00; } +static u8 *pageFindSlot(MemPage *pPg, int nByte){ + const int hdr = pPg->hdrOffset; + u8 * const aData = pPg->aData; + int iAddr; + int pc; + int usableSize = pPg->pBt->usableSize; + + for(iAddr=hdr+1; (pc = get2byte(&aData[iAddr]))>0; iAddr=pc){ + int size; /* Size of the free slot */ + if( pc>usableSize-4 || pc=nByte ){ + int x = size - nByte; + testcase( x==4 ); + testcase( x==3 ); + if( x<4 ){ + if( aData[hdr+7]>=60 ) return 0; + /* Remove the slot from the free-list. Update the number of + ** fragmented bytes within the page. */ + memcpy(&aData[iAddr], &aData[pc], 2); + aData[hdr+7] += (u8)x; + }else if( size+pc > usableSize ){ + return 0; + }else{ + /* The slot remains on the free-list. Reduce its size to account + ** for the portion used by the new allocation. */ + put2byte(&aData[pc+2], x); + } + return &aData[pc + x]; + } + } + + return 0; +} + +static int pageInsertArray( + MemPage *pPg, + u8 *pBegin, + u8 **ppData, + u8 *pCellptr, + int nCell, + u8 **apCell, /* Array of cells */ + u16 *szCell /* Array of cell sizes */ +){ + int i; + u8 *aData = pPg->aData; + u8 *pData = *ppData; + for(i=0; iaData; + u8 * const pEnd = &aData[pPg->pBt->usableSize]; + int nRet = 0; + int i; + u8 *pFree = 0; + int szFree = 0; + + for(i=0; iaData && pCellnFree field is invalid when this function returns. It is the +** responsibility of the caller to set it correctly. +*/ static void editPage( MemPage *pPg, /* Edit this page */ int iOld, /* Index of first cell currently on page */ @@ -6027,113 +6127,93 @@ static void editPage( u8 **apCell, /* Array of cells */ u16 *szCell /* Array of cell sizes */ ){ - - if( 1 ){ - u8 * const aData = pPg->aData; - const int hdr = pPg->hdrOffset; - u8 *pBegin = &pPg->aCellIdx[nNew * 2]; - int nFree = pPg->nFree; /* Free bytes on pPg */ - int nCell = pPg->nCell; /* Cells stored on pPg */ - u8 *pData; - u8 *pCellptr; - int i; - int iOldEnd = iOld + pPg->nCell + pPg->nOverflow; + u8 * const aData = pPg->aData; + const int hdr = pPg->hdrOffset; + u8 *pBegin = &pPg->aCellIdx[nNew * 2]; + int nCell = pPg->nCell; /* Cells stored on pPg */ + u8 *pData; + u8 *pCellptr; + int i; + int iOldEnd = iOld + pPg->nCell + pPg->nOverflow; + int iNewEnd = iNew + nNew; #ifdef SQLITE_DEBUG - u8 *pTmp = sqlite3PagerTempSpace(pPg->pBt->pPager); - memcpy(pTmp, aData, pPg->pBt->usableSize); + u8 *pTmp = sqlite3PagerTempSpace(pPg->pBt->pPager); + memcpy(pTmp, aData, pPg->pBt->usableSize); #endif - /* Remove cells from the start and end of the page */ - if( iOldaData && apCell[i]<&aData[pPg->pBt->usableSize] ){ - freeSpace(pPg, apCell[i] - aData, szCell[i]); - nFree += szCell[i] + 2; - nShift++; - } - } - nCell -= nShift; - memmove(pPg->aCellIdx, &pPg->aCellIdx[nShift*2], nCell*2); - } - for(i=iNew+nNew; iaData && apCell[i]<&aData[pPg->pBt->usableSize] ){ - freeSpace(pPg, apCell[i] - aData, szCell[i]); - nFree += szCell[i] + 2; - nCell--; - } - } - pData = &aData[get2byte(&aData[hdr+5])]; - if( pDataaCellIdx, &pPg->aCellIdx[nShift*2], nCell*2); + nCell -= nShift; + } + if( iNewEnd < iOldEnd ){ + nCell -= pageFreeArray( + pPg, iOldEnd-iNewEnd, &apCell[iNewEnd], &szCell[iNewEnd] + ); + } - /* Add cells to the start of the page */ - if( iNewaCellIdx; - memmove(&pCellptr[(iOld-iNew)*2], pCellptr, nCell*2); - for(i=iNew; iaCellIdx; + memmove(&pCellptr[nAdd*2], pCellptr, nCell*2); + if( pageInsertArray( + pPg, pBegin, &pData, pCellptr, + nAdd, &apCell[iNew], &szCell[iNew] + ) ) goto editpage_fail; + nCell += nAdd; + } + + /* Add any overflow cells */ + for(i=0; inOverflow; i++){ + int iCell = (iOld + pPg->aiOvfl[i]) - iNew; + if( iCell>=0 && iCellaCellIdx[iCell * 2]; + memmove(&pCellptr[2], pCellptr, (nCell - iCell) * 2); + nCell++; + if( pageInsertArray( + pPg, pBegin, &pData, pCellptr, + 1, &apCell[iCell + iNew], &szCell[iCell + iNew] + ) ) goto editpage_fail; } + } - /* Add any overflow cells */ - for(i=0; inOverflow; i++){ - int iCell = (iOld + pPg->aiOvfl[i]) - iNew; - if( iCell>=0 && iCellaCellIdx[iCell * 2]; - int sz = szCell[iCell+iNew]; - memmove(&pCellptr[2], pCellptr, (nCell - iCell) * 2); - pData -= sz; - if( pDataaCellIdx[nCell*2]; + if( pageInsertArray( + pPg, pBegin, &pData, pCellptr, + nNew-nCell, &apCell[iNew+nCell], &szCell[iNew+nCell] + ) ) goto editpage_fail; - /* Append cells to the end of the page */ - pCellptr = &pPg->aCellIdx[nCell*2]; - for(i=iNew+nCell; i<(iNew+nNew); i++){ - pData -= szCell[i]; - if( pDatanCell = nNew; + pPg->nOverflow = 0; - pPg->nFree = nFree; - pPg->nCell = nNew; - pPg->nOverflow = 0; - - put2byte(&aData[hdr+3], pPg->nCell); - put2byte(&aData[hdr+5], pData - aData); + put2byte(&aData[hdr+3], pPg->nCell); + put2byte(&aData[hdr+5], pData - aData); #ifdef SQLITE_DEBUG - for(i=0; iaCellIdx[i*2]); - if( pCell>=aData && pCell<&aData[pPg->pBt->usableSize] ){ - pCell = &pTmp[pCell - aData]; - } - assert( 0==memcmp(pCell, &aData[iOff], szCell[i+iNew]) ); + for(i=0; iaCellIdx[i*2]); + if( pCell>=aData && pCell<&aData[pPg->pBt->usableSize] ){ + pCell = &pTmp[pCell - aData]; } + assert( 0==memcmp(pCell, &aData[iOff], szCell[i+iNew]) ); + } #endif #if 0 - printf("EDIT\n"); +printf("EDIT\n"); #endif - return; - } - + return; editpage_fail: #if 0 printf("REBUILD\n"); @@ -6662,7 +6742,7 @@ static int balance_nonroot( assert( i usableSpace ){ - szNew[k] = subtotal - szCell[i]; + szNew[k] = subtotal - szCell[i] - 2; cntNew[k] = i; if( leafData ){ i--; } subtotal = 0; @@ -7002,6 +7082,7 @@ static int balance_nonroot( editPage(apNew[iPg], iOld, iNew, nNewCell, apCell, szCell); abDone[iPg] = 1; + apNew[iPg]->nFree = usableSpace-szNew[iPg]; assert( apNew[iPg]->nOverflow==0 ); assert( apNew[iPg]->nCell==nNewCell ); } From 8e9ba0c335c5dc263f18f42f5c014233c7ad8976 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 14 Oct 2014 17:27:04 +0000 Subject: [PATCH 005/133] Fix some code duplication issues on this branch. Add minor optimizations to the new code. FossilOrigin-Name: 58d7793bd5d608ba9fc3a2cd44b9d9512e0332ba --- manifest | 12 +- manifest.uuid | 2 +- src/btree.c | 295 +++++++++++++++++++++----------------------------- 3 files changed, 128 insertions(+), 181 deletions(-) diff --git a/manifest b/manifest index f6d9403e16..347e58fbc2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\strunk\schanges\sinto\sthis\sbranch. -D 2014-10-13T18:09:13.737 +C Fix\ssome\scode\sduplication\sissues\son\sthis\sbranch.\sAdd\sminor\soptimizations\sto\sthe\snew\scode. +D 2014-10-14T17:27:04.829 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 3d9d1db1d05724346876cf456a850628a9e75331 +F src/btree.c 3c72c5aa96b1a06ea15da24b82c526e6f14010ab F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 1bd7957161a1346a914f1f09231610e777a8e58d F src/build.c 9dc2bd94347b878c89627000c92b0c8d97ec2919 @@ -1204,7 +1204,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 6594f9b420e2fa642737722ff8521f756ecef227 04892f8ba6c55cec4fe37bfe59b6349fd2a40698 -R 76183a9b7ddeb938a48604dbc4dd8cd6 +P d5b7c5a88dd58de85b3060a1f28b6d96e6e21207 +R ff1ec1f8aba483920a0100e96e95dccf U dan -Z 504cabb1e1fb0c47450f76fec3d8495b +Z 9c750816a0b42cc370356f0c03af45ae diff --git a/manifest.uuid b/manifest.uuid index 32265a861c..065d26bf44 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d5b7c5a88dd58de85b3060a1f28b6d96e6e21207 \ No newline at end of file +58d7793bd5d608ba9fc3a2cd44b9d9512e0332ba \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 7136b45b24..acf466304f 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1224,6 +1224,57 @@ static int defragmentPage(MemPage *pPage){ return SQLITE_OK; } +/* +** Search the free-list on page pPg for space to store a cell nByte bytes in +** size. If one can be found, return a pointer to the space and remove it +** from the free-list. +** +** If no suitable space can be found on the free-list, return NULL. +** +** This function may detect corruption within pPg. If it does and argument +** pRc is non-NULL, then *pRc is set to SQLITE_CORRUPT and NULL is returned. +** Or, if corruption is detected by pRc is NULL, NULL is returned and the +** corruption goes unreported. +*/ +static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){ + const int hdr = pPg->hdrOffset; + u8 * const aData = pPg->aData; + int iAddr; + int pc; + int usableSize = pPg->pBt->usableSize; + + for(iAddr=hdr+1; (pc = get2byte(&aData[iAddr]))>0; iAddr=pc){ + int size; /* Size of the free slot */ + if( pc>usableSize-4 || pc=nByte ){ + int x = size - nByte; + testcase( x==4 ); + testcase( x==3 ); + if( x<4 ){ + if( aData[hdr+7]>=60 ) return 0; + /* Remove the slot from the free-list. Update the number of + ** fragmented bytes within the page. */ + memcpy(&aData[iAddr], &aData[pc], 2); + aData[hdr+7] += (u8)x; + }else if( size+pc > usableSize ){ + if( pRc ) *pRc = SQLITE_CORRUPT_BKPT; + return 0; + }else{ + /* The slot remains on the free-list. Reduce its size to account + ** for the portion used by the new allocation. */ + put2byte(&aData[pc+2], x); + } + return &aData[pc + x]; + } + } + + return 0; +} + /* ** Allocate nByte bytes of space from within the B-Tree page passed ** as the first argument. Write into *pIdx the index into pPage->aData[] @@ -1274,33 +1325,12 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ testcase( gap+1==top ); testcase( gap==top ); if( gap+2<=top && (data[hdr+1] || data[hdr+2]) ){ - int pc, addr; - for(addr=hdr+1; (pc = get2byte(&data[addr]))>0; addr=pc){ - int size; /* Size of the free slot */ - if( pc>usableSize-4 || pc=nByte ){ - int x = size - nByte; - testcase( x==4 ); - testcase( x==3 ); - if( x<4 ){ - if( data[hdr+7]>=60 ) goto defragment_page; - /* Remove the slot from the free-list. Update the number of - ** fragmented bytes within the page. */ - memcpy(&data[addr], &data[pc], 2); - data[hdr+7] += (u8)x; - }else if( size+pc > usableSize ){ - return SQLITE_CORRUPT_BKPT; - }else{ - /* The slot remains on the free-list. Reduce its size to account - ** for the portion used by the new allocation. */ - put2byte(&data[pc+2], x); - } - *pIdx = pc + x; - return SQLITE_OK; - } + int rc = SQLITE_OK; + u8 *pSpace = pageFindSlot(pPage, nByte, &rc); + if( rc ) return rc; + if( pSpace ){ + *pIdx = pSpace - data; + return SQLITE_OK; } } @@ -1309,7 +1339,6 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ */ testcase( gap+2+nByte==top ); if( gap+2+nByte>top ){ -defragment_page: testcase( pPage->nCell==0 ); rc = defragmentPage(pPage); if( rc ) return rc; @@ -5933,48 +5962,18 @@ static void insertCell( } /* -** Add a list of cells to a page. The page should be initially empty. -** The cells are guaranteed to fit on the page. +** Array apCell[] contains pointers to nCell b-tree page cells. The +** szCell[] array contains the size in bytes of each cell. This function +** replaces the current contents of page pPg with the contents of the cell +** array. +** +** Some of the cells in apCell[] may currently be stored in pPg. This +** function works around problems caused by this by making a copy of any +** such cells before overwriting the page data. +** +** The MemPage.nFree field is invalidated by this function. It is the +** responsibility of the caller to set it correctly. */ -static void assemblePage( - MemPage *pPage, /* The page to be assembled */ - int nCell, /* The number of cells to add to this page */ - u8 **apCell, /* Pointers to cell bodies */ - u16 *aSize /* Sizes of the cells */ -){ - int i; /* Loop counter */ - u8 *pCellptr; /* Address of next cell pointer */ - int cellbody; /* Address of next cell body */ - u8 * const data = pPage->aData; /* Pointer to data for pPage */ - const int hdr = pPage->hdrOffset; /* Offset of header on pPage */ - const int nUsable = pPage->pBt->usableSize; /* Usable size of page */ - - assert( pPage->nOverflow==0 ); - assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - assert( nCell>=0 && nCell<=(int)MX_CELL(pPage->pBt) - && (int)MX_CELL(pPage->pBt)<=10921); - assert( sqlite3PagerIswriteable(pPage->pDbPage) ); - - /* Check that the page has just been zeroed by zeroPage() */ - assert( pPage->nCell==0 ); - assert( get2byteNotZero(&data[hdr+5])==nUsable ); - - pCellptr = pPage->aCellIdx; - cellbody = nUsable; - for(i=0; inFree -= (nCell*2 + nUsable - cellbody); - pPage->nCell = (u16)nCell; -} - - static void rebuildPage( MemPage *pPg, /* Edit this page */ int nCell, /* Final number of cells on page */ @@ -5992,8 +5991,8 @@ static void rebuildPage( i = get2byte(&aData[hdr+5]); memcpy(&pTmp[i], &aData[i], usableSize - i); - pData = &aData[usableSize]; + pData = pEnd; for(i=0; iaData && pCellhdrOffset; - u8 * const aData = pPg->aData; - int iAddr; - int pc; - int usableSize = pPg->pBt->usableSize; - - for(iAddr=hdr+1; (pc = get2byte(&aData[iAddr]))>0; iAddr=pc){ - int size; /* Size of the free slot */ - if( pc>usableSize-4 || pc=nByte ){ - int x = size - nByte; - testcase( x==4 ); - testcase( x==3 ); - if( x<4 ){ - if( aData[hdr+7]>=60 ) return 0; - /* Remove the slot from the free-list. Update the number of - ** fragmented bytes within the page. */ - memcpy(&aData[iAddr], &aData[pc], 2); - aData[hdr+7] += (u8)x; - }else if( size+pc > usableSize ){ - return 0; - }else{ - /* The slot remains on the free-list. Reduce its size to account - ** for the portion used by the new allocation. */ - put2byte(&aData[pc+2], x); - } - return &aData[pc + x]; - } - } - - return 0; -} - +/* +** Array apCell[] contains nCell pointers to b-tree cells. Array szCell +** contains the size in bytes of each such cell. This function attempts to +** add the cells stored in the array to page pPg. If it cannot (because +** the page needs to be defragmented before the cells will fit), non-zero +** is returned. Otherwise, if the cells are added successfully, zero is +** returned. +** +** Argument pCellptr points to the first entry in the cell-pointer array +** (part of page pPg) to populate. After cell apCell[0] is written to the +** page body, a 16-bit offset is written to pCellptr. And so on, for each +** cell in the array. It is the responsibility of the caller to ensure +** that it is safe to overwrite this part of the cell-pointer array. +** +** When this function is called, *ppData points to the start of the +** content area on page pPg. If the size of the content area is extended, +** *ppData is updated to point to the new start of the content area +** before returning. +** +** Finally, argument pBegin points to the byte immediately following the +** end of the space required by this page for the cell-pointer area (for +** all cells - not just those inserted by the current call). If the content +** area must be extended to before this point in order to accomodate all +** cells in apCell[], then the cells do not fit and non-zero is returned. +*/ static int pageInsertArray( - MemPage *pPg, - u8 *pBegin, - u8 **ppData, - u8 *pCellptr, - int nCell, + MemPage *pPg, /* Page to add cells to */ + u8 *pBegin, /* End of cell-pointer array */ + u8 **ppData, /* IN/OUT: Page content -area pointer */ + u8 *pCellptr, /* Pointer to cell-pointer area */ + int nCell, /* Number of cells to add to pPg */ u8 **apCell, /* Array of cells */ u16 *szCell /* Array of cell sizes */ ){ int i; u8 *aData = pPg->aData; u8 *pData = *ppData; + const int bFreelist = aData[1] || aData[2]; + assert( pPg->hdrOffset==0 ); /* Never called on page 1 */ for(i=0; inFree field is invalid when this function returns. It is the ** responsibility of the caller to set it correctly. @@ -6206,15 +6205,8 @@ static void editPage( } #endif -#if 0 -printf("EDIT\n"); -#endif - return; editpage_fail: -#if 0 - printf("REBUILD\n"); -#endif /* Unable to edit this page. Rebuild it from scratch instead. */ rebuildPage(pPg, nNew, &apCell[iNew], &szCell[iNew]); } @@ -6288,7 +6280,8 @@ static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){ assert( sqlite3PagerIswriteable(pNew->pDbPage) ); assert( pPage->aData[0]==(PTF_INTKEY|PTF_LEAFDATA|PTF_LEAF) ); zeroPage(pNew, PTF_INTKEY|PTF_LEAFDATA|PTF_LEAF); - assemblePage(pNew, 1, &pCell, &szCell); + rebuildPage(pNew, 1, &pCell, &szCell); + pNew->nFree = pBt->usableSize - pNew->cellOffset - 2 - szCell; /* If this is an auto-vacuum database, update the pointer map ** with entries for the new page, and any pointer from the @@ -6518,8 +6511,6 @@ static int balance_nonroot( u8 *aSpace1; /* Space for copies of dividers cells */ Pgno pgno; /* Temp var to store a page number in */ - int aShiftLeft[NB+2]; - int aShiftRight[NB+2]; u8 abDone[NB+2]; Pgno aPgno[NB+2]; u16 aPgFlags[NB+2]; @@ -6887,49 +6878,6 @@ static int balance_nonroot( assert( sqlite3PagerIswriteable(pParent->pDbPage) ); put4byte(pRight, apNew[nNew-1]->pgno); - j = 0; - for(i=0; i %d@%d", apNew[i]->pgno, - pNew->nCell+pNew->nOverflow, j, - cntNew[i] - iFirst, iFirst - ); - for(iCell=iFirst; iCellaData - || apCell[iCell]>=&pNew->aData[pBt->usableSize] - ){ - nCta += szCell[iCell]; - } - } - nFree = get2byte(&pNew->aData[pNew->hdrOffset+5]); - nFree -= (pNew->cellOffset + (cntNew[i] - iFirst) * 2); - printf(" cta=%d free=%d\n", nCta, nFree); - if( i==(nNew-1) ){ - printf("-----\n"); - fflush(stdout); - } -#endif - - assert( inCell + apNew[i]->nOverflow; - aShiftRight[i] = cntNew[i] - j; - assert( i!=nOld-1 || j==nCell ); - if( j=0 && aShiftLeft[0]==0 ); for(i=0; i=nNew ? i-nNew : nNew-1-i); if( abDone[iPg]==0 - && (aShiftLeft[iPg]>=0 || abDone[iPg-1]) - && (aShiftRight[iPg]>=0 || abDone[iPg+1]) + && (iPg==0 || cntOld[iPg-1]>=cntNew[iPg-1] || abDone[iPg-1]) + && (cntNew[iPg]>=cntOld[iPg] || abDone[iPg+1]) ){ int iNew; int iOld; From c8bbce1e6a373fa7b74165aeddfc5be999cff8d9 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 21 Oct 2014 01:05:09 +0000 Subject: [PATCH 006/133] If a skip-scan is a proper subset of some other scan, then adjust the cost of the skip-scan upward so that it is more costly than the other scan. Such a cost imbalance can arise under STAT4 because of difficulties in getting an accurate estimate for skip-scans. FossilOrigin-Name: f4b22a2620a5dc48949048c2ecbd226755d4b2c3 --- manifest | 15 ++--- manifest.uuid | 2 +- src/where.c | 121 +++++++++++++++++------------------- src/whereInt.h | 4 +- test/skipscan6.test | 145 ++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 212 insertions(+), 75 deletions(-) create mode 100644 test/skipscan6.test diff --git a/manifest b/manifest index 894c7cd51a..3c0097dfe9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\s(probably\sharmless)\sbut\sin\sthe\sCSV\soutput\smode\sof\sthe\scommand-line\nshell. -D 2014-10-17T21:35:05.954 +C If\sa\sskip-scan\sis\sa\sproper\ssubset\sof\ssome\sother\sscan,\sthen\sadjust\sthe\ncost\sof\sthe\sskip-scan\supward\sso\sthat\sit\sis\smore\scostly\sthan\sthe\sother\sscan.\nSuch\sa\scost\simbalance\scan\sarise\sunder\sSTAT4\sbecause\sof\sdifficulties\sin\sgetting\nan\saccurate\sestimate\sfor\sskip-scans. +D 2014-10-21T01:05:09.795 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -302,8 +302,8 @@ F src/vtab.c cb0c194303fea276b48d7d4b6d970b5a96bde8de F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 2947912f1f3d6a7766fe087fd532a5d688d745b1 -F src/whereInt.h 124d970450955a6982e174b07c320ae6d62a595c +F src/where.c 2d1ff51eede0e4dcc87569dc8e3161237295162a +F src/whereInt.h 4b459cdbfc9b01f5f27673a35f9967e4dea917e8 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test b35b4cd69fc913f90d39a575e171e1116c3a4bb7 @@ -844,6 +844,7 @@ F test/skipscan1.test 7e15e1cc524524e7b2c4595ec85c75501d22f4ff F test/skipscan2.test d1d1450952b7275f0b0a3a981f0230532743951a F test/skipscan3.test ec5bab3f81c7038b43450e7b3062e04a198bdbb5 F test/skipscan5.test 67817a4b6857c47e0e33ba3e506da6f23ef68de2 +F test/skipscan6.test 3a891b45d6df266ced861a2ad9d03fca2bc7fcc5 F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24 F test/sort.test c4400e7533748f6bd7413851ff148645e82b9e2d @@ -1204,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P e4ab094f8afce0817f4074e823fabe59fc29ebb4 -R bacb142e035c538b5014a902fee951f4 +P 19fe4a0a475bd94f491031aea7a183f7c0515cf3 +R 7ae877aeccb485915279368cd2f54415 U drh -Z 99accb7d1ae416e10fed302750dd8f99 +Z dece11d7f7de853e09f0182395012c8f diff --git a/manifest.uuid b/manifest.uuid index db419ccbec..ee9b76f602 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -19fe4a0a475bd94f491031aea7a183f7c0515cf3 \ No newline at end of file +f4b22a2620a5dc48949048c2ecbd226755d4b2c3 \ No newline at end of file diff --git a/src/where.c b/src/where.c index bc0110779e..00b1880880 100644 --- a/src/where.c +++ b/src/where.c @@ -2638,7 +2638,7 @@ static int codeAllEqualityTerms( pLoop = pLevel->pWLoop; assert( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 ); nEq = pLoop->u.btree.nEq; - nSkip = pLoop->u.btree.nSkip; + nSkip = pLoop->nSkip; pIdx = pLoop->u.btree.pIndex; assert( pIdx!=0 ); @@ -2752,7 +2752,7 @@ static void explainAppendTerm( static void explainIndexRange(StrAccum *pStr, WhereLoop *pLoop, Table *pTab){ Index *pIndex = pLoop->u.btree.pIndex; u16 nEq = pLoop->u.btree.nEq; - u16 nSkip = pLoop->u.btree.nSkip; + u16 nSkip = pLoop->nSkip; int i, j; Column *aCol = pTab->aCol; i16 *aiColumn = pIndex->aiColumn; @@ -3189,7 +3189,7 @@ static Bitmask codeOneLoopStart( pIdx = pLoop->u.btree.pIndex; iIdxCur = pLevel->iIdxCur; - assert( nEq>=pLoop->u.btree.nSkip ); + assert( nEq>=pLoop->nSkip ); /* If this loop satisfies a sort order (pOrderBy) request that ** was passed to this function to implement a "SELECT min(x) ..." @@ -3206,7 +3206,7 @@ static Bitmask codeOneLoopStart( && pWInfo->nOBSat>0 && (pIdx->nKeyCol>nEq) ){ - assert( pLoop->u.btree.nSkip==0 ); + assert( pLoop->nSkip==0 ); bSeekPastNull = 1; nExtraReg = 1; } @@ -3827,7 +3827,7 @@ static void whereLoopPrint(WhereLoop *p, WhereClause *pWC){ sqlite3_free(z); } if( p->wsFlags & WHERE_SKIPSCAN ){ - sqlite3DebugPrintf(" f %05x %d-%d", p->wsFlags, p->nLTerm,p->u.btree.nSkip); + sqlite3DebugPrintf(" f %05x %d-%d", p->wsFlags, p->nLTerm,p->nSkip); }else{ sqlite3DebugPrintf(" f %05x N %d", p->wsFlags, p->nLTerm); } @@ -3956,12 +3956,15 @@ static int whereLoopCheaperProperSubset( const WhereLoop *pY /* Compare against this WhereLoop */ ){ int i, j; - if( pX->nLTerm >= pY->nLTerm ) return 0; /* X is not a subset of Y */ + if( pX->nLTerm-pX->nSkip >= pY->nLTerm-pY->nSkip ){ + return 0; /* X is not a subset of Y */ + } if( pX->rRun >= pY->rRun ){ if( pX->rRun > pY->rRun ) return 0; /* X costs more than Y */ if( pX->nOut > pY->nOut ) return 0; /* X costs more than Y */ } for(i=pX->nLTerm-1; i>=0; i--){ + if( pX->aLTerm[i]==0 ) continue; for(j=pY->nLTerm-1; j>=0; j--){ if( pY->aLTerm[j]==pX->aLTerm[i] ) break; } @@ -3983,25 +3986,12 @@ static int whereLoopCheaperProperSubset( ** To say "WhereLoop X is a proper subset of Y" means that X uses fewer ** WHERE clause terms than Y and that every WHERE clause term used by X is ** also used by Y. -** -** This adjustment is omitted for SKIPSCAN loops. In a SKIPSCAN loop, the -** WhereLoop.nLTerm field is not an accurate measure of the number of WHERE -** clause terms covered, since some of the first nLTerm entries in aLTerm[] -** will be NULL (because they are skipped). That makes it more difficult -** to compare the loops. We could add extra code to do the comparison, and -** perhaps we will someday. But SKIPSCAN is sufficiently uncommon, and this -** adjustment is sufficient minor, that it is very difficult to construct -** a test case where the extra code would improve the query plan. Better -** to avoid the added complexity and just omit cost adjustments to SKIPSCAN -** loops. */ static void whereLoopAdjustCost(const WhereLoop *p, WhereLoop *pTemplate){ if( (pTemplate->wsFlags & WHERE_INDEXED)==0 ) return; - if( (pTemplate->wsFlags & WHERE_SKIPSCAN)!=0 ) return; for(; p; p=p->pNextLoop){ if( p->iTab!=pTemplate->iTab ) continue; if( (p->wsFlags & WHERE_INDEXED)==0 ) continue; - if( (p->wsFlags & WHERE_SKIPSCAN)!=0 ) continue; if( whereLoopCheaperProperSubset(p, pTemplate) ){ /* Adjust pTemplate cost downward so that it is cheaper than its ** subset p */ @@ -4295,7 +4285,7 @@ static int whereLoopAddBtreeIndex( Bitmask saved_prereq; /* Original value of pNew->prereq */ u16 saved_nLTerm; /* Original value of pNew->nLTerm */ u16 saved_nEq; /* Original value of pNew->u.btree.nEq */ - u16 saved_nSkip; /* Original value of pNew->u.btree.nSkip */ + u16 saved_nSkip; /* Original value of pNew->nSkip */ u32 saved_wsFlags; /* Original value of pNew->wsFlags */ LogEst saved_nOut; /* Original value of pNew->nOut */ int iCol; /* Index of the column in the table */ @@ -4324,7 +4314,7 @@ static int whereLoopAddBtreeIndex( pTerm = whereScanInit(&scan, pBuilder->pWC, pSrc->iCursor, iCol, opMask, pProbe); saved_nEq = pNew->u.btree.nEq; - saved_nSkip = pNew->u.btree.nSkip; + saved_nSkip = pNew->nSkip; saved_nLTerm = pNew->nLTerm; saved_wsFlags = pNew->wsFlags; saved_prereq = pNew->prereq; @@ -4332,44 +4322,6 @@ static int whereLoopAddBtreeIndex( pNew->rSetup = 0; rSize = pProbe->aiRowLogEst[0]; rLogSize = estLog(rSize); - - /* Consider using a skip-scan if there are no WHERE clause constraints - ** available for the left-most terms of the index, and if the average - ** number of repeats in the left-most terms is at least 18. - ** - ** The magic number 18 is selected on the basis that scanning 17 rows - ** is almost always quicker than an index seek (even though if the index - ** contains fewer than 2^17 rows we assume otherwise in other parts of - ** the code). And, even if it is not, it should not be too much slower. - ** On the other hand, the extra seeks could end up being significantly - ** more expensive. */ - assert( 42==sqlite3LogEst(18) ); - if( saved_nEq==saved_nSkip - && saved_nEq+1nKeyCol - && pProbe->aiRowLogEst[saved_nEq+1]>=42 /* TUNING: Minimum for skip-scan */ - && (rc = whereLoopResize(db, pNew, pNew->nLTerm+1))==SQLITE_OK - ){ - LogEst nIter; - pNew->u.btree.nEq++; - pNew->u.btree.nSkip++; - pNew->aLTerm[pNew->nLTerm++] = 0; - pNew->wsFlags |= WHERE_SKIPSCAN; - nIter = pProbe->aiRowLogEst[saved_nEq] - pProbe->aiRowLogEst[saved_nEq+1]; - if( pTerm ){ - /* TUNING: When estimating skip-scan for a term that is also indexable, - ** multiply the cost of the skip-scan by 2.0, to make it a little less - ** desirable than the regular index lookup. */ - nIter += 10; assert( 10==sqlite3LogEst(2) ); - } - pNew->nOut -= nIter; - /* TUNING: Because uncertainties in the estimates for skip-scan queries, - ** add a 1.375 fudge factor to make skip-scan slightly less likely. */ - nIter += 5; - whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nIter + nInMul); - pNew->nOut = saved_nOut; - pNew->u.btree.nEq = saved_nEq; - pNew->u.btree.nSkip = saved_nSkip; - } for(; rc==SQLITE_OK && pTerm!=0; pTerm = whereScanNext(&scan)){ u16 eOp = pTerm->eOperator; /* Shorthand for pTerm->eOperator */ LogEst rCostIdx; @@ -4532,10 +4484,50 @@ static int whereLoopAddBtreeIndex( } pNew->prereq = saved_prereq; pNew->u.btree.nEq = saved_nEq; - pNew->u.btree.nSkip = saved_nSkip; + pNew->nSkip = saved_nSkip; pNew->wsFlags = saved_wsFlags; pNew->nOut = saved_nOut; pNew->nLTerm = saved_nLTerm; + + /* Consider using a skip-scan if there are no WHERE clause constraints + ** available for the left-most terms of the index, and if the average + ** number of repeats in the left-most terms is at least 18. + ** + ** The magic number 18 is selected on the basis that scanning 17 rows + ** is almost always quicker than an index seek (even though if the index + ** contains fewer than 2^17 rows we assume otherwise in other parts of + ** the code). And, even if it is not, it should not be too much slower. + ** On the other hand, the extra seeks could end up being significantly + ** more expensive. */ + assert( 42==sqlite3LogEst(18) ); + if( saved_nEq==saved_nSkip + && saved_nEq+1nKeyCol + && pProbe->aiRowLogEst[saved_nEq+1]>=42 /* TUNING: Minimum for skip-scan */ + && (rc = whereLoopResize(db, pNew, pNew->nLTerm+1))==SQLITE_OK + ){ + LogEst nIter; + pNew->u.btree.nEq++; + pNew->nSkip++; + pNew->aLTerm[pNew->nLTerm++] = 0; + pNew->wsFlags |= WHERE_SKIPSCAN; + nIter = pProbe->aiRowLogEst[saved_nEq] - pProbe->aiRowLogEst[saved_nEq+1]; + if( pTerm ){ + /* TUNING: When estimating skip-scan for a term that is also indexable, + ** multiply the cost of the skip-scan by 2.0, to make it a little less + ** desirable than the regular index lookup. */ + nIter += 10; assert( 10==sqlite3LogEst(2) ); + } + pNew->nOut -= nIter; + /* TUNING: Because uncertainties in the estimates for skip-scan queries, + ** add a 1.375 fudge factor to make skip-scan slightly less likely. */ + nIter += 5; + whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nIter + nInMul); + pNew->nOut = saved_nOut; + pNew->u.btree.nEq = saved_nEq; + pNew->nSkip = saved_nSkip; + pNew->wsFlags = saved_wsFlags; + } + return rc; } @@ -4714,7 +4706,7 @@ static int whereLoopAddBtree( if( pTerm->prereqRight & pNew->maskSelf ) continue; if( termCanDriveIndex(pTerm, pSrc, 0) ){ pNew->u.btree.nEq = 1; - pNew->u.btree.nSkip = 0; + pNew->nSkip = 0; pNew->u.btree.pIndex = 0; pNew->nLTerm = 1; pNew->aLTerm[0] = pTerm; @@ -4755,7 +4747,7 @@ static int whereLoopAddBtree( } rSize = pProbe->aiRowLogEst[0]; pNew->u.btree.nEq = 0; - pNew->u.btree.nSkip = 0; + pNew->nSkip = 0; pNew->nLTerm = 0; pNew->iSortIdx = 0; pNew->rSetup = 0; @@ -5305,7 +5297,7 @@ static i8 wherePathSatisfiesOrderBy( /* Skip over == and IS NULL terms */ if( ju.btree.nEq - && pLoop->u.btree.nSkip==0 + && pLoop->nSkip==0 && ((i = pLoop->aLTerm[j]->eOperator) & (WO_EQ|WO_ISNULL))!=0 ){ if( i & WO_ISNULL ){ @@ -5878,7 +5870,7 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){ pWC = &pWInfo->sWC; pLoop = pBuilder->pNew; pLoop->wsFlags = 0; - pLoop->u.btree.nSkip = 0; + pLoop->nSkip = 0; pTerm = findTerm(pWC, iCur, -1, 0, WO_EQ, 0); if( pTerm ){ pLoop->wsFlags = WHERE_COLUMN_EQ|WHERE_IPK|WHERE_ONEROW; @@ -5890,7 +5882,6 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){ }else{ for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ assert( pLoop->aLTermSpace==pLoop->aLTerm ); - assert( ArraySize(pLoop->aLTermSpace)==4 ); if( !IsUniqueIndex(pIdx) || pIdx->pPartIdxWhere!=0 || pIdx->nKeyCol>ArraySize(pLoop->aLTermSpace) diff --git a/src/whereInt.h b/src/whereInt.h index f17906e63a..e9eb8b7dd2 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -115,7 +115,6 @@ struct WhereLoop { union { struct { /* Information for internal btree tables */ u16 nEq; /* Number of equality constraints */ - u16 nSkip; /* Number of initial index columns to skip */ Index *pIndex; /* Index used, or NULL */ } btree; struct { /* Information for virtual tables */ @@ -128,12 +127,13 @@ struct WhereLoop { } u; u32 wsFlags; /* WHERE_* flags describing the plan */ u16 nLTerm; /* Number of entries in aLTerm[] */ + u16 nSkip; /* Number of NULL aLTerm[] entries */ /**** whereLoopXfer() copies fields above ***********************/ # define WHERE_LOOP_XFER_SZ offsetof(WhereLoop,nLSlot) u16 nLSlot; /* Number of slots allocated for aLTerm[] */ WhereTerm **aLTerm; /* WhereTerms used */ WhereLoop *pNextLoop; /* Next WhereLoop object in the WhereClause */ - WhereTerm *aLTermSpace[4]; /* Initial aLTerm[] space */ + WhereTerm *aLTermSpace[3]; /* Initial aLTerm[] space */ }; /* This object holds the prerequisites and the cost of running a diff --git a/test/skipscan6.test b/test/skipscan6.test new file mode 100644 index 0000000000..9eda9a66f3 --- /dev/null +++ b/test/skipscan6.test @@ -0,0 +1,145 @@ +# 2014-10-21 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file implements tests of the "skip-scan" query strategy. In +# particular, this file verifies that use of all columns of an index +# is always preferred over the use of a skip-scan on some columns of +# the same index. Because of difficulties in scoring a skip-scan, +# the skip-scan can sometimes come out with a lower raw score when +# using STAT4. But the query planner should detect this and use the +# full index rather than the skip-scan. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix skipscan6 + +ifcapable !stat4 { + finish_test + return +} + +do_execsql_test 1.1 { + CREATE TABLE t1( + aa int, + bb int, + cc int, + dd int, + ee int + ); + CREATE INDEX ix on t1(aa, bb, cc, dd DESC); + ANALYZE sqlite_master; + INSERT INTO sqlite_stat1 VALUES('t1','ix','2695116 1347558 264 18 2'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 196859 196859 32 1','0 15043 15043 92468 92499','0 19 286 81846 92499',X'0609010804031552977BD725BD28'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 14687 161 1 1','0 289067 299306 299457 299457','0 199 6772 273984 299457',X'060902020403013406314D67456415B819'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 19313 19308 22 1','0 325815 325815 343725 343746','0 261 9545 315009 343746',X'060902080403018A49B0A3AD1ED931'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 25047 9051 15 1','0 350443 350443 356590 356604','0 266 9795 325519 356604',X'06090208040301914C2DD2E91F93CF'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 42327 9906 7 1','0 376381 376381 380291 380297','0 268 10100 344232 380297',X'06090208040301934BF672511F7ED3'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 24513 2237 1 1','0 455150 467779 470015 470015','0 286 10880 425401 470015',X'06090202040301A703464A28F2611EF1EE'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 18730 18724 15 1','0 479663 479663 498271 498285','0 287 10998 450793 498285',X'06090208040301A8494AF3A41EC50C'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 119603 47125 1 1','0 572425 572425 598915 598915','0 404 14230 546497 598915',X'06090208040302474FD1929A03194F'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 1454 1454 1 1','0 898346 898346 898373 898373','0 952 31165 827562 898373',X'06090208040304FD53F6A2A2097F64'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 57138 7069 1 1','0 1122389 1122389 1129457 1129457','0 1967 46801 1045943 1129457',X'06090208040309884BC4C52F1F6EB7'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 285 11 1 1','0 1197683 1197824 1197831 1197831','0 2033 50990 1112280 1197831',X'06090202040309D80346503FE2A9038E4F'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 25365 9773 1 1','0 1301013 1301013 1310785 1310785','0 2561 58806 1217877 1310785',X'0609020804030C5F4C8F88AB0AF2A2'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 45180 7222 1 1','0 1326378 1326378 1333599 1333599','0 2562 59921 1240187 1333599',X'0609020804030C604CAB75490B0351'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 8537 41 1 1','0 1496959 1497288 1497289 1497289','0 3050 68246 1394126 1497289',X'0609020204030EA0057F527459B0257C4B'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 26139 26131 17 1','0 1507977 1507977 1520578 1520594','0 3074 69188 1416111 1520594',X'0609020804030EB95169453423D4EA'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 102894 29678 1 1','0 1537421 1550467 1564894 1564894','0 3109 69669 1459820 1564894',X'0609020204030EE3183652A6ED3006EBCB'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 319 3 1 1','0 1796728 1796746 1796747 1796747','0 3650 86468 1682243 1796747',X'0609020204031163033550D0C41018C28D'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 127 127 1 1','0 2096194 2096194 2096205 2096205','0 5145 106437 1951535 2096205',X'060902080403180F53BB1AF727EE50'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 66574 5252 1 1','0 2230524 2265961 2271212 2271212','0 5899 114976 2085829 2271212',X'0609020204031B8A05195009976D223B90'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 19440 19440 1 1','0 2391680 2391680 2395663 2395663','0 6718 123714 2184781 2395663',X'0609020804031F7452E00A7B07431A'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 18321 2177 1 1','0 2522928 2523231 2525407 2525407','0 7838 139084 2299958 2525407',X'06090201040324A7475231103B1AA7B8'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 22384 1361 1 1','0 2541249 2544834 2546194 2546194','0 7839 139428 2308416 2546194',X'06090202040324A8011652323D4B1AA9EB'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 18699 855 1 1','0 2563633 2578178 2579032 2579032','0 7840 139947 2321671 2579032',X'06090202040324A9077452323D7D1052C5'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','17965 1579 1579 1 1','2677151 2690666 2690666 2692244 2692244','1 9870 153959 2418294 2692244',X'060102080403021B8A4FE1AB84032B35'); + ANALYZE sqlite_master; +} {} +do_execsql_test 1.2 { + EXPLAIN QUERY PLAN + SELECT COUNT(*) + FROM t1 + WHERE bb=21 + AND aa=1 + AND dd BETWEEN 1413833728 and 1413837331; +} {/INDEX ix .aa=. AND bb=../} + +do_execsql_test 2.1 { + DROP INDEX ix; + CREATE INDEX good on t1(bb, aa, dd DESC); + CREATE INDEX bad on t1(aa, bb, cc, dd DESC); + DELETE FROM sqlite_stat1; + DELETE FROM sqlite_stat4; + INSERT INTO sqlite_stat1 VALUES('t1','good','2695116 299 264 2'); + INSERT INTO sqlite_stat1 VALUES('t1','bad','2695116 1347558 264 18 2'); + INSERT INTO sqlite_stat4 VALUES('t1','good','197030 196859 32 1','15086 15086 92511 92536','19 25 81644 92536',X'05010904031552977BD725BD22'); + INSERT INTO sqlite_stat4 VALUES('t1','good','14972 14687 1 1','289878 289878 299457 299457','199 244 267460 299457',X'050209040301344F7E569402C419'); + INSERT INTO sqlite_stat4 VALUES('t1','good','19600 19313 22 1','327127 327127 346222 346243','261 319 306884 346243',X'0502090403018A49503BC01EC577'); + INSERT INTO sqlite_stat4 VALUES('t1','good','25666 25047 15 1','352087 352087 372692 372706','266 327 325601 372706',X'050209040301914C2DD2E91F93CF'); + INSERT INTO sqlite_stat4 VALUES('t1','good','42392 42327 26 1','378657 378657 382547 382572','268 331 333529 382572',X'05020904030193533B2FE326ED48'); + INSERT INTO sqlite_stat4 VALUES('t1','good','24619 24513 11 1','457872 457872 461748 461758','286 358 399322 461758',X'050209040301A752B1557825EA7C'); + INSERT INTO sqlite_stat4 VALUES('t1','good','18969 18730 15 1','482491 482491 501105 501119','287 360 433605 501119',X'050209040301A8494AF3A41EC50C'); + INSERT INTO sqlite_stat4 VALUES('t1','good','119710 119603 1 1','576500 576500 598915 598915','404 505 519877 598915',X'05020904030247539A7A7912F617'); + INSERT INTO sqlite_stat4 VALUES('t1','good','11955 11946 1 1','889796 889796 898373 898373','938 1123 794694 898373',X'050209040304EF4DF9C4150BBB28'); + INSERT INTO sqlite_stat4 VALUES('t1','good','57197 57138 24 1','1129865 1129865 1151492 1151515','1967 2273 1027048 1151515',X'05020904030988533510BC26E20A'); + INSERT INTO sqlite_stat4 VALUES('t1','good','3609 3543 1 1','1196265 1196265 1197831 1197831','2002 2313 1070108 1197831',X'050209040309B050E95CD718D94D'); + INSERT INTO sqlite_stat4 VALUES('t1','good','25391 25365 13 1','1309378 1309378 1315567 1315579','2561 2936 1178358 1315579',X'05020904030C5F53DF9E13283570'); + INSERT INTO sqlite_stat4 VALUES('t1','good','45232 45180 17 1','1334769 1334769 1337946 1337962','2562 2938 1198998 1337962',X'05020904030C60541CACEE28BCAC'); + INSERT INTO sqlite_stat4 VALUES('t1','good','5496 5493 1 1','1495882 1495882 1497289 1497289','3043 3479 1348695 1497289',X'05020904030E99515C62AD0F0B34'); + INSERT INTO sqlite_stat4 VALUES('t1','good','26348 26139 17 1','1517381 1517381 1529990 1530006','3074 3519 1378320 1530006',X'05020904030EB95169453423D4EA'); + INSERT INTO sqlite_stat4 VALUES('t1','good','102927 102894 10 1','1547088 1547088 1649950 1649959','3109 3559 1494260 1649959',X'05020904030EE34D309F671FFA47'); + INSERT INTO sqlite_stat4 VALUES('t1','good','3602 3576 1 1','1793873 1793873 1796747 1796747','3601 4128 1630783 1796747',X'050209040311294FE88B432219B9'); + INSERT INTO sqlite_stat4 VALUES('t1','good','154 154 1 1','2096059 2096059 2096205 2096205','5037 5779 1893039 2096205',X'050209040317994EFF05A016DCED'); + INSERT INTO sqlite_stat4 VALUES('t1','good','68153 66574 60 1','2244039 2244039 2268892 2268951','5899 6749 2027553 2268951',X'05020904031B8A532DBC5A26D2BA'); + INSERT INTO sqlite_stat4 VALUES('t1','good','321 321 1 1','2395618 2395618 2395663 2395663','6609 7528 2118435 2395663',X'05020904031EFA54078EEE1E2D65'); + INSERT INTO sqlite_stat4 VALUES('t1','good','19449 19440 22 1','2407769 2407769 2426049 2426070','6718 7651 2146904 2426070',X'05020904031F7450E6118C2336BD'); + INSERT INTO sqlite_stat4 VALUES('t1','good','18383 18321 56 1','2539949 2539949 2551080 2551135','7838 8897 2245459 2551135',X'050209040324A752EA2E1E2642B2'); + INSERT INTO sqlite_stat4 VALUES('t1','good','22479 22384 60 1','2558332 2558332 2565233 2565292','7839 8899 2251202 2565292',X'050209040324A853926538279A5F'); + INSERT INTO sqlite_stat4 VALUES('t1','good','18771 18699 63 1','2580811 2580811 2596914 2596976','7840 8901 2263572 2596976',X'050209040324A9526C1DE9256E72'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 196859 196859 32 1','0 15043 15043 92468 92499','0 19 286 81846 92499',X'0609010804031552977BD725BD28'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 14687 161 1 1','0 289067 299306 299457 299457','0 199 6772 273984 299457',X'060902020403013406314D67456415B819'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 19313 19308 22 1','0 325815 325815 343725 343746','0 261 9545 315009 343746',X'060902080403018A49B0A3AD1ED931'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 25047 9051 15 1','0 350443 350443 356590 356604','0 266 9795 325519 356604',X'06090208040301914C2DD2E91F93CF'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 42327 9906 7 1','0 376381 376381 380291 380297','0 268 10100 344232 380297',X'06090208040301934BF672511F7ED3'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 24513 2237 1 1','0 455150 467779 470015 470015','0 286 10880 425401 470015',X'06090202040301A703464A28F2611EF1EE'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 18730 18724 15 1','0 479663 479663 498271 498285','0 287 10998 450793 498285',X'06090208040301A8494AF3A41EC50C'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 119603 47125 1 1','0 572425 572425 598915 598915','0 404 14230 546497 598915',X'06090208040302474FD1929A03194F'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 1454 1454 1 1','0 898346 898346 898373 898373','0 952 31165 827562 898373',X'06090208040304FD53F6A2A2097F64'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 57138 7069 1 1','0 1122389 1122389 1129457 1129457','0 1967 46801 1045943 1129457',X'06090208040309884BC4C52F1F6EB7'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 285 11 1 1','0 1197683 1197824 1197831 1197831','0 2033 50990 1112280 1197831',X'06090202040309D80346503FE2A9038E4F'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 25365 9773 1 1','0 1301013 1301013 1310785 1310785','0 2561 58806 1217877 1310785',X'0609020804030C5F4C8F88AB0AF2A2'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 45180 7222 1 1','0 1326378 1326378 1333599 1333599','0 2562 59921 1240187 1333599',X'0609020804030C604CAB75490B0351'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 8537 41 1 1','0 1496959 1497288 1497289 1497289','0 3050 68246 1394126 1497289',X'0609020204030EA0057F527459B0257C4B'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 26139 26131 17 1','0 1507977 1507977 1520578 1520594','0 3074 69188 1416111 1520594',X'0609020804030EB95169453423D4EA'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 102894 29678 1 1','0 1537421 1550467 1564894 1564894','0 3109 69669 1459820 1564894',X'0609020204030EE3183652A6ED3006EBCB'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 319 3 1 1','0 1796728 1796746 1796747 1796747','0 3650 86468 1682243 1796747',X'0609020204031163033550D0C41018C28D'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 127 127 1 1','0 2096194 2096194 2096205 2096205','0 5145 106437 1951535 2096205',X'060902080403180F53BB1AF727EE50'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 66574 5252 1 1','0 2230524 2265961 2271212 2271212','0 5899 114976 2085829 2271212',X'0609020204031B8A05195009976D223B90'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 19440 19440 1 1','0 2391680 2391680 2395663 2395663','0 6718 123714 2184781 2395663',X'0609020804031F7452E00A7B07431A'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 18321 2177 1 1','0 2522928 2523231 2525407 2525407','0 7838 139084 2299958 2525407',X'06090201040324A7475231103B1AA7B8'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 22384 1361 1 1','0 2541249 2544834 2546194 2546194','0 7839 139428 2308416 2546194',X'06090202040324A8011652323D4B1AA9EB'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 18699 855 1 1','0 2563633 2578178 2579032 2579032','0 7840 139947 2321671 2579032',X'06090202040324A9077452323D7D1052C5'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','17965 1579 1579 1 1','2677151 2690666 2690666 2692244 2692244','1 9870 153959 2418294 2692244',X'060102080403021B8A4FE1AB84032B35'); + ANALYZE sqlite_master; +} {} +do_execsql_test 2.2 { + EXPLAIN QUERY PLAN + SELECT COUNT(*) + FROM t1 + WHERE bb=21 + AND aa=1 + AND dd BETWEEN 1413833728 and 1413837331; +} {/INDEX good .bb=. AND aa=. AND dd>. AND dd<../} + + + +finish_test From 1b131b7a7acc83c7bd1a5e7d0872c968d9e26489 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 21 Oct 2014 16:01:40 +0000 Subject: [PATCH 007/133] Improvements to the WHERETRACE debugging logic. FossilOrigin-Name: ec1e942f08548695ff02645b3f3cd6bb2516bc9a --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 6 +++++- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 3c0097dfe9..d708dca4f3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C If\sa\sskip-scan\sis\sa\sproper\ssubset\sof\ssome\sother\sscan,\sthen\sadjust\sthe\ncost\sof\sthe\sskip-scan\supward\sso\sthat\sit\sis\smore\scostly\sthan\sthe\sother\sscan.\nSuch\sa\scost\simbalance\scan\sarise\sunder\sSTAT4\sbecause\sof\sdifficulties\sin\sgetting\nan\saccurate\sestimate\sfor\sskip-scans. -D 2014-10-21T01:05:09.795 +C Improvements\sto\sthe\sWHERETRACE\sdebugging\slogic. +D 2014-10-21T16:01:40.774 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -302,7 +302,7 @@ F src/vtab.c cb0c194303fea276b48d7d4b6d970b5a96bde8de F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 2d1ff51eede0e4dcc87569dc8e3161237295162a +F src/where.c 45cb63cb1422d7e5a9229c297e978d294ae51e16 F src/whereInt.h 4b459cdbfc9b01f5f27673a35f9967e4dea917e8 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 19fe4a0a475bd94f491031aea7a183f7c0515cf3 -R 7ae877aeccb485915279368cd2f54415 +P f4b22a2620a5dc48949048c2ecbd226755d4b2c3 +R 59b7c6284af5e31186eaa6edb85f9e59 U drh -Z dece11d7f7de853e09f0182395012c8f +Z 3eaff9a0b8f4f97b5dd119bc1e1acc8b diff --git a/manifest.uuid b/manifest.uuid index ee9b76f602..838729f698 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f4b22a2620a5dc48949048c2ecbd226755d4b2c3 \ No newline at end of file +ec1e942f08548695ff02645b3f3cd6bb2516bc9a \ No newline at end of file diff --git a/src/where.c b/src/where.c index 00b1880880..7dfe0f02bb 100644 --- a/src/where.c +++ b/src/where.c @@ -3995,11 +3995,15 @@ static void whereLoopAdjustCost(const WhereLoop *p, WhereLoop *pTemplate){ if( whereLoopCheaperProperSubset(p, pTemplate) ){ /* Adjust pTemplate cost downward so that it is cheaper than its ** subset p */ + WHERETRACE(0x80,("subset cost adjustment %d,%d to %d,%d\n", + pTemplate->rRun, pTemplate->nOut, p->rRun, p->nOut-1)); pTemplate->rRun = p->rRun; pTemplate->nOut = p->nOut - 1; }else if( whereLoopCheaperProperSubset(pTemplate, p) ){ /* Adjust pTemplate cost upward so that it is costlier than p since ** pTemplate is a proper subset of p */ + WHERETRACE(0x80,("subset cost adjustment %d,%d to %d,%d\n", + pTemplate->rRun, pTemplate->nOut, p->rRun, p->nOut+1)); pTemplate->rRun = p->rRun; pTemplate->nOut = p->nOut + 1; } @@ -5751,7 +5755,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ } #ifdef WHERETRACE_ENABLED /* >=2 */ - if( sqlite3WhereTrace>=2 ){ + if( sqlite3WhereTrace & 0x02 ){ sqlite3DebugPrintf("---- after round %d ----\n", iLoop); for(ii=0, pTo=aTo; ii Date: Tue, 21 Oct 2014 18:16:21 +0000 Subject: [PATCH 008/133] Further tuning of the cost estimates for skip-scan loops, especially for cases when skip-scan loops are in competition with regular loops. FossilOrigin-Name: a27861c28c4791e51d797aa37e9cca806cb58775 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 10 +++------- 3 files changed, 10 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index d708dca4f3..3daca49fdd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improvements\sto\sthe\sWHERETRACE\sdebugging\slogic. -D 2014-10-21T16:01:40.774 +C Further\stuning\sof\sthe\scost\sestimates\sfor\sskip-scan\sloops,\sespecially\sfor\scases\nwhen\sskip-scan\sloops\sare\sin\scompetition\swith\sregular\sloops. +D 2014-10-21T18:16:21.388 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -302,7 +302,7 @@ F src/vtab.c cb0c194303fea276b48d7d4b6d970b5a96bde8de F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 45cb63cb1422d7e5a9229c297e978d294ae51e16 +F src/where.c 994b38c8697aad095878ef1e4860902df457427f F src/whereInt.h 4b459cdbfc9b01f5f27673a35f9967e4dea917e8 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f4b22a2620a5dc48949048c2ecbd226755d4b2c3 -R 59b7c6284af5e31186eaa6edb85f9e59 +P ec1e942f08548695ff02645b3f3cd6bb2516bc9a +R 4f5d15d10f268e58d7c047d490bf8adc U drh -Z 3eaff9a0b8f4f97b5dd119bc1e1acc8b +Z 7d7f3f0271fe8b8df2863a2c22053442 diff --git a/manifest.uuid b/manifest.uuid index 838729f698..52d4aafd99 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ec1e942f08548695ff02645b3f3cd6bb2516bc9a \ No newline at end of file +a27861c28c4791e51d797aa37e9cca806cb58775 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 7dfe0f02bb..7f51d00c87 100644 --- a/src/where.c +++ b/src/where.c @@ -3994,7 +3994,9 @@ static void whereLoopAdjustCost(const WhereLoop *p, WhereLoop *pTemplate){ if( (p->wsFlags & WHERE_INDEXED)==0 ) continue; if( whereLoopCheaperProperSubset(p, pTemplate) ){ /* Adjust pTemplate cost downward so that it is cheaper than its - ** subset p */ + ** subset p. Except, do not adjust the cost estimate downward for + ** a loop that skips more columns. */ + if( pTemplate->nSkip>p->nSkip ) continue; WHERETRACE(0x80,("subset cost adjustment %d,%d to %d,%d\n", pTemplate->rRun, pTemplate->nOut, p->rRun, p->nOut-1)); pTemplate->rRun = p->rRun; @@ -4515,12 +4517,6 @@ static int whereLoopAddBtreeIndex( pNew->aLTerm[pNew->nLTerm++] = 0; pNew->wsFlags |= WHERE_SKIPSCAN; nIter = pProbe->aiRowLogEst[saved_nEq] - pProbe->aiRowLogEst[saved_nEq+1]; - if( pTerm ){ - /* TUNING: When estimating skip-scan for a term that is also indexable, - ** multiply the cost of the skip-scan by 2.0, to make it a little less - ** desirable than the regular index lookup. */ - nIter += 10; assert( 10==sqlite3LogEst(2) ); - } pNew->nOut -= nIter; /* TUNING: Because uncertainties in the estimates for skip-scan queries, ** add a 1.375 fudge factor to make skip-scan slightly less likely. */ From 442c5cd3cfc67d3e10aa64d9f180ef94fb3597cf Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 21 Oct 2014 21:56:06 +0000 Subject: [PATCH 009/133] Call fsync() right after ftruncate() when in journal_mode=TRUNCATE and when synchronous=FULL in order to ensure that transactions are durable across a power loss that happens moments after the commit. Proposed fix for [https://bugzilla.mozilla.org/show_bug.cgi?id=1072773]. FossilOrigin-Name: 3e922208b68563489c7766abb9afb4885113e7b8 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pager.c | 8 ++++++++ 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 3daca49fdd..f4a8a2a556 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\stuning\sof\sthe\scost\sestimates\sfor\sskip-scan\sloops,\sespecially\sfor\scases\nwhen\sskip-scan\sloops\sare\sin\scompetition\swith\sregular\sloops. -D 2014-10-21T18:16:21.388 +C Call\sfsync()\sright\safter\sftruncate()\swhen\sin\sjournal_mode=TRUNCATE\sand\nwhen\ssynchronous=FULL\sin\sorder\sto\sensure\sthat\stransactions\sare\sdurable\nacross\sa\spower\sloss\sthat\shappens\smoments\safter\sthe\scommit.\s\sProposed\nfix\sfor\s[https://bugzilla.mozilla.org/show_bug.cgi?id=1072773]. +D 2014-10-21T21:56:06.890 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -215,7 +215,7 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c fb587121840f690101336879adfa6d0b2cd0e8c7 F src/os_win.c a019caaae2bcbbc0cc4c39af6e7d7e43d8426053 F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 -F src/pager.c a171cf9dd09c6cb162b262c328d4dfd198e04f80 +F src/pager.c a98547ad9b1b5dbbc5e7d1c520be041b5d2c0926 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 F src/parse.y 5dfead8aed90cb0c7c1115898ee2266804daff45 F src/pcache.c 4121a0571c18581ee9f82f086d5e2030051ebd6a @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ec1e942f08548695ff02645b3f3cd6bb2516bc9a -R 4f5d15d10f268e58d7c047d490bf8adc +P a27861c28c4791e51d797aa37e9cca806cb58775 +R 1375151e695773da263f40095fae22af U drh -Z 7d7f3f0271fe8b8df2863a2c22053442 +Z be5219411fed69e2fe0e0626925bcd5e diff --git a/manifest.uuid b/manifest.uuid index 52d4aafd99..db15d6664e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a27861c28c4791e51d797aa37e9cca806cb58775 \ No newline at end of file +3e922208b68563489c7766abb9afb4885113e7b8 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index d3a36ef484..d840a39a15 100644 --- a/src/pager.c +++ b/src/pager.c @@ -1941,6 +1941,14 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){ rc = SQLITE_OK; }else{ rc = sqlite3OsTruncate(pPager->jfd, 0); + if( rc==SQLITE_OK && pPager->fullSync ){ + /* Make sure the new file size is written into the inode right away. + ** Otherwise the journal might resurrect following a power loss and + ** cause the last transaction to roll back. See + ** https://bugzilla.mozilla.org/show_bug.cgi?id=1072773 + */ + rc = sqlite3OsSync(pPager->jfd, pPager->syncFlags); + } } pPager->journalOff = 0; }else if( pPager->journalMode==PAGER_JOURNALMODE_PERSIST From 9f07cf7b2e43548f60bd7497dafba30856df3531 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 22 Oct 2014 15:27:05 +0000 Subject: [PATCH 010/133] Take steps to avoid misestimating range query costs based on STAT4 data due to the roundoff error of converting from integers to LogEst and back to integers. FossilOrigin-Name: 3c933bf95f291f7957580d823dce92c981375a5c --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/analyze.c | 1 + src/sqliteInt.h | 3 ++- src/where.c | 2 +- 5 files changed, 13 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index f4a8a2a556..8290b14522 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Call\sfsync()\sright\safter\sftruncate()\swhen\sin\sjournal_mode=TRUNCATE\sand\nwhen\ssynchronous=FULL\sin\sorder\sto\sensure\sthat\stransactions\sare\sdurable\nacross\sa\spower\sloss\sthat\shappens\smoments\safter\sthe\scommit.\s\sProposed\nfix\sfor\s[https://bugzilla.mozilla.org/show_bug.cgi?id=1072773]. -D 2014-10-21T21:56:06.890 +C Take\ssteps\sto\savoid\smisestimating\srange\squery\scosts\sbased\son\sSTAT4\sdata\ndue\sto\sthe\sroundoff\serror\sof\sconverting\sfrom\sintegers\sto\sLogEst\sand\sback\nto\sintegers. +D 2014-10-22T15:27:05.734 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -166,7 +166,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 3d8b83c91651f53472ca17599dae3457b8b89494 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c ba266a779bc7ce10e52e59e7d3dc79fa342e8fdb -F src/analyze.c 8c322e1ecc08909526dbd5ab4421889d05f2263d +F src/analyze.c 567c94b763b67f7abda06dbf0ba34b0343ed9447 F src/attach.c f4e94df2d1826feda65eb0939f7f6f5f923a0ad9 F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e @@ -232,7 +232,7 @@ F src/shell.c 282f8f5278e0c78eb442217531172ec9e1538796 F src/sqlite.h.in 4a5e5158c189d2bcd45c7c4607c2c0eb6d25c153 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d -F src/sqliteInt.h f7812f74f2d0c6041ef6b91a99c5a45f775dd408 +F src/sqliteInt.h d6d423b0f62846eb441236bc15417aeede2ebbdc F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 961d5926e5a8fda611d385ec22c226b8635cd1cb F src/table.c 2e99ef7ef16187e17033d9398dc962ce22dab5cb @@ -302,7 +302,7 @@ F src/vtab.c cb0c194303fea276b48d7d4b6d970b5a96bde8de F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 994b38c8697aad095878ef1e4860902df457427f +F src/where.c 5099c42e24c63969b3cf3b52e18c1a36cb841a34 F src/whereInt.h 4b459cdbfc9b01f5f27673a35f9967e4dea917e8 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P a27861c28c4791e51d797aa37e9cca806cb58775 -R 1375151e695773da263f40095fae22af +P 3e922208b68563489c7766abb9afb4885113e7b8 +R 5c508733aebac553de79291a7eabd709 U drh -Z be5219411fed69e2fe0e0626925bcd5e +Z a3d7104c7f173585d7e788d6453aa065 diff --git a/manifest.uuid b/manifest.uuid index db15d6664e..b84af28cee 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3e922208b68563489c7766abb9afb4885113e7b8 \ No newline at end of file +3c933bf95f291f7957580d823dce92c981375a5c \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 7d36f01318..67bba1a28a 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1599,6 +1599,7 @@ static void initAvgEq(Index *pIdx){ nRow = pIdx->aiRowEst[0]; nDist100 = ((i64)100 * pIdx->aiRowEst[0]) / pIdx->aiRowEst[iCol+1]; } + pIdx->nRowEst0 = nRow; /* Set nSum to the number of distinct (iCol+1) field prefixes that ** occur in the stat4 table for this index. Set sumEq to the sum of diff --git a/src/sqliteInt.h b/src/sqliteInt.h index cba89b03e7..5409f7086f 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1801,7 +1801,8 @@ struct Index { int nSampleCol; /* Size of IndexSample.anEq[] and so on */ tRowcnt *aAvgEq; /* Average nEq values for keys not in aSample */ IndexSample *aSample; /* Samples of the left-most key */ - tRowcnt *aiRowEst; /* Non-logarithmic stat1 data for this table */ + tRowcnt *aiRowEst; /* Non-logarithmic stat1 data for this index */ + tRowcnt nRowEst0; /* Non-logarithmic number of rows in the index */ #endif }; diff --git a/src/where.c b/src/where.c index 7f51d00c87..f708d7fad7 100644 --- a/src/where.c +++ b/src/where.c @@ -2198,7 +2198,7 @@ static int whereRangeScanEst( /* Determine iLower and iUpper using ($P) only. */ if( nEq==0 ){ iLower = 0; - iUpper = sqlite3LogEstToInt(p->aiRowLogEst[0]); + iUpper = p->nRowEst0; }else{ /* Note: this call could be optimized away - since the same values must ** have been requested when testing key $P in whereEqualScanEst(). */ From d7d71470226bcc59799713b95ad9e10f9eff63cc Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 22 Oct 2014 19:57:16 +0000 Subject: [PATCH 011/133] Change the 0x800 bit of SQLITE_TESTCTRL_OPTIMIZATIONS so that it disables the loading of STAT3 and STAT4 content, not just the using of that content. Change the internal name of that bit to SQLITE_Stat34. FossilOrigin-Name: ca3b00c44ec52d209642a5ba9ef82e085fac39db --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/analyze.c | 2 +- src/sqliteInt.h | 2 +- src/test1.c | 3 ++- src/where.c | 4 +--- 6 files changed, 15 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 8290b14522..0f41135502 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Take\ssteps\sto\savoid\smisestimating\srange\squery\scosts\sbased\son\sSTAT4\sdata\ndue\sto\sthe\sroundoff\serror\sof\sconverting\sfrom\sintegers\sto\sLogEst\sand\sback\nto\sintegers. -D 2014-10-22T15:27:05.734 +C Change\sthe\s0x800\sbit\sof\sSQLITE_TESTCTRL_OPTIMIZATIONS\sso\sthat\sit\sdisables\nthe\sloading\sof\sSTAT3\sand\sSTAT4\scontent,\snot\sjust\sthe\susing\sof\sthat\scontent.\nChange\sthe\sinternal\sname\sof\sthat\sbit\sto\sSQLITE_Stat34. +D 2014-10-22T19:57:16.520 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -166,7 +166,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 3d8b83c91651f53472ca17599dae3457b8b89494 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c ba266a779bc7ce10e52e59e7d3dc79fa342e8fdb -F src/analyze.c 567c94b763b67f7abda06dbf0ba34b0343ed9447 +F src/analyze.c afbcca663c3f3625340b8e30d440cd7a97ded6bc F src/attach.c f4e94df2d1826feda65eb0939f7f6f5f923a0ad9 F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e @@ -232,12 +232,12 @@ F src/shell.c 282f8f5278e0c78eb442217531172ec9e1538796 F src/sqlite.h.in 4a5e5158c189d2bcd45c7c4607c2c0eb6d25c153 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d -F src/sqliteInt.h d6d423b0f62846eb441236bc15417aeede2ebbdc +F src/sqliteInt.h 6e9e125698c1e5c78a51050ea61f179a281c766d F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 961d5926e5a8fda611d385ec22c226b8635cd1cb F src/table.c 2e99ef7ef16187e17033d9398dc962ce22dab5cb F src/tclsqlite.c c67d310c833046cccc192125d64ad422ab882684 -F src/test1.c 518db4305d76b29dd9da3f022ca899c8fcdf9fc7 +F src/test1.c 63d4b1707c4052cf9c05c1cbb4a62666d70a0b48 F src/test2.c 98049e51a17dc62606a99a9eb95ee477f9996712 F src/test3.c 1c0e5d6f080b8e33c1ce8b3078e7013fdbcd560c F src/test4.c 9b32d22f5f150abe23c1830e2057c4037c45b3df @@ -302,7 +302,7 @@ F src/vtab.c cb0c194303fea276b48d7d4b6d970b5a96bde8de F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 5099c42e24c63969b3cf3b52e18c1a36cb841a34 +F src/where.c 2cd9e0af718d736459ae9d3b0f4532b4a80640d0 F src/whereInt.h 4b459cdbfc9b01f5f27673a35f9967e4dea917e8 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 3e922208b68563489c7766abb9afb4885113e7b8 -R 5c508733aebac553de79291a7eabd709 +P 3c933bf95f291f7957580d823dce92c981375a5c +R 3efbbff265eea57d45c364e4606463ea U drh -Z a3d7104c7f173585d7e788d6453aa065 +Z b4247f0ab9b19770a6354621b6169ee4 diff --git a/manifest.uuid b/manifest.uuid index b84af28cee..fff2e6a3fa 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3c933bf95f291f7957580d823dce92c981375a5c \ No newline at end of file +ca3b00c44ec52d209642a5ba9ef82e085fac39db \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 67bba1a28a..597885237c 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1861,7 +1861,7 @@ int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ /* Load the statistics from the sqlite_stat4 table. */ #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 - if( rc==SQLITE_OK ){ + if( rc==SQLITE_OK && OptimizationEnabled(db, SQLITE_Stat34) ){ int lookasideEnabled = db->lookaside.bEnabled; db->lookaside.bEnabled = 0; rc = loadStat4(db, sInfo.zDatabase); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 5409f7086f..1b3138be44 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1214,7 +1214,7 @@ struct sqlite3 { #define SQLITE_SubqCoroutine 0x0100 /* Evaluate subqueries as coroutines */ #define SQLITE_Transitive 0x0200 /* Transitive constraints */ #define SQLITE_OmitNoopJoin 0x0400 /* Omit unused tables in joins */ -#define SQLITE_Stat3 0x0800 /* Use the SQLITE_STAT3 table */ +#define SQLITE_Stat34 0x0800 /* Use STAT3 or STAT4 data */ #define SQLITE_AllOpts 0xffff /* All optimizations */ /* diff --git a/src/test1.c b/src/test1.c index 85a16488ba..802aa99b5b 100644 --- a/src/test1.c +++ b/src/test1.c @@ -6293,7 +6293,8 @@ static int optimization_control( { "transitive", SQLITE_Transitive }, { "subquery-coroutine", SQLITE_SubqCoroutine }, { "omit-noop-join", SQLITE_OmitNoopJoin }, - { "stat3", SQLITE_Stat3 }, + { "stat3", SQLITE_Stat34 }, + { "stat4", SQLITE_Stat34 }, }; if( objc!=4 ){ diff --git a/src/where.c b/src/where.c index f708d7fad7..198a6c3fdd 100644 --- a/src/where.c +++ b/src/where.c @@ -1350,7 +1350,7 @@ static void exprAnalyze( if( pExpr->op==TK_NOTNULL && pExpr->pLeft->op==TK_COLUMN && pExpr->pLeft->iColumn>=0 - && OptimizationEnabled(db, SQLITE_Stat3) + && OptimizationEnabled(db, SQLITE_Stat34) ){ Expr *pNewExpr; Expr *pLeft = pExpr->pLeft; @@ -2159,7 +2159,6 @@ static int whereRangeScanEst( if( p->nSample>0 && nEqnSampleCol - && OptimizationEnabled(pParse->db, SQLITE_Stat3) ){ if( nEq==pBuilder->nRecValid ){ UnpackedRecord *pRec = pBuilder->pRec; @@ -4422,7 +4421,6 @@ static int whereLoopAddBtreeIndex( if( nInMul==0 && pProbe->nSample && pNew->u.btree.nEq<=pProbe->nSampleCol - && OptimizationEnabled(db, SQLITE_Stat3) && ((eOp & WO_IN)==0 || !ExprHasProperty(pTerm->pExpr, EP_xIsSelect)) ){ Expr *pExpr = pTerm->pExpr; From b7288e28e48235e8f7671ae7b613f4c5c7251c44 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 22 Oct 2014 20:07:19 +0000 Subject: [PATCH 012/133] Disable the use of strchrnul() unless specifically enabled by compile-time options. FossilOrigin-Name: e580470db77d6da970c755102790e603fb26b3c6 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/printf.c | 6 +----- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 0f41135502..ae5d008c78 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\s0x800\sbit\sof\sSQLITE_TESTCTRL_OPTIMIZATIONS\sso\sthat\sit\sdisables\nthe\sloading\sof\sSTAT3\sand\sSTAT4\scontent,\snot\sjust\sthe\susing\sof\sthat\scontent.\nChange\sthe\sinternal\sname\sof\sthat\sbit\sto\sSQLITE_Stat34. -D 2014-10-22T19:57:16.520 +C Disable\sthe\suse\sof\sstrchrnul()\sunless\sspecifically\senabled\sby\scompile-time\noptions. +D 2014-10-22T20:07:19.558 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -223,7 +223,7 @@ F src/pcache.h 9b559127b83f84ff76d735c8262f04853be0c59a F src/pcache1.c e412cb585f777c840ddce0500eddc5c6043c2bb5 F src/pragma.c 3f3e959390a10c0131676f0e307acce372777e0f F src/prepare.c 6ef0cf2f9274982988ed6b7cab1be23147e94196 -F src/printf.c 6b79bbd063dcbadca4cf617a4cde255bcc13ea64 +F src/printf.c 090fac0f779c93c8a95089a125339686648835e4 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c a3466128b52a86c466e47ac1a19e2174f7b5cf89 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 3c933bf95f291f7957580d823dce92c981375a5c -R 3efbbff265eea57d45c364e4606463ea +P ca3b00c44ec52d209642a5ba9ef82e085fac39db +R 8aff7924cf3a3f93f4ee99fb545d1f5f U drh -Z b4247f0ab9b19770a6354621b6169ee4 +Z c6210337ac863b7dfe5cbe19c541a0aa diff --git a/manifest.uuid b/manifest.uuid index fff2e6a3fa..857c7b0e10 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ca3b00c44ec52d209642a5ba9ef82e085fac39db \ No newline at end of file +e580470db77d6da970c755102790e603fb26b3c6 \ No newline at end of file diff --git a/src/printf.c b/src/printf.c index c0b3c70f6b..1df287fbb6 100644 --- a/src/printf.c +++ b/src/printf.c @@ -21,11 +21,7 @@ ** the glibc version so the glibc version is definitely preferred. */ #if !defined(HAVE_STRCHRNUL) -# if defined(linux) -# define HAVE_STRCHRNUL 1 -# else -# define HAVE_STRCHRNUL 0 -# endif +# define HAVE_STRCHRNUL 0 #endif From 4f81bbb5289cdd248c21775b9e4cdb92e110e139 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 23 Oct 2014 01:01:26 +0000 Subject: [PATCH 013/133] Implement sqlite3_create_collation() by invoking sqlite3_create_collation_v2() with a NULL destructor argument. This saves a little space. FossilOrigin-Name: 9762ad0639cca2fc1ef0573113fb613ce9e7e83e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/main.c | 8 +------- 3 files changed, 8 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index ae5d008c78..5b87a53662 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Disable\sthe\suse\sof\sstrchrnul()\sunless\sspecifically\senabled\sby\scompile-time\noptions. -D 2014-10-22T20:07:19.558 +C Implement\ssqlite3_create_collation()\sby\sinvoking\s\nsqlite3_create_collation_v2()\swith\sa\sNULL\sdestructor\sargument.\s\sThis\ssaves\na\slittle\sspace. +D 2014-10-23T01:01:26.636 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -194,7 +194,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 F src/loadext.c de741e66e5ddc1598d904d7289239696e40ed994 -F src/main.c bbe872b0ac0007bed0ebe1936fc493b039ad4f51 +F src/main.c 8207ece1b4005b1efab55d1505d8ffb45bfced05 F src/malloc.c 3c3ac67969612493d435e14b6832793209afd2ec F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c faf615aafd8be74a71494dfa027c113ea5c6615f @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ca3b00c44ec52d209642a5ba9ef82e085fac39db -R 8aff7924cf3a3f93f4ee99fb545d1f5f +P e580470db77d6da970c755102790e603fb26b3c6 +R f159389d59e4350424d5f438676d787b U drh -Z c6210337ac863b7dfe5cbe19c541a0aa +Z 177632e852e703fce9e831bfbe27f418 diff --git a/manifest.uuid b/manifest.uuid index 857c7b0e10..b0a7f1f074 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e580470db77d6da970c755102790e603fb26b3c6 \ No newline at end of file +9762ad0639cca2fc1ef0573113fb613ce9e7e83e \ No newline at end of file diff --git a/src/main.c b/src/main.c index ea03f2639f..e6cb4cd3df 100644 --- a/src/main.c +++ b/src/main.c @@ -2751,13 +2751,7 @@ int sqlite3_create_collation( void* pCtx, int(*xCompare)(void*,int,const void*,int,const void*) ){ - int rc; - sqlite3_mutex_enter(db->mutex); - assert( !db->mallocFailed ); - rc = createCollation(db, zName, (u8)enc, pCtx, xCompare, 0); - rc = sqlite3ApiExit(db, rc); - sqlite3_mutex_leave(db->mutex); - return rc; + return sqlite3_create_collation_v2(db, zName, enc, pCtx, xCompare, 0); } /* From 9ca95730e3a25bf5b335ec73a0a14f0112a421fb Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 24 Oct 2014 00:35:58 +0000 Subject: [PATCH 014/133] Add the SQLITE_ENABLE_API_ARMOR compile-time option. This is a work in progress and is not yet completely functional. FossilOrigin-Name: c297a84bc678f81ffc0aa9139ab73f0ca87c1971 --- manifest | 49 +++++++------- manifest.uuid | 2 +- src/auth.c | 3 + src/backup.c | 22 +++++++ src/build.c | 6 +- src/complete.c | 7 ++ src/ctime.c | 10 +++ src/main.c | 165 ++++++++++++++++++++++++++++++++++++++++++++++- src/malloc.c | 3 + src/mutex_unix.c | 8 ++- src/os.c | 4 ++ src/prepare.c | 13 ++-- src/printf.c | 21 ++++++ src/random.c | 11 ++-- src/status.c | 8 +++ src/table.c | 3 + src/util.c | 10 +++ src/vdbeapi.c | 31 +++++++-- src/vdbeblob.c | 5 ++ src/vtab.c | 16 ++++- 20 files changed, 351 insertions(+), 46 deletions(-) diff --git a/manifest b/manifest index 5b87a53662..0b02968a4f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Implement\ssqlite3_create_collation()\sby\sinvoking\s\nsqlite3_create_collation_v2()\swith\sa\sNULL\sdestructor\sargument.\s\sThis\ssaves\na\slittle\sspace. -D 2014-10-23T01:01:26.636 +C Add\sthe\sSQLITE_ENABLE_API_ARMOR\scompile-time\soption.\s\sThis\sis\sa\swork\sin\nprogress\sand\sis\snot\syet\scompletely\sfunctional. +D 2014-10-24T00:35:58.052 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -168,17 +168,17 @@ F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c ba266a779bc7ce10e52e59e7d3dc79fa342e8fdb F src/analyze.c afbcca663c3f3625340b8e30d440cd7a97ded6bc F src/attach.c f4e94df2d1826feda65eb0939f7f6f5f923a0ad9 -F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 -F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e +F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 +F src/backup.c 7f841396adfd47507ff670a471162d2bfcda3136 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 F src/btree.c 1b1123cba0c65caa0baa51e71b8c089e3167c3ed F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 -F src/build.c 9dc2bd94347b878c89627000c92b0c8d97ec2919 +F src/build.c 67bb05b1077e0cdaccb2e36bfcbe7a5df9ed31e8 F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 -F src/complete.c 535183afb3c75628b78ce82612931ac7cdf26f14 -F src/ctime.c bb434068b5308a857b181c2d204a320ff0d6c638 +F src/complete.c c4ba6e0626bb94bc77a0861735f3382fcf7cc818 +F src/ctime.c dfa83bfebb4201d07b16534acb8a0149592c3a25 F src/date.c 57a7f9ba9f6b4d5268f5e411739066a611f99036 F src/delete.c fae81cc2eb14b75267d4f47d3cfc9ae02aae726f F src/expr.c fc204d08af06437ddaffe5a1b1f1f6f9e1a55d6d @@ -194,8 +194,8 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 F src/loadext.c de741e66e5ddc1598d904d7289239696e40ed994 -F src/main.c 8207ece1b4005b1efab55d1505d8ffb45bfced05 -F src/malloc.c 3c3ac67969612493d435e14b6832793209afd2ec +F src/main.c 2b882f64580ea72e2d972a5296f9eaa75a353161 +F src/malloc.c 5e02eab3e70cfcc265817f4de008f55e768dca51 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c faf615aafd8be74a71494dfa027c113ea5c6615f F src/mem2.c f1940d9e91948dd6a908fbb9ce3835c36b5d83c3 @@ -205,10 +205,10 @@ F src/memjournal.c 3eb2c0b51adbd869cb6a44780323f05fa904dc85 F src/mutex.c 84a073c9a23a8d7bdd2ea832522d1730df18812c F src/mutex.h 779d588e3b7756ec3ecf7d78cde1d84aba414f85 F src/mutex_noop.c f3f09fd7a2eb4287cfc799753ffc30380e7b71a1 -F src/mutex_unix.c 1b10d5413dfc794364a8adf3eb3a192926b43fa3 +F src/mutex_unix.c 551e2f25f0fa0ee8fd7a43f50fc3d8be00e95dde F src/mutex_w32.c 06bfff9a3a83b53389a51a967643db3967032e1e F src/notify.c 9711a7575036f0d3040ba61bc6e217f13a9888e7 -F src/os.c 1b147e4cf7cc39e618115c14a086aed44bc91ace +F src/os.c 8fd25588eeba74068d41102d26810e216999b6c8 F src/os.h 3e57a24e2794a94d3cf2342c6d9a884888cd96bf F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa @@ -222,9 +222,9 @@ F src/pcache.c 4121a0571c18581ee9f82f086d5e2030051ebd6a F src/pcache.h 9b559127b83f84ff76d735c8262f04853be0c59a F src/pcache1.c e412cb585f777c840ddce0500eddc5c6043c2bb5 F src/pragma.c 3f3e959390a10c0131676f0e307acce372777e0f -F src/prepare.c 6ef0cf2f9274982988ed6b7cab1be23147e94196 -F src/printf.c 090fac0f779c93c8a95089a125339686648835e4 -F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece +F src/prepare.c b7b7bf020bd4c962f7c8aed5a3c542c7dfe9f9c7 +F src/printf.c c31012ac23e458081df4a32634b60424e0cdfaf3 +F src/random.c 689c95a50b90e7a8b3e6088e33d1d2d4212e6e88 F src/resolve.c a3466128b52a86c466e47ac1a19e2174f7b5cf89 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b @@ -234,8 +234,8 @@ F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 6e9e125698c1e5c78a51050ea61f179a281c766d F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d -F src/status.c 961d5926e5a8fda611d385ec22c226b8635cd1cb -F src/table.c 2e99ef7ef16187e17033d9398dc962ce22dab5cb +F src/status.c 81712116e826b0089bb221b018929536b2b5406f +F src/table.c f142bba7903e93ca8d113a5b8877a108ad1a27dc F src/tclsqlite.c c67d310c833046cccc192125d64ad422ab882684 F src/test1.c 63d4b1707c4052cf9c05c1cbb4a62666d70a0b48 F src/test2.c 98049e51a17dc62606a99a9eb95ee477f9996712 @@ -287,18 +287,18 @@ F src/tokenize.c cc9016e5007fc5e76789079616d2f26741bcc689 F src/trigger.c 25571661fdeae8c7f975ff40ffec205520a3f92f F src/update.c 729f6f18fc27740591d085e1172cebe311144bf0 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c -F src/util.c 4006c01772bd8d8ac4306d523bbcee41d3e392d8 +F src/util.c 3b627daa45c7308c1e36e3dbaa3f9ce7e5c7fa73 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a F src/vdbe.c 5ee15a66ce07e0482b92aa29e4dd0c5827a22d79 F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 F src/vdbeInt.h e2a060a55ee18a6ab973353a5e2ec7ee569bf787 -F src/vdbeapi.c 37a6c6ae284a97bcace365f2f0a225680c0499d9 +F src/vdbeapi.c 02d8afcff710eb35e3d9e49cb677308296b00009 F src/vdbeaux.c edbb7a9c8b2a8f7a68ac75c2475edd4040266b76 -F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 +F src/vdbeblob.c 8b5442ff0954c44b45cbabbe2e94091a2e16fdef F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f F src/vdbesort.c 975aeffa99acb0991b2f288d30294756bff41438 F src/vdbetrace.c 7e4222955e07dd707a2f360c0eb73452be1cb010 -F src/vtab.c cb0c194303fea276b48d7d4b6d970b5a96bde8de +F src/vtab.c 2a30791bbd7926b589401bd09c3abb33de563793 F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 @@ -1205,7 +1205,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P e580470db77d6da970c755102790e603fb26b3c6 -R f159389d59e4350424d5f438676d787b +P 9762ad0639cca2fc1ef0573113fb613ce9e7e83e +R 98ebc598759882f163c17892cc0d9e3e +T *branch * api-armor +T *sym-api-armor * +T -sym-trunk * U drh -Z 177632e852e703fce9e831bfbe27f418 +Z 48418cab28475295c0204ad2ebe00aca diff --git a/manifest.uuid b/manifest.uuid index b0a7f1f074..64b7efe90b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9762ad0639cca2fc1ef0573113fb613ce9e7e83e \ No newline at end of file +c297a84bc678f81ffc0aa9139ab73f0ca87c1971 \ No newline at end of file diff --git a/src/auth.c b/src/auth.c index 1680c9a7c2..9768fc2fc0 100644 --- a/src/auth.c +++ b/src/auth.c @@ -72,6 +72,9 @@ int sqlite3_set_authorizer( int (*xAuth)(void*,int,const char*,const char*,const char*,const char*), void *pArg ){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); db->xAuth = (sqlite3_xauth)xAuth; db->pAuthArg = pArg; diff --git a/src/backup.c b/src/backup.c index 92c6334bde..da4303e5fd 100644 --- a/src/backup.c +++ b/src/backup.c @@ -138,6 +138,13 @@ sqlite3_backup *sqlite3_backup_init( ){ sqlite3_backup *p; /* Value to return */ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(pSrcDb)||!sqlite3SafetyCheckOk(pDestDb) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + /* Lock the source database handle. The destination database ** handle is not locked in this routine, but it is locked in ** sqlite3_backup_step(). The user is required to ensure that no @@ -334,6 +341,9 @@ int sqlite3_backup_step(sqlite3_backup *p, int nPage){ int pgszSrc = 0; /* Source page size */ int pgszDest = 0; /* Destination page size */ +#ifdef SQLITE_ENABLE_API_ARMOR + if( p==0 ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(p->pSrcDb->mutex); sqlite3BtreeEnter(p->pSrc); if( p->pDestDb ){ @@ -623,6 +633,12 @@ int sqlite3_backup_finish(sqlite3_backup *p){ ** call to sqlite3_backup_step(). */ int sqlite3_backup_remaining(sqlite3_backup *p){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( p==0 ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif return p->nRemaining; } @@ -631,6 +647,12 @@ int sqlite3_backup_remaining(sqlite3_backup *p){ ** recent call to sqlite3_backup_step(). */ int sqlite3_backup_pagecount(sqlite3_backup *p){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( p==0 ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif return p->nPagecount; } diff --git a/src/build.c b/src/build.c index b897494db3..0b4affc664 100644 --- a/src/build.c +++ b/src/build.c @@ -307,7 +307,11 @@ int sqlite3UserAuthTable(const char *zTable){ Table *sqlite3FindTable(sqlite3 *db, const char *zName, const char *zDatabase){ Table *p = 0; int i; - assert( zName!=0 ); + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zName==0 ) return 0; +#endif + /* All mutexes are required for schema access. Make sure we hold them. */ assert( zDatabase!=0 || sqlite3BtreeHoldsAllMutexes(db) ); #if SQLITE_USER_AUTHENTICATION diff --git a/src/complete.c b/src/complete.c index 6ab6f4a042..c439cfe181 100644 --- a/src/complete.c +++ b/src/complete.c @@ -105,6 +105,13 @@ int sqlite3_complete(const char *zSql){ u8 state = 0; /* Current state, using numbers defined in header comment */ u8 token; /* Value of the next token */ +#ifdef SQLITE_ENABLE_API_ARMOR + if( zSql==0 ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + #ifndef SQLITE_OMIT_TRIGGER /* A complex statement machine used to detect the end of a CREATE TRIGGER ** statement. This is the normal case. diff --git a/src/ctime.c b/src/ctime.c index 82a2f35204..36d0c266ee 100644 --- a/src/ctime.c +++ b/src/ctime.c @@ -63,6 +63,9 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_DISABLE_LFS "DISABLE_LFS", #endif +#ifdef SQLITE_ENABLE_API_ARMOR + "ENABLE_API_ARMOR", +#endif #ifdef SQLITE_ENABLE_ATOMIC_WRITE "ENABLE_ATOMIC_WRITE", #endif @@ -388,6 +391,13 @@ static const char * const azCompileOpt[] = { */ int sqlite3_compileoption_used(const char *zOptName){ int i, n; + +#ifdef SQLITE_ENABLE_API_ARMORE + if( zOptName==0 ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif if( sqlite3StrNICmp(zOptName, "SQLITE_", 7)==0 ) zOptName += 7; n = sqlite3Strlen30(zOptName); diff --git a/src/main.c b/src/main.c index e6cb4cd3df..5308698797 100644 --- a/src/main.c +++ b/src/main.c @@ -598,6 +598,12 @@ static int setupLookaside(sqlite3 *db, void *pBuf, int sz, int cnt){ ** Return the mutex associated with a database connection. */ sqlite3_mutex *sqlite3_db_mutex(sqlite3 *db){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif return db->mutex; } @@ -607,6 +613,10 @@ sqlite3_mutex *sqlite3_db_mutex(sqlite3 *db){ */ int sqlite3_db_release_memory(sqlite3 *db){ int i; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); sqlite3BtreeEnterAll(db); for(i=0; inDb; i++){ @@ -737,6 +747,12 @@ static int nocaseCollatingFunc( ** Return the ROWID of the most recent insert */ sqlite_int64 sqlite3_last_insert_rowid(sqlite3 *db){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif return db->lastRowid; } @@ -744,6 +760,12 @@ sqlite_int64 sqlite3_last_insert_rowid(sqlite3 *db){ ** Return the number of changes in the most recent call to sqlite3_exec(). */ int sqlite3_changes(sqlite3 *db){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif return db->nChange; } @@ -751,6 +773,12 @@ int sqlite3_changes(sqlite3 *db){ ** Return the number of changes since the database handle was opened. */ int sqlite3_total_changes(sqlite3 *db){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif return db->nTotalChange; } @@ -1296,6 +1324,9 @@ int sqlite3_busy_handler( int (*xBusy)(void*,int), void *pArg ){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE; +#endif sqlite3_mutex_enter(db->mutex); db->busyHandler.xFunc = xBusy; db->busyHandler.pArg = pArg; @@ -1317,6 +1348,12 @@ void sqlite3_progress_handler( int (*xProgress)(void*), void *pArg ){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return; + } +#endif sqlite3_mutex_enter(db->mutex); if( nOps>0 ){ db->xProgress = xProgress; @@ -1337,6 +1374,9 @@ void sqlite3_progress_handler( ** specified number of milliseconds before returning 0. */ int sqlite3_busy_timeout(sqlite3 *db, int ms){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif if( ms>0 ){ sqlite3_busy_handler(db, sqliteDefaultBusyCallback, (void*)db); db->busyTimeout = ms; @@ -1350,6 +1390,12 @@ int sqlite3_busy_timeout(sqlite3 *db, int ms){ ** Cause any pending operation to stop at its earliest opportunity. */ void sqlite3_interrupt(sqlite3 *db){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return; + } +#endif db->u1.isInterrupted = 1; } @@ -1487,6 +1533,12 @@ int sqlite3_create_function_v2( ){ int rc = SQLITE_ERROR; FuncDestructor *pArg = 0; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + return SQLITE_MISUSE_BKPT; + } +#endif sqlite3_mutex_enter(db->mutex); if( xDestroy ){ pArg = (FuncDestructor *)sqlite3DbMallocZero(db, sizeof(FuncDestructor)); @@ -1523,6 +1575,10 @@ int sqlite3_create_function16( ){ int rc; char *zFunc8; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zFunctionName==0 ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); assert( !db->mallocFailed ); zFunc8 = sqlite3Utf16to8(db, zFunctionName, -1, SQLITE_UTF16NATIVE); @@ -1554,6 +1610,12 @@ int sqlite3_overload_function( ){ int nName = sqlite3Strlen30(zName); int rc = SQLITE_OK; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zName==0 || nArg<-2 ){ + return SQLITE_MISUSE_BKPT; + } +#endif sqlite3_mutex_enter(db->mutex); if( sqlite3FindFunction(db, zName, nName, nArg, SQLITE_UTF8, 0)==0 ){ rc = sqlite3CreateFunc(db, zName, nArg, SQLITE_UTF8, @@ -1575,6 +1637,13 @@ int sqlite3_overload_function( */ void *sqlite3_trace(sqlite3 *db, void (*xTrace)(void*,const char*), void *pArg){ void *pOld; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif sqlite3_mutex_enter(db->mutex); pOld = db->pTraceArg; db->xTrace = xTrace; @@ -1596,6 +1665,13 @@ void *sqlite3_profile( void *pArg ){ void *pOld; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif sqlite3_mutex_enter(db->mutex); pOld = db->pProfileArg; db->xProfile = xProfile; @@ -1616,6 +1692,13 @@ void *sqlite3_commit_hook( void *pArg /* Argument to the function */ ){ void *pOld; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif sqlite3_mutex_enter(db->mutex); pOld = db->pCommitArg; db->xCommitCallback = xCallback; @@ -1634,6 +1717,13 @@ void *sqlite3_update_hook( void *pArg /* Argument to the function */ ){ void *pRet; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif sqlite3_mutex_enter(db->mutex); pRet = db->pUpdateArg; db->xUpdateCallback = xCallback; @@ -1652,6 +1742,13 @@ void *sqlite3_rollback_hook( void *pArg /* Argument to the function */ ){ void *pRet; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif sqlite3_mutex_enter(db->mutex); pRet = db->pRollbackArg; db->xRollbackCallback = xCallback; @@ -1698,6 +1795,9 @@ int sqlite3_wal_autocheckpoint(sqlite3 *db, int nFrame){ UNUSED_PARAMETER(db); UNUSED_PARAMETER(nFrame); #else +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif if( nFrame>0 ){ sqlite3_wal_hook(db, sqlite3WalDefaultHook, SQLITE_INT_TO_PTR(nFrame)); }else{ @@ -1718,6 +1818,12 @@ void *sqlite3_wal_hook( ){ #ifndef SQLITE_OMIT_WAL void *pRet; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif sqlite3_mutex_enter(db->mutex); pRet = db->pWalArg; db->xWalCallback = xCallback; @@ -1745,6 +1851,10 @@ int sqlite3_wal_checkpoint_v2( int rc; /* Return code */ int iDb = SQLITE_MAX_ATTACHED; /* sqlite3.aDb[] index of db to checkpoint */ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + /* Initialize the output variables to -1 in case an error occurs. */ if( pnLog ) *pnLog = -1; if( pnCkpt ) *pnCkpt = -1; @@ -2141,6 +2251,12 @@ static const int aHardLimit[] = { int sqlite3_limit(sqlite3 *db, int limitId, int newLimit){ int oldLimit; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return -1; + } +#endif /* EVIDENCE-OF: R-30189-54097 For each limit category SQLITE_LIMIT_NAME ** there is a hard upper bound set at compile-time by a C preprocessor @@ -2426,6 +2542,9 @@ static int openDatabase( char *zOpen = 0; /* Filename argument to pass to BtreeOpen() */ char *zErrMsg = 0; /* Error message from sqlite3ParseUri() */ +#ifdef SQLITE_ENABLE_API_ARMOR + if( ppDb==0 ) return SQLITE_MISUSE_BKPT; +#endif *ppDb = 0; #ifndef SQLITE_OMIT_AUTOINIT rc = sqlite3_initialize(); @@ -2715,13 +2834,15 @@ int sqlite3_open16( sqlite3_value *pVal; int rc; - assert( zFilename ); - assert( ppDb ); +#ifdef SQLITE_ENABLE_API_ARMOR + if( ppDb==0 ) return SQLITE_MISUSE_BKPT; +#endif *ppDb = 0; #ifndef SQLITE_OMIT_AUTOINIT rc = sqlite3_initialize(); if( rc ) return rc; #endif + if( zFilename==0 ) zFilename = "\000\000"; pVal = sqlite3ValueNew(0); sqlite3ValueSetStr(pVal, -1, zFilename, SQLITE_UTF16NATIVE, SQLITE_STATIC); zFilename8 = sqlite3ValueText(pVal, SQLITE_UTF8); @@ -2766,6 +2887,10 @@ int sqlite3_create_collation_v2( void(*xDel)(void*) ){ int rc; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zName==0 ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); assert( !db->mallocFailed ); rc = createCollation(db, zName, (u8)enc, pCtx, xCompare, xDel); @@ -2787,6 +2912,10 @@ int sqlite3_create_collation16( ){ int rc = SQLITE_OK; char *zName8; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zName==0 ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); assert( !db->mallocFailed ); zName8 = sqlite3Utf16to8(db, zName, -1, SQLITE_UTF16NATIVE); @@ -2809,6 +2938,9 @@ int sqlite3_collation_needed( void *pCollNeededArg, void(*xCollNeeded)(void*,sqlite3*,int eTextRep,const char*) ){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); db->xCollNeeded = xCollNeeded; db->xCollNeeded16 = 0; @@ -2827,6 +2959,9 @@ int sqlite3_collation_needed16( void *pCollNeededArg, void(*xCollNeeded16)(void*,sqlite3*,int eTextRep,const void*) ){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); db->xCollNeeded = 0; db->xCollNeeded16 = xCollNeeded16; @@ -2853,6 +2988,12 @@ int sqlite3_global_recover(void){ ** by the next COMMIT or ROLLBACK. */ int sqlite3_get_autocommit(sqlite3 *db){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif return db->autoCommit; } @@ -3035,6 +3176,9 @@ int sqlite3_sleep(int ms){ ** Enable or disable the extended result codes. */ int sqlite3_extended_result_codes(sqlite3 *db, int onoff){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); db->errMask = onoff ? 0xffffffff : 0xff; sqlite3_mutex_leave(db->mutex); @@ -3048,6 +3192,9 @@ int sqlite3_file_control(sqlite3 *db, const char *zDbName, int op, void *pArg){ int rc = SQLITE_ERROR; Btree *pBtree; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); pBtree = sqlite3DbNameToBtree(db, zDbName); if( pBtree ){ @@ -3390,7 +3537,7 @@ int sqlite3_test_control(int op, ...){ ** returns a NULL pointer. */ const char *sqlite3_uri_parameter(const char *zFilename, const char *zParam){ - if( zFilename==0 ) return 0; + if( zFilename==0 || zParam==0 ) return 0; zFilename += sqlite3Strlen30(zFilename) + 1; while( zFilename[0] ){ int x = strcmp(zFilename, zParam); @@ -3446,6 +3593,12 @@ Btree *sqlite3DbNameToBtree(sqlite3 *db, const char *zDbName){ ** connection. */ const char *sqlite3_db_filename(sqlite3 *db, const char *zDbName){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif Btree *pBt = sqlite3DbNameToBtree(db, zDbName); return pBt ? sqlite3BtreeGetFilename(pBt) : 0; } @@ -3455,6 +3608,12 @@ const char *sqlite3_db_filename(sqlite3 *db, const char *zDbName){ ** no such database exists. */ int sqlite3_db_readonly(sqlite3 *db, const char *zDbName){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return -1; + } +#endif Btree *pBt = sqlite3DbNameToBtree(db, zDbName); return pBt ? sqlite3BtreeIsReadonly(pBt) : -1; } diff --git a/src/malloc.c b/src/malloc.c index 6fb9d53d1b..c3c644fd6a 100644 --- a/src/malloc.c +++ b/src/malloc.c @@ -475,6 +475,9 @@ sqlite3_uint64 sqlite3_msize(void *p){ ** Free memory previously obtained from sqlite3Malloc(). */ void sqlite3_free(void *p){ +#if defined(SQLITE_ENABLE_API_ARMOR) && !defined(SQLITE_OMIT_AUTOINIT) + if( sqlite3_initialize() ) return; +#endif if( p==0 ) return; /* IMP: R-49053-54554 */ assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); assert( sqlite3MemdebugNoType(p, ~MEMTYPE_HEAP) ); diff --git a/src/mutex_unix.c b/src/mutex_unix.c index c8663144e8..c936914d8a 100644 --- a/src/mutex_unix.c +++ b/src/mutex_unix.c @@ -175,8 +175,12 @@ static sqlite3_mutex *pthreadMutexAlloc(int iType){ break; } default: { - assert( iType-2 >= 0 ); - assert( iType-2 < ArraySize(staticMutexes) ); +#ifdef SQLITE_ENABLE_API_ARMOR + if( iType-2<0 || iType-2>=ArraySize(staticMutexes) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif p = &staticMutexes[iType-2]; #if SQLITE_MUTEX_NREF p->id = iType; diff --git a/src/os.c b/src/os.c index b6c28a1dc4..2a2cf13c5e 100644 --- a/src/os.c +++ b/src/os.c @@ -361,6 +361,10 @@ int sqlite3_vfs_register(sqlite3_vfs *pVfs, int makeDflt){ int rc = sqlite3_initialize(); if( rc ) return rc; #endif +#ifdef SQLITE_ENABLE_API_ARMOR + if( pVfs==0 ) return SQLITE_MISUSE_BKPT; +#endif + MUTEX_LOGIC( mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); ) sqlite3_mutex_enter(mutex); vfsUnlink(pVfs); diff --git a/src/prepare.c b/src/prepare.c index a05e619f3e..ca9c64b441 100644 --- a/src/prepare.c +++ b/src/prepare.c @@ -709,9 +709,12 @@ static int sqlite3LockAndPrepare( const char **pzTail /* OUT: End of parsed string */ ){ int rc; - assert( ppStmt!=0 ); + +#ifdef SQLITE_ENABLE_API_ARMOR + if( ppStmt==0 ) return SQLITE_MISUSE_BKPT; +#endif *ppStmt = 0; - if( !sqlite3SafetyCheckOk(db) ){ + if( !sqlite3SafetyCheckOk(db)||zSql==0 ){ return SQLITE_MISUSE_BKPT; } sqlite3_mutex_enter(db->mutex); @@ -818,9 +821,11 @@ static int sqlite3Prepare16( const char *zTail8 = 0; int rc = SQLITE_OK; - assert( ppStmt ); +#ifdef SQLITE_ENABLE_API_ARMOR + if( ppStmt==0 ) return SQLITE_MISUSE_BKPT; +#endif *ppStmt = 0; - if( !sqlite3SafetyCheckOk(db) ){ + if( !sqlite3SafetyCheckOk(db)||zSql==0 ){ return SQLITE_MISUSE_BKPT; } if( nBytes>=0 ){ diff --git a/src/printf.c b/src/printf.c index 1df287fbb6..f000da7fcc 100644 --- a/src/printf.c +++ b/src/printf.c @@ -223,6 +223,13 @@ void sqlite3VXPrintf( PrintfArguments *pArgList = 0; /* Arguments for SQLITE_PRINTF_SQLFUNC */ char buf[etBUFSIZE]; /* Conversion buffer */ +#ifdef SQLITE_ENABLE_API_ARMOR + if( ap==0 ){ + (void)SQLITE_MISUSE_BKPT; + sqlite3StrAccumReset(pAccum); + return; + } +#endif bufpt = 0; if( bFlags ){ if( (bArgList = (bFlags & SQLITE_PRINTF_SQLFUNC))!=0 ){ @@ -943,6 +950,13 @@ char *sqlite3_vmprintf(const char *zFormat, va_list ap){ char *z; char zBase[SQLITE_PRINT_BUF_SIZE]; StrAccum acc; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( zFormat==0 ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif #ifndef SQLITE_OMIT_AUTOINIT if( sqlite3_initialize() ) return 0; #endif @@ -985,6 +999,13 @@ char *sqlite3_mprintf(const char *zFormat, ...){ char *sqlite3_vsnprintf(int n, char *zBuf, const char *zFormat, va_list ap){ StrAccum acc; if( n<=0 ) return zBuf; +#ifdef SQLITE_ENABLE_API_ARMOR + if( zBuf==0 || zFormat==0 ) { + (void)SQLITE_MISUSE_BKPT; + if( zBuf && n>0 ) zBuf[0] = 0; + return zBuf; + } +#endif sqlite3StrAccumInit(&acc, zBuf, n, 0); acc.useMalloc = 0; sqlite3VXPrintf(&acc, 0, zFormat, ap); diff --git a/src/random.c b/src/random.c index b82566524c..7bca0d71af 100644 --- a/src/random.c +++ b/src/random.c @@ -34,6 +34,11 @@ void sqlite3_randomness(int N, void *pBuf){ unsigned char t; unsigned char *zBuf = pBuf; +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return; +#endif + if( pBuf==0 || N<=0 ) return; + /* The "wsdPrng" macro will resolve to the pseudo-random number generator ** state vector. If writable static data is unsupported on the target, ** we have to locate the state vector at run-time. In the more common @@ -52,12 +57,6 @@ void sqlite3_randomness(int N, void *pBuf){ sqlite3_mutex_enter(mutex); #endif - if( N<=0 ){ - wsdPrng.isInit = 0; - sqlite3_mutex_leave(mutex); - return; - } - /* Initialize the state of the random number generator once, ** the first time this routine is called. The seed value does ** not need to contain a lot of randomness since we are not diff --git a/src/status.c b/src/status.c index 79a8001b8a..4c2eabb661 100644 --- a/src/status.c +++ b/src/status.c @@ -86,6 +86,9 @@ int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetFlag){ if( op<0 || op>=ArraySize(wsdStat.nowValue) ){ return SQLITE_MISUSE_BKPT; } +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCurrent==0 || pHighwater==0 ) return SQLITE_MISUSE_BKPT; +#endif *pCurrent = wsdStat.nowValue[op]; *pHighwater = wsdStat.mxValue[op]; if( resetFlag ){ @@ -105,6 +108,11 @@ int sqlite3_db_status( int resetFlag /* Reset high-water mark if true */ ){ int rc = SQLITE_OK; /* Return code */ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || pCurrent==0|| pHighwater==0 ){ + return SQLITE_MISUSE_BKPT; + } +#endif sqlite3_mutex_enter(db->mutex); switch( op ){ case SQLITE_DBSTATUS_LOOKASIDE_USED: { diff --git a/src/table.c b/src/table.c index c435b2bc02..6e1df30643 100644 --- a/src/table.c +++ b/src/table.c @@ -126,6 +126,9 @@ int sqlite3_get_table( int rc; TabResult res; +#ifdef SQLITE_ENABLE_API_ARMOR + if( pazResult==0 ) return SQLITE_MISUSE_BKPT; +#endif *pazResult = 0; if( pnColumn ) *pnColumn = 0; if( pnRow ) *pnRow = 0; diff --git a/src/util.c b/src/util.c index 9bb8d89157..ab409fa256 100644 --- a/src/util.c +++ b/src/util.c @@ -251,6 +251,11 @@ int sqlite3Dequote(char *z){ */ int sqlite3_stricmp(const char *zLeft, const char *zRight){ register unsigned char *a, *b; + if( zLeft==0 ){ + return zRight ? -1 : 0; + }else if( zRight==0 ){ + return 1; + } a = (unsigned char *)zLeft; b = (unsigned char *)zRight; while( *a!=0 && UpperToLower[*a]==UpperToLower[*b]){ a++; b++; } @@ -258,6 +263,11 @@ int sqlite3_stricmp(const char *zLeft, const char *zRight){ } int sqlite3_strnicmp(const char *zLeft, const char *zRight, int N){ register unsigned char *a, *b; + if( zLeft==0 ){ + return zRight ? -1 : 0; + }else if( zRight==0 ){ + return 1; + } a = (unsigned char *)zLeft; b = (unsigned char *)zRight; while( N-- > 0 && *a!=0 && UpperToLower[*a]==UpperToLower[*b]){ a++; b++; } diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 0ab76e0784..daf7b101d2 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -966,11 +966,19 @@ static const void *columnName( const void *(*xFunc)(Mem*), int useType ){ - const void *ret = 0; - Vdbe *p = (Vdbe *)pStmt; + const void *ret; + Vdbe *p; int n; - sqlite3 *db = p->db; - + sqlite3 *db; +#ifdef SQLITE_ENABLE_API_ARMOR + if( pStmt==0 ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + ret = 0; + p = (Vdbe *)pStmt; + db = p->db; assert( db!=0 ); n = sqlite3_column_count(pStmt); if( N=0 ){ @@ -1435,6 +1443,12 @@ int sqlite3_stmt_busy(sqlite3_stmt *pStmt){ */ sqlite3_stmt *sqlite3_next_stmt(sqlite3 *pDb, sqlite3_stmt *pStmt){ sqlite3_stmt *pNext; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(pDb) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif sqlite3_mutex_enter(pDb->mutex); if( pStmt==0 ){ pNext = (sqlite3_stmt*)pDb->pVdbe; @@ -1450,7 +1464,14 @@ sqlite3_stmt *sqlite3_next_stmt(sqlite3 *pDb, sqlite3_stmt *pStmt){ */ int sqlite3_stmt_status(sqlite3_stmt *pStmt, int op, int resetFlag){ Vdbe *pVdbe = (Vdbe*)pStmt; - u32 v = pVdbe->aCounter[op]; + u32 v; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !pStmt ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + v = pVdbe->aCounter[op]; if( resetFlag ) pVdbe->aCounter[op] = 0; return (int)v; } diff --git a/src/vdbeblob.c b/src/vdbeblob.c index 71bd8816d5..0cf2b0652b 100644 --- a/src/vdbeblob.c +++ b/src/vdbeblob.c @@ -153,6 +153,11 @@ int sqlite3_blob_open( Parse *pParse = 0; Incrblob *pBlob = 0; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || ppBlob==0 || zTable==0 ){ + return SQLITE_MISUSE_BKPT; + } +#endif flags = !!flags; /* flags = (flags ? 1 : 0); */ *ppBlob = 0; diff --git a/src/vtab.c b/src/vtab.c index faee4ae478..334de9aacb 100644 --- a/src/vtab.c +++ b/src/vtab.c @@ -81,6 +81,9 @@ int sqlite3_create_module( const sqlite3_module *pModule, /* The definition of the module */ void *pAux /* Context pointer for xCreate/xConnect */ ){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zName==0 ) return SQLITE_MISUSE_BKPT; +#endif return createModule(db, zName, pModule, pAux, 0); } @@ -94,6 +97,9 @@ int sqlite3_create_module_v2( void *pAux, /* Context pointer for xCreate/xConnect */ void (*xDestroy)(void *) /* Module destructor function */ ){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zName==0 ) return SQLITE_MISUSE_BKPT; +#endif return createModule(db, zName, pModule, pAux, xDestroy); } @@ -698,6 +704,9 @@ int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ Table *pTab; char *zErr = 0; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); if( !db->pVtabCtx || !(pTab = db->pVtabCtx->pTab) ){ sqlite3Error(db, SQLITE_MISUSE); @@ -1054,6 +1063,9 @@ int sqlite3_vtab_on_conflict(sqlite3 *db){ static const unsigned char aMap[] = { SQLITE_ROLLBACK, SQLITE_ABORT, SQLITE_FAIL, SQLITE_IGNORE, SQLITE_REPLACE }; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif assert( OE_Rollback==1 && OE_Abort==2 && OE_Fail==3 ); assert( OE_Ignore==4 && OE_Replace==5 ); assert( db->vtabOnConflict>=1 && db->vtabOnConflict<=5 ); @@ -1069,8 +1081,10 @@ int sqlite3_vtab_config(sqlite3 *db, int op, ...){ va_list ap; int rc = SQLITE_OK; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); - va_start(ap, op); switch( op ){ case SQLITE_VTAB_CONSTRAINT_SUPPORT: { From 5a5d120bcc08af809b8687ff9fe39fce27e7796c Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 24 Oct 2014 12:37:00 +0000 Subject: [PATCH 015/133] Fix two problems. Tests now passing. FossilOrigin-Name: 1c220b806d56e163842e17038c3331f71861bd9c --- manifest | 17 +++++++---------- manifest.uuid | 2 +- src/malloc.c | 3 --- src/random.c | 7 ++++++- 4 files changed, 14 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 0b02968a4f..da62517a33 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sSQLITE_ENABLE_API_ARMOR\scompile-time\soption.\s\sThis\sis\sa\swork\sin\nprogress\sand\sis\snot\syet\scompletely\sfunctional. -D 2014-10-24T00:35:58.052 +C Fix\stwo\sproblems.\s\sTests\snow\spassing. +D 2014-10-24T12:37:00.827 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -195,7 +195,7 @@ F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 F src/loadext.c de741e66e5ddc1598d904d7289239696e40ed994 F src/main.c 2b882f64580ea72e2d972a5296f9eaa75a353161 -F src/malloc.c 5e02eab3e70cfcc265817f4de008f55e768dca51 +F src/malloc.c 3c3ac67969612493d435e14b6832793209afd2ec F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c faf615aafd8be74a71494dfa027c113ea5c6615f F src/mem2.c f1940d9e91948dd6a908fbb9ce3835c36b5d83c3 @@ -224,7 +224,7 @@ F src/pcache1.c e412cb585f777c840ddce0500eddc5c6043c2bb5 F src/pragma.c 3f3e959390a10c0131676f0e307acce372777e0f F src/prepare.c b7b7bf020bd4c962f7c8aed5a3c542c7dfe9f9c7 F src/printf.c c31012ac23e458081df4a32634b60424e0cdfaf3 -F src/random.c 689c95a50b90e7a8b3e6088e33d1d2d4212e6e88 +F src/random.c b8a058131851de1a37801b5587845ee73411c064 F src/resolve.c a3466128b52a86c466e47ac1a19e2174f7b5cf89 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b @@ -1205,10 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 9762ad0639cca2fc1ef0573113fb613ce9e7e83e -R 98ebc598759882f163c17892cc0d9e3e -T *branch * api-armor -T *sym-api-armor * -T -sym-trunk * +P c297a84bc678f81ffc0aa9139ab73f0ca87c1971 +R 1cc86c78be65c50602286adf14ad3d43 U drh -Z 48418cab28475295c0204ad2ebe00aca +Z 4a2cbf72bfaa58a272d60444a54fd92d diff --git a/manifest.uuid b/manifest.uuid index 64b7efe90b..0375d856fe 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c297a84bc678f81ffc0aa9139ab73f0ca87c1971 \ No newline at end of file +1c220b806d56e163842e17038c3331f71861bd9c \ No newline at end of file diff --git a/src/malloc.c b/src/malloc.c index c3c644fd6a..6fb9d53d1b 100644 --- a/src/malloc.c +++ b/src/malloc.c @@ -475,9 +475,6 @@ sqlite3_uint64 sqlite3_msize(void *p){ ** Free memory previously obtained from sqlite3Malloc(). */ void sqlite3_free(void *p){ -#if defined(SQLITE_ENABLE_API_ARMOR) && !defined(SQLITE_OMIT_AUTOINIT) - if( sqlite3_initialize() ) return; -#endif if( p==0 ) return; /* IMP: R-49053-54554 */ assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); assert( sqlite3MemdebugNoType(p, ~MEMTYPE_HEAP) ); diff --git a/src/random.c b/src/random.c index 7bca0d71af..c4241362de 100644 --- a/src/random.c +++ b/src/random.c @@ -37,7 +37,6 @@ void sqlite3_randomness(int N, void *pBuf){ #ifndef SQLITE_OMIT_AUTOINIT if( sqlite3_initialize() ) return; #endif - if( pBuf==0 || N<=0 ) return; /* The "wsdPrng" macro will resolve to the pseudo-random number generator ** state vector. If writable static data is unsupported on the target, @@ -57,6 +56,12 @@ void sqlite3_randomness(int N, void *pBuf){ sqlite3_mutex_enter(mutex); #endif + if( N<=0 || pBuf==0 ){ + wsdPrng.isInit = 0; + sqlite3_mutex_leave(mutex); + return; + } + /* Initialize the state of the random number generator once, ** the first time this routine is called. The seed value does ** not need to contain a lot of randomness since we are not From 9769efcc4c785dc7552a2e22782f12e3c56e0169 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 24 Oct 2014 14:32:21 +0000 Subject: [PATCH 016/133] Get the likelihood() functions working on operators like BETWEEN that create virtual terms in the WHERE-clause analysis. FossilOrigin-Name: 03d0498d0f24bec2383d5d79edf25069effecd59 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 29 ++++++++++++++++------------- 3 files changed, 23 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index 5b87a53662..67132856cf 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Implement\ssqlite3_create_collation()\sby\sinvoking\s\nsqlite3_create_collation_v2()\swith\sa\sNULL\sdestructor\sargument.\s\sThis\ssaves\na\slittle\sspace. -D 2014-10-23T01:01:26.636 +C Get\sthe\slikelihood()\sfunctions\sworking\son\soperators\slike\sBETWEEN\sthat\ncreate\svirtual\sterms\sin\sthe\sWHERE-clause\sanalysis. +D 2014-10-24T14:32:21.050 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -302,7 +302,7 @@ F src/vtab.c cb0c194303fea276b48d7d4b6d970b5a96bde8de F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 2cd9e0af718d736459ae9d3b0f4532b4a80640d0 +F src/where.c 746d4f22d75c1f000d6dae251dc65bf755c395a8 F src/whereInt.h 4b459cdbfc9b01f5f27673a35f9967e4dea917e8 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P e580470db77d6da970c755102790e603fb26b3c6 -R f159389d59e4350424d5f438676d787b +P 9762ad0639cca2fc1ef0573113fb613ce9e7e83e +R f460b87a81ee1fb775b7700e4252ece6 U drh -Z 177632e852e703fce9e831bfbe27f418 +Z 1428b6fe81396b49c1c6b7022fde286d diff --git a/manifest.uuid b/manifest.uuid index b0a7f1f074..59bf5ae4e6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9762ad0639cca2fc1ef0573113fb613ce9e7e83e \ No newline at end of file +03d0498d0f24bec2383d5d79edf25069effecd59 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 198a6c3fdd..d4969fad1e 100644 --- a/src/where.c +++ b/src/where.c @@ -756,6 +756,15 @@ static void transferJoinMarkings(Expr *pDerived, Expr *pBase){ } } +/* +** Mark term iChild as being a child of term iParent +*/ +static void markTermAsChild(WhereClause *pWC, int iChild, int iParent){ + pWC->a[iChild].iParent = iParent; + pWC->a[iChild].truthProb = pWC->a[iParent].truthProb; + pWC->a[iParent].nChild++; +} + #if !defined(SQLITE_OMIT_OR_OPTIMIZATION) && !defined(SQLITE_OMIT_SUBQUERY) /* ** Analyze a term that consists of two or more OR-connected @@ -1053,8 +1062,7 @@ static void exprAnalyzeOrTerm( testcase( idxNew==0 ); exprAnalyze(pSrc, pWC, idxNew); pTerm = &pWC->a[idxTerm]; - pWC->a[idxNew].iParent = idxTerm; - pTerm->nChild = 1; + markTermAsChild(pWC, idxNew, idxTerm); }else{ sqlite3ExprListDelete(db, pList); } @@ -1156,9 +1164,8 @@ static void exprAnalyze( idxNew = whereClauseInsert(pWC, pDup, TERM_VIRTUAL|TERM_DYNAMIC); if( idxNew==0 ) return; pNew = &pWC->a[idxNew]; - pNew->iParent = idxTerm; + markTermAsChild(pWC, idxNew, idxTerm); pTerm = &pWC->a[idxTerm]; - pTerm->nChild = 1; pTerm->wtFlags |= TERM_COPIED; if( pExpr->op==TK_EQ && !ExprHasProperty(pExpr, EP_FromJoin) @@ -1215,9 +1222,8 @@ static void exprAnalyze( testcase( idxNew==0 ); exprAnalyze(pSrc, pWC, idxNew); pTerm = &pWC->a[idxTerm]; - pWC->a[idxNew].iParent = idxTerm; + markTermAsChild(pWC, idxNew, idxTerm); } - pTerm->nChild = 2; } #endif /* SQLITE_OMIT_BETWEEN_OPTIMIZATION */ @@ -1292,9 +1298,8 @@ static void exprAnalyze( exprAnalyze(pSrc, pWC, idxNew2); pTerm = &pWC->a[idxTerm]; if( isComplete ){ - pWC->a[idxNew1].iParent = idxTerm; - pWC->a[idxNew2].iParent = idxTerm; - pTerm->nChild = 2; + markTermAsChild(pWC, idxNew1, idxTerm); + markTermAsChild(pWC, idxNew2, idxTerm); } } #endif /* SQLITE_OMIT_LIKE_OPTIMIZATION */ @@ -1327,9 +1332,8 @@ static void exprAnalyze( pNewTerm->leftCursor = pLeft->iTable; pNewTerm->u.leftColumn = pLeft->iColumn; pNewTerm->eOperator = WO_MATCH; - pNewTerm->iParent = idxTerm; + markTermAsChild(pWC, idxNew, idxTerm); pTerm = &pWC->a[idxTerm]; - pTerm->nChild = 1; pTerm->wtFlags |= TERM_COPIED; pNewTerm->prereqAll = pTerm->prereqAll; } @@ -1369,9 +1373,8 @@ static void exprAnalyze( pNewTerm->leftCursor = pLeft->iTable; pNewTerm->u.leftColumn = pLeft->iColumn; pNewTerm->eOperator = WO_GT; - pNewTerm->iParent = idxTerm; + markTermAsChild(pWC, idxNew, idxTerm); pTerm = &pWC->a[idxTerm]; - pTerm->nChild = 1; pTerm->wtFlags |= TERM_COPIED; pNewTerm->prereqAll = pTerm->prereqAll; } From 4dd96a8315b3f3b2a516513b06754e1887e426eb Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 24 Oct 2014 15:26:29 +0000 Subject: [PATCH 017/133] Honor a high likelihood() on range constraints. FossilOrigin-Name: 401235edf40fcd665eaf426cf5155ac6855e8537 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 7 +++++-- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 67132856cf..e98be71834 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Get\sthe\slikelihood()\sfunctions\sworking\son\soperators\slike\sBETWEEN\sthat\ncreate\svirtual\sterms\sin\sthe\sWHERE-clause\sanalysis. -D 2014-10-24T14:32:21.050 +C Honor\sa\shigh\slikelihood()\son\srange\sconstraints. +D 2014-10-24T15:26:29.800 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -302,7 +302,7 @@ F src/vtab.c cb0c194303fea276b48d7d4b6d970b5a96bde8de F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 746d4f22d75c1f000d6dae251dc65bf755c395a8 +F src/where.c f5c13d9c1929bcc9d571f1e7bf7bfeb8b872ef99 F src/whereInt.h 4b459cdbfc9b01f5f27673a35f9967e4dea917e8 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 9762ad0639cca2fc1ef0573113fb613ce9e7e83e -R f460b87a81ee1fb775b7700e4252ece6 +P 03d0498d0f24bec2383d5d79edf25069effecd59 +R eb013b99b9ff12a41d5ffc947e4a6227 U drh -Z 1428b6fe81396b49c1c6b7022fde286d +Z 3215dd92b9608b7b9136fb91eb8178e8 diff --git a/manifest.uuid b/manifest.uuid index 59bf5ae4e6..4ee03d34ab 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -03d0498d0f24bec2383d5d79edf25069effecd59 \ No newline at end of file +401235edf40fcd665eaf426cf5155ac6855e8537 \ No newline at end of file diff --git a/src/where.c b/src/where.c index d4969fad1e..e13b223366 100644 --- a/src/where.c +++ b/src/where.c @@ -2275,12 +2275,15 @@ static int whereRangeScanEst( nNew = whereRangeAdjust(pLower, nOut); nNew = whereRangeAdjust(pUpper, nNew); - /* TUNING: If there is both an upper and lower limit, assume the range is + /* TUNING: If there is both an upper and lower limit and neither limit + ** has an application-defined likelihood(), assume the range is ** reduced by an additional 75%. This means that, by default, an open-ended ** range query (e.g. col > ?) is assumed to match 1/4 of the rows in the ** index. While a closed range (e.g. col BETWEEN ? AND ?) is estimated to ** match 1/64 of the index. */ - if( pLower && pUpper ) nNew -= 20; + if( pLower && pLower->truthProb>0 && pUpper && pUpper->truthProb>0 ){ + nNew -= 20; + } nOut -= (pLower!=0) + (pUpper!=0); if( nNew<10 ) nNew = 10; From e6593d8e8c51fd05c762f34c444b19a74137fedc Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 24 Oct 2014 16:40:49 +0000 Subject: [PATCH 018/133] Fix some minor formatting and code organization issues. FossilOrigin-Name: eab8706dc47aa0a44caf73619de858397c3e0b4e --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/btree.c | 16 +++++++--------- src/pager.c | 15 +++++++++++---- src/pager.h | 2 +- 5 files changed, 28 insertions(+), 23 deletions(-) diff --git a/manifest b/manifest index 396fe366d0..b52c190c13 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\slatest\strunk\swith\sthis\sbranch. -D 2014-10-22T18:42:31.680 +C Fix\ssome\sminor\sformatting\sand\scode\sorganization\sissues. +D 2014-10-24T16:40:49.448 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 37229f3134416f24828af823a3a6f8535923d45b +F src/btree.c 56381ce7614853ec0e32bb187e85db4da774c7c5 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 9dc2bd94347b878c89627000c92b0c8d97ec2919 @@ -215,8 +215,8 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c fb587121840f690101336879adfa6d0b2cd0e8c7 F src/os_win.c a019caaae2bcbbc0cc4c39af6e7d7e43d8426053 F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 -F src/pager.c 18be49e363106bdb4c982fa27dbee7a8cd297113 -F src/pager.h 8b6707cb32c788cf36bfc3d63f6d4b4fa689e7c2 +F src/pager.c d02833adf331a5913226595306d64731a3da33f6 +F src/pager.h d1eee3c3f741be247ce6d82752a178515fc8578b F src/parse.y 5dfead8aed90cb0c7c1115898ee2266804daff45 F src/pcache.c 4121a0571c18581ee9f82f086d5e2030051ebd6a F src/pcache.h 9b559127b83f84ff76d735c8262f04853be0c59a @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 58d7793bd5d608ba9fc3a2cd44b9d9512e0332ba 3c933bf95f291f7957580d823dce92c981375a5c -R 738ce949461e4c95ef4d450eb06d976a +P 854a54c6c21e800b0cd999023014813f7c50b23f +R b3aec92126a25950ff876c6cdf2fb4df U dan -Z 1bc59f5d05ced43bcf6fa4690adf0fcc +Z 9e7a15bc5953228881d999c99d9f0682 diff --git a/manifest.uuid b/manifest.uuid index b48e7f9fc1..5987e57b67 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -854a54c6c21e800b0cd999023014813f7c50b23f \ No newline at end of file +eab8706dc47aa0a44caf73619de858397c3e0b4e \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index d1457c5f39..b5e8cb2e0c 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1233,7 +1233,7 @@ static int defragmentPage(MemPage *pPage){ ** ** This function may detect corruption within pPg. If it does and argument ** pRc is non-NULL, then *pRc is set to SQLITE_CORRUPT and NULL is returned. -** Or, if corruption is detected by pRc is NULL, NULL is returned and the +** Or, if corruption is detected and pRc is NULL, NULL is returned and the ** corruption goes unreported. */ static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){ @@ -6345,7 +6345,7 @@ static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){ } #endif /* SQLITE_OMIT_QUICKBALANCE */ -#if 1 +#if 0 /* ** This function does not contribute anything to the operation of SQLite. ** it is sometimes activated temporarily while debugging code responsible @@ -6522,10 +6522,9 @@ static int balance_nonroot( u16 *szCell; /* Local size of all cells in apCell[] */ u8 *aSpace1; /* Space for copies of dividers cells */ Pgno pgno; /* Temp var to store a page number in */ - - u8 abDone[NB+2]; - Pgno aPgno[NB+2]; - u16 aPgFlags[NB+2]; + u8 abDone[NB+2]; /* True after i'th new page is populated */ + Pgno aPgno[NB+2]; /* Page numbers of new pages before shuffling */ + u16 aPgFlags[NB+2]; /* flags field of new pages before shuffling */ memset(abDone, 0, sizeof(abDone)); pBt = pParent->pBt; @@ -6868,8 +6867,7 @@ static int balance_nonroot( } } if( apNew[i]->pgno!=iMin ){ - apNew[i]->pDbPage->flags = flags; - sqlite3PagerRekey(apNew[i]->pDbPage, iMin); + sqlite3PagerRekey(apNew[i]->pDbPage, iMin, flags); apNew[i]->pgno = iMin; } } @@ -7090,7 +7088,7 @@ static int balance_nonroot( freePage(apOld[i], &rc); } -#if 1 +#if 0 if( ISAUTOVACUUM && rc==SQLITE_OK && apNew[0]->isInit ){ /* The ptrmapCheckPages() contains assert() statements that verify that ** all pointer map pages are set correctly. This is helpful while diff --git a/src/pager.c b/src/pager.c index 74667afab0..dc79e6754a 100644 --- a/src/pager.c +++ b/src/pager.c @@ -6845,16 +6845,23 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ return SQLITE_OK; } +#endif -void sqlite3PagerRekey(DbPage *pPage, Pgno iNew){ +/* +** The page handle passed as the first argument refers to a dirty page +** with a page number other than iNew. This function changes the page's +** page number to iNew and sets the value of the PgHdr.flags field to +** the value passed as the third parameter. +*/ +void sqlite3PagerRekey(DbPage *pPage, Pgno iNew, u16 flags){ PgHdr *pPg = (PgHdr*)pPage; - assert( pPg->flags & PGHDR_DIRTY ); + assert( (flags & PGHDR_DIRTY) && (pPg->flags & PGHDR_DIRTY) ); assert( !subjRequiresPage(pPg) ); + assert( pPg->pgno!=iNew ); + pPg->flags = flags; sqlite3PcacheMove(pPg, iNew); } -#endif - /* ** Return a pointer to the data for the specified page. */ diff --git a/src/pager.h b/src/pager.h index f3a04bbca6..764c7bd197 100644 --- a/src/pager.h +++ b/src/pager.h @@ -188,7 +188,7 @@ int sqlite3SectorSize(sqlite3_file *); /* Functions used to truncate the database file. */ void sqlite3PagerTruncateImage(Pager*,Pgno); -void sqlite3PagerRekey(DbPage*, Pgno); +void sqlite3PagerRekey(DbPage*, Pgno, u16); #if defined(SQLITE_HAS_CODEC) && !defined(SQLITE_OMIT_WAL) void *sqlite3PagerCodec(DbPage *); From 23eba45d23291617e5f69ffb15c0cfbb1bab5121 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 24 Oct 2014 18:43:57 +0000 Subject: [PATCH 019/133] Fix some issues in the new code on this branch related to the handling of corrupt databases. FossilOrigin-Name: 19736dd9fbbb7e252c4f8715e2277d48ac41f5bc --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 14 ++++++++++---- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index b52c190c13..10be86a2cc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\ssome\sminor\sformatting\sand\scode\sorganization\sissues. -D 2014-10-24T16:40:49.448 +C Fix\ssome\sissues\sin\sthe\snew\scode\son\sthis\sbranch\srelated\sto\sthe\shandling\sof\scorrupt\sdatabases. +D 2014-10-24T18:43:57.313 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 56381ce7614853ec0e32bb187e85db4da774c7c5 +F src/btree.c df15daf116e9ef1a7ff88257f6fb678ba385b5ea F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 9dc2bd94347b878c89627000c92b0c8d97ec2919 @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 854a54c6c21e800b0cd999023014813f7c50b23f -R b3aec92126a25950ff876c6cdf2fb4df +P eab8706dc47aa0a44caf73619de858397c3e0b4e +R 8a34c36d0ed8bbbc0bb7360d0dc30d47 U dan -Z 9e7a15bc5953228881d999c99d9f0682 +Z 610a2814ec6cdf214c41e43dd2355678 diff --git a/manifest.uuid b/manifest.uuid index 5987e57b67..11462e357c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -eab8706dc47aa0a44caf73619de858397c3e0b4e \ No newline at end of file +19736dd9fbbb7e252c4f8715e2277d48ac41f5bc \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index b5e8cb2e0c..f326f713fb 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1386,7 +1386,7 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ assert( pPage->pBt!=0 ); assert( sqlite3PagerIswriteable(pPage->pDbPage) ); assert( iStart>=pPage->hdrOffset+6+pPage->childPtrSize ); - assert( iEnd <= pPage->pBt->usableSize ); + assert( CORRUPT_DB || iEnd <= pPage->pBt->usableSize ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); assert( iSize>=4 ); /* Minimum cell size is 4 */ assert( iStart<=iLast ); @@ -6065,7 +6065,7 @@ static int pageInsertArray( u8 *aData = pPg->aData; u8 *pData = *ppData; const int bFreelist = aData[1] || aData[2]; - assert( pPg->hdrOffset==0 ); /* Never called on page 1 */ + assert( CORRUPT_DB || pPg->hdrOffset==0 ); /* Never called on page 1 */ for(i=0; iaCellIdx[i*2]); if( pCell>=aData && pCell<&aData[pPg->pBt->usableSize] ){ @@ -6866,7 +6866,13 @@ static int balance_nonroot( flags = aPgFlags[j]; } } - if( apNew[i]->pgno!=iMin ){ + if( iMin==0 ){ + /* This case can only occur if aPgno[] contains duplicate page + ** numbers. Which can only happen if the database is corrupt. */ + assert( CORRUPT_DB ); + rc = SQLITE_CORRUPT_BKPT; + goto balance_cleanup; + }else if( apNew[i]->pgno!=iMin ){ sqlite3PagerRekey(apNew[i]->pDbPage, iMin, flags); apNew[i]->pgno = iMin; } From 059b2d50e1c6a57ca301f3c9639f92f7e16ff96e Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 24 Oct 2014 19:28:09 +0000 Subject: [PATCH 020/133] Enhance the automatic index logic so that it creates a partial index when doing so gives the same answer for less work. FossilOrigin-Name: d95d0313c447f5baeabdb17284d8606331ab7d49 --- manifest | 19 +++++----- manifest.uuid | 2 +- src/expr.c | 85 +++++++++++++++++++++++++++----------------- src/resolve.c | 4 +-- src/sqliteInt.h | 5 ++- src/where.c | 23 ++++++++++-- test/autoindex4.test | 52 +++++++++++++++++++++++++++ 7 files changed, 143 insertions(+), 47 deletions(-) create mode 100644 test/autoindex4.test diff --git a/manifest b/manifest index e98be71834..e831ecb526 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Honor\sa\shigh\slikelihood()\son\srange\sconstraints. -D 2014-10-24T15:26:29.800 +C Enhance\sthe\sautomatic\sindex\slogic\sso\sthat\sit\screates\sa\spartial\sindex\swhen\ndoing\sso\sgives\sthe\ssame\sanswer\sfor\sless\swork. +D 2014-10-24T19:28:09.216 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -181,7 +181,7 @@ F src/complete.c 535183afb3c75628b78ce82612931ac7cdf26f14 F src/ctime.c bb434068b5308a857b181c2d204a320ff0d6c638 F src/date.c 57a7f9ba9f6b4d5268f5e411739066a611f99036 F src/delete.c fae81cc2eb14b75267d4f47d3cfc9ae02aae726f -F src/expr.c fc204d08af06437ddaffe5a1b1f1f6f9e1a55d6d +F src/expr.c 0391a657df4959eaf2a2fd7d77de5ebe750686ee F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c da985ae673efef2c712caef825a5d2edb087ead7 F src/func.c ba47c1671ab3cfdafa6e9d6ee490939ea578adee @@ -225,14 +225,14 @@ F src/pragma.c 3f3e959390a10c0131676f0e307acce372777e0f F src/prepare.c 6ef0cf2f9274982988ed6b7cab1be23147e94196 F src/printf.c 090fac0f779c93c8a95089a125339686648835e4 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece -F src/resolve.c a3466128b52a86c466e47ac1a19e2174f7b5cf89 +F src/resolve.c 57d5ad93913beb43ad9b8ade435a54e5fb8ccd40 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c 282f8f5278e0c78eb442217531172ec9e1538796 F src/sqlite.h.in 4a5e5158c189d2bcd45c7c4607c2c0eb6d25c153 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d -F src/sqliteInt.h 6e9e125698c1e5c78a51050ea61f179a281c766d +F src/sqliteInt.h 123b28f3552d4ffdd3e53707fe8120a069df69e4 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 961d5926e5a8fda611d385ec22c226b8635cd1cb F src/table.c 2e99ef7ef16187e17033d9398dc962ce22dab5cb @@ -302,7 +302,7 @@ F src/vtab.c cb0c194303fea276b48d7d4b6d970b5a96bde8de F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c f5c13d9c1929bcc9d571f1e7bf7bfeb8b872ef99 +F src/where.c cc0733c59bd8bf6027d28b7d24b1887f4a870215 F src/whereInt.h 4b459cdbfc9b01f5f27673a35f9967e4dea917e8 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -346,6 +346,7 @@ F test/autoinc.test c58912526998a39e11f66b533e23cfabea7f25b7 F test/autoindex1.test 6ff78b94f43a59616c06c11c55b12935173506d7 F test/autoindex2.test 60d2fc6f38364308ce73a9beb01b47ded38697de F test/autoindex3.test 8254f689c3241081fad52b7bea18ba53e07e14a2 +F test/autoindex4.test fc807f9efd158bec60f5dfdf34ebe46fb274612d F test/autovacuum.test 941892505d2c0f410a0cb5970dfa1c7c4e5f6e74 F test/autovacuum_ioerr2.test 8a367b224183ad801e0e24dcb7d1501f45f244b4 F test/avtrans.test 0252654f4295ddda3b2cce0e894812259e655a85 @@ -1205,7 +1206,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 03d0498d0f24bec2383d5d79edf25069effecd59 -R eb013b99b9ff12a41d5ffc947e4a6227 +P 401235edf40fcd665eaf426cf5155ac6855e8537 +R def8f7d87d0f5784d9eaab2c56b2a690 U drh -Z 3215dd92b9608b7b9136fb91eb8178e8 +Z cb611c4b71e8fc233ea4e1005f56a9e0 diff --git a/manifest.uuid b/manifest.uuid index 4ee03d34ab..7417ef4fc1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -401235edf40fcd665eaf426cf5155ac6855e8537 \ No newline at end of file +d95d0313c447f5baeabdb17284d8606331ab7d49 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 1ad9a879a3..13a9cb46fd 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1210,20 +1210,24 @@ void sqlite3ExprListDelete(sqlite3 *db, ExprList *pList){ } /* -** These routines are Walker callbacks. Walker.u.pi is a pointer -** to an integer. These routines are checking an expression to see -** if it is a constant. Set *Walker.u.i to 0 if the expression is -** not constant. +** These routines are Walker callbacks used to check expressions to +** see if they are "constant" for some definition of constant. The +** Walker.eCode value determines the type of "constant" we are looking +** for. ** ** These callback routines are used to implement the following: ** -** sqlite3ExprIsConstant() pWalker->u.i==1 -** sqlite3ExprIsConstantNotJoin() pWalker->u.i==2 -** sqlite3ExprIsConstantOrFunction() pWalker->u.i==3 or 4 +** sqlite3ExprIsConstant() pWalker->eCode==1 +** sqlite3ExprIsConstantNotJoin() pWalker->eCode==2 +** sqlite3ExprRefOneTableOnly() pWalker->eCode==3 +** sqlite3ExprIsConstantOrFunction() pWalker->eCode==4 or 5 +** +** In all cases, the callbacks set Walker.eCode=0 and abort if the expression +** is found to not be a constant. ** ** The sqlite3ExprIsConstantOrFunction() is used for evaluating expressions -** in a CREATE TABLE statement. The Walker.u.i value is 4 when parsing -** an existing schema and 3 when processing a new statement. A bound +** in a CREATE TABLE statement. The Walker.eCode value is 5 when parsing +** an existing schema and 4 when processing a new statement. A bound ** parameter raises an error for new statements, but is silently converted ** to NULL for existing schemas. This allows sqlite_master tables that ** contain a bound parameter because they were generated by older versions @@ -1232,23 +1236,25 @@ void sqlite3ExprListDelete(sqlite3 *db, ExprList *pList){ */ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ - /* If pWalker->u.i is 2 then any term of the expression that comes from - ** the ON or USING clauses of a join disqualifies the expression + /* If pWalker->eCode is 2 then any term of the expression that comes from + ** the ON or USING clauses of a left join disqualifies the expression ** from being considered constant. */ - if( pWalker->u.i==2 && ExprHasProperty(pExpr, EP_FromJoin) ){ - pWalker->u.i = 0; + if( pWalker->eCode==2 && ExprHasProperty(pExpr, EP_FromJoin) ){ + pWalker->eCode = 0; return WRC_Abort; } switch( pExpr->op ){ /* Consider functions to be constant if all their arguments are constant - ** and either pWalker->u.i==3 or 4 or the function as the SQLITE_FUNC_CONST - ** flag. */ + ** and either pWalker->eCode==4 or 5 or the function has the + ** SQLITE_FUNC_CONST flag. */ case TK_FUNCTION: - if( pWalker->u.i>=3 || ExprHasProperty(pExpr,EP_Constant) ){ + if( pWalker->eCode>=4 || ExprHasProperty(pExpr,EP_Constant) ){ return WRC_Continue; + }else{ + pWalker->eCode = 0; + return WRC_Abort; } - /* Fall through */ case TK_ID: case TK_COLUMN: case TK_AGG_FUNCTION: @@ -1257,18 +1263,22 @@ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ testcase( pExpr->op==TK_COLUMN ); testcase( pExpr->op==TK_AGG_FUNCTION ); testcase( pExpr->op==TK_AGG_COLUMN ); - pWalker->u.i = 0; - return WRC_Abort; + if( pWalker->eCode==3 && pExpr->iTable==pWalker->u.iCur ){ + return WRC_Continue; + }else{ + pWalker->eCode = 0; + return WRC_Abort; + } case TK_VARIABLE: - if( pWalker->u.i==4 ){ + if( pWalker->eCode==5 ){ /* Silently convert bound parameters that appear inside of CREATE ** statements into a NULL when parsing the CREATE statement text out ** of the sqlite_master table */ pExpr->op = TK_NULL; - }else if( pWalker->u.i==3 ){ + }else if( pWalker->eCode==4 ){ /* A bound parameter in a CREATE statement that originates from ** sqlite3_prepare() causes an error */ - pWalker->u.i = 0; + pWalker->eCode = 0; return WRC_Abort; } /* Fall through */ @@ -1280,21 +1290,22 @@ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ } static int selectNodeIsConstant(Walker *pWalker, Select *NotUsed){ UNUSED_PARAMETER(NotUsed); - pWalker->u.i = 0; + pWalker->eCode = 0; return WRC_Abort; } -static int exprIsConst(Expr *p, int initFlag){ +static int exprIsConst(Expr *p, int initFlag, int iCur){ Walker w; memset(&w, 0, sizeof(w)); - w.u.i = initFlag; + w.eCode = initFlag; w.xExprCallback = exprNodeIsConstant; w.xSelectCallback = selectNodeIsConstant; + w.u.iCur = iCur; sqlite3WalkExpr(&w, p); - return w.u.i; + return w.eCode; } /* -** Walk an expression tree. Return 1 if the expression is constant +** Walk an expression tree. Return non-zero if the expression is constant ** and 0 if it involves variables or function calls. ** ** For the purposes of this function, a double-quoted string (ex: "abc") @@ -1302,21 +1313,31 @@ static int exprIsConst(Expr *p, int initFlag){ ** a constant. */ int sqlite3ExprIsConstant(Expr *p){ - return exprIsConst(p, 1); + return exprIsConst(p, 1, 0); } /* -** Walk an expression tree. Return 1 if the expression is constant +** Walk an expression tree. Return non-zero if the expression is constant ** that does no originate from the ON or USING clauses of a join. ** Return 0 if it involves variables or function calls or terms from ** an ON or USING clause. */ int sqlite3ExprIsConstantNotJoin(Expr *p){ - return exprIsConst(p, 2); + return exprIsConst(p, 2, 0); } /* -** Walk an expression tree. Return 1 if the expression is constant +** Walk an expression tree. Return non-zero if the expression constant +** for any single row of the table with cursor iCur. In other words, the +** expression must not refer to any non-deterministic function nor any +** table other than iCur. +*/ +int sqlite3ExprIsTableConstant(Expr *p, int iCur){ + return exprIsConst(p, 3, iCur); +} + +/* +** Walk an expression tree. Return non-zero if the expression is constant ** or a function call with constant arguments. Return and 0 if there ** are any variables. ** @@ -1326,7 +1347,7 @@ int sqlite3ExprIsConstantNotJoin(Expr *p){ */ int sqlite3ExprIsConstantOrFunction(Expr *p, u8 isInit){ assert( isInit==0 || isInit==1 ); - return exprIsConst(p, 3+isInit); + return exprIsConst(p, 4+isInit, 0); } /* diff --git a/src/resolve.c b/src/resolve.c index d6a865caef..e507ccb810 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -28,7 +28,7 @@ ** is a helper function - a callback for the tree walker. */ static int incrAggDepth(Walker *pWalker, Expr *pExpr){ - if( pExpr->op==TK_AGG_FUNCTION ) pExpr->op2 += pWalker->u.i; + if( pExpr->op==TK_AGG_FUNCTION ) pExpr->op2 += pWalker->u.n; return WRC_Continue; } static void incrAggFunctionDepth(Expr *pExpr, int N){ @@ -36,7 +36,7 @@ static void incrAggFunctionDepth(Expr *pExpr, int N){ Walker w; memset(&w, 0, sizeof(w)); w.xExprCallback = incrAggDepth; - w.u.i = N; + w.u.n = N; sqlite3WalkExpr(&w, pExpr); } } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 1b3138be44..b7f992c34d 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2892,9 +2892,11 @@ struct Walker { void (*xSelectCallback2)(Walker*,Select*);/* Second callback for SELECTs */ Parse *pParse; /* Parser context. */ int walkerDepth; /* Number of subqueries */ + u8 eCode; /* A small processing code */ union { /* Extra data for callback */ NameContext *pNC; /* Naming context */ - int i; /* Integer value */ + int n; /* A counter */ + int iCur; /* A cursor number */ SrcList *pSrcList; /* FROM clause */ struct SrcCount *pSrcCount; /* Counting column references */ } u; @@ -3295,6 +3297,7 @@ void sqlite3LeaveMutexAndCloseZombie(sqlite3*); int sqlite3ExprIsConstant(Expr*); int sqlite3ExprIsConstantNotJoin(Expr*); int sqlite3ExprIsConstantOrFunction(Expr*, u8); +int sqlite3ExprIsTableConstant(Expr*,int); int sqlite3ExprIsInteger(Expr*, int*); int sqlite3ExprCanBeNull(const Expr*); int sqlite3ExprNeedsNoAffinityChange(const Expr*, char); diff --git a/src/where.c b/src/where.c index e13b223366..8d2a891870 100644 --- a/src/where.c +++ b/src/where.c @@ -1594,6 +1594,8 @@ static void constructAutomaticIndex( Bitmask idxCols; /* Bitmap of columns used for indexing */ Bitmask extraCols; /* Bitmap of additional columns */ u8 sentWarning = 0; /* True if a warnning has been issued */ + Expr *pPartial = 0; /* Partial Index Expression */ + int iContinue = 0; /* Jump here to skip excluded rows */ /* Generate code to skip over the creation and initialization of the ** transient index on 2nd and subsequent iterations of the loop. */ @@ -1609,6 +1611,11 @@ static void constructAutomaticIndex( pLoop = pLevel->pWLoop; idxCols = 0; for(pTerm=pWC->a; pTermprereq==0 + && sqlite3ExprIsTableConstant(pTerm->pExpr, pSrc->iCursor) ){ + pPartial = sqlite3ExprAnd(pParse->db, pPartial, + sqlite3ExprDup(pParse->db, pTerm->pExpr, 0)); + } if( termCanDriveIndex(pTerm, pSrc, notReady) ){ int iCol = pTerm->u.leftColumn; Bitmask cMask = iCol>=BMS ? MASKBIT(BMS-1) : MASKBIT(iCol); @@ -1621,7 +1628,9 @@ static void constructAutomaticIndex( sentWarning = 1; } if( (idxCols & cMask)==0 ){ - if( whereLoopResize(pParse->db, pLoop, nKeyCol+1) ) return; + if( whereLoopResize(pParse->db, pLoop, nKeyCol+1) ){ + goto end_auto_index_create; + } pLoop->aLTerm[nKeyCol++] = pTerm; idxCols |= cMask; } @@ -1654,7 +1663,7 @@ static void constructAutomaticIndex( /* Construct the Index object to describe this index */ pIdx = sqlite3AllocateIndexObject(pParse->db, nKeyCol+1, 0, &zNotUsed); - if( pIdx==0 ) return; + if( pIdx==0 ) goto end_auto_index_create; pLoop->u.btree.pIndex = pIdx; pIdx->zName = "auto-index"; pIdx->pTable = pTable; @@ -1706,18 +1715,28 @@ static void constructAutomaticIndex( VdbeComment((v, "for %s", pTable->zName)); /* Fill the automatic index with content */ + sqlite3ExprCachePush(pParse); addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, pLevel->iTabCur); VdbeCoverage(v); + if( pPartial ){ + iContinue = sqlite3VdbeMakeLabel(v); + sqlite3ExprIfFalse(pParse, pPartial, iContinue, SQLITE_JUMPIFNULL); + } regRecord = sqlite3GetTempReg(pParse); sqlite3GenerateIndexKey(pParse, pIdx, pLevel->iTabCur, regRecord, 0, 0, 0, 0); sqlite3VdbeAddOp2(v, OP_IdxInsert, pLevel->iIdxCur, regRecord); sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); + if( pPartial ) sqlite3VdbeResolveLabel(v, iContinue); sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1); VdbeCoverage(v); sqlite3VdbeChangeP5(v, SQLITE_STMTSTATUS_AUTOINDEX); sqlite3VdbeJumpHere(v, addrTop); sqlite3ReleaseTempReg(pParse, regRecord); + sqlite3ExprCachePop(pParse); /* Jump here when skipping the initialization */ sqlite3VdbeJumpHere(v, addrInit); + +end_auto_index_create: + sqlite3ExprDelete(pParse->db, pPartial); } #endif /* SQLITE_OMIT_AUTOMATIC_INDEX */ diff --git a/test/autoindex4.test b/test/autoindex4.test new file mode 100644 index 0000000000..6d0865bf72 --- /dev/null +++ b/test/autoindex4.test @@ -0,0 +1,52 @@ +# 2014-10-24 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# +# This file implements regression tests for SQLite library. The +# focus of this script is testing automatic index creation logic, +# and specifically creation of automatic partial indexes. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +do_execsql_test autoindex4-1.0 { + CREATE TABLE t1(a,b); + INSERT INTO t1 VALUES(123,'abc'),(234,'def'),(234,'ghi'),(345,'jkl'); + CREATE TABLE t2(x,y); + INSERT INTO t2 VALUES(987,'zyx'),(654,'wvu'),(987,'rqp'); + + SELECT *, '|' FROM t1, t2 WHERE a=234 AND x=987 ORDER BY +b; +} {234 def 987 rqp | 234 def 987 zyx | 234 ghi 987 rqp | 234 ghi 987 zyx |} +do_execsql_test autoindex4-1.1 { + SELECT *, '|' FROM t1, t2 WHERE a=234 AND x=555; +} {} + +do_execsql_test autoindex4-1.2 { + SELECT *, '|' FROM t1 LEFT JOIN t2 ON a=234 AND x=555; +} {123 abc {} {} | 234 def {} {} | 234 ghi {} {} | 345 jkl {} {} |} +do_execsql_test autoindex4-1.3 { + SELECT *, '|' FROM t1 LEFT JOIN t2 ON x=555 WHERE a=234; +} {234 def {} {} | 234 ghi {} {} |} +do_execsql_test autoindex4-1.4 { + SELECT *, '|' FROM t1 LEFT JOIN t2 WHERE a=234 AND x=555; +} {} + + +do_execsql_test autoindex4-2.0 { + CREATE TABLE t3(e,f); + INSERT INTO t3 VALUES(123,654),(555,444),(234,987); + + SELECT (SELECT count(*) FROM t1, t2 WHERE a=e AND x=f), e, f, '|' + FROM t3 + ORDER BY rowid; +} {1 123 654 | 0 555 444 | 4 234 987 |} + +finish_test From 31f4e99d449b5938f46723f6d2ec96595ff88513 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 24 Oct 2014 20:57:03 +0000 Subject: [PATCH 021/133] Ensure that the "Any prior cache entry associated with newKey is guaranteed not to be pinned" guarantee made to xRekey implementations is not violated. FossilOrigin-Name: ecc3544e712041736af7c7b4f34864a1f2e30ff7 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/btree.c | 30 ++++++++++++------------------ src/pager.c | 5 +---- 4 files changed, 21 insertions(+), 30 deletions(-) diff --git a/manifest b/manifest index 10be86a2cc..6c0464217b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\ssome\sissues\sin\sthe\snew\scode\son\sthis\sbranch\srelated\sto\sthe\shandling\sof\scorrupt\sdatabases. -D 2014-10-24T18:43:57.313 +C Ensure\sthat\sthe\s"Any\sprior\scache\sentry\sassociated\swith\snewKey\sis\sguaranteed\snot\sto\sbe\spinned"\sguarantee\smade\sto\sxRekey\simplementations\sis\snot\sviolated. +D 2014-10-24T20:57:03.500 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c df15daf116e9ef1a7ff88257f6fb678ba385b5ea +F src/btree.c 4decfb3b97d16afdd0e5a7e5f876af1f528e8a69 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 9dc2bd94347b878c89627000c92b0c8d97ec2919 @@ -215,7 +215,7 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c fb587121840f690101336879adfa6d0b2cd0e8c7 F src/os_win.c a019caaae2bcbbc0cc4c39af6e7d7e43d8426053 F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 -F src/pager.c d02833adf331a5913226595306d64731a3da33f6 +F src/pager.c 8d97b3633f098fef817656dcbf167ca904511d78 F src/pager.h d1eee3c3f741be247ce6d82752a178515fc8578b F src/parse.y 5dfead8aed90cb0c7c1115898ee2266804daff45 F src/pcache.c 4121a0571c18581ee9f82f086d5e2030051ebd6a @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P eab8706dc47aa0a44caf73619de858397c3e0b4e -R 8a34c36d0ed8bbbc0bb7360d0dc30d47 +P 19736dd9fbbb7e252c4f8715e2277d48ac41f5bc +R c9d1872515b77329bc9fbfe4c8e91668 U dan -Z 610a2814ec6cdf214c41e43dd2355678 +Z 931e363227b1144b3ac19bee87a82036 diff --git a/manifest.uuid b/manifest.uuid index 11462e357c..470b2345ea 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -19736dd9fbbb7e252c4f8715e2277d48ac41f5bc \ No newline at end of file +ecc3544e712041736af7c7b4f34864a1f2e30ff7 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index f326f713fb..83ed66414c 100644 --- a/src/btree.c +++ b/src/btree.c @@ -6856,25 +6856,19 @@ static int balance_nonroot( aPgFlags[i] = apNew[i]->pDbPage->flags; } for(i=0; ipgno); - Pgno iMin = 0; - u16 flags = 0; - for(j=0; jiGt && (iMin==0 || iPgnopgno!=iMin ){ - sqlite3PagerRekey(apNew[i]->pDbPage, iMin, flags); - apNew[i]->pgno = iMin; + pgno = aPgno[iBest]; + aPgno[iBest] = 0xffffffff; + if( iBest!=i ){ + if( iBest>i ){ + sqlite3PagerRekey(apNew[iBest]->pDbPage, pBt->nPage+iBest+1, 0); + } + sqlite3PagerRekey(apNew[i]->pDbPage, pgno, aPgFlags[iBest]); + apNew[i]->pgno = pgno; } } diff --git a/src/pager.c b/src/pager.c index dc79e6754a..997f842d0a 100644 --- a/src/pager.c +++ b/src/pager.c @@ -6853,10 +6853,7 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ ** page number to iNew and sets the value of the PgHdr.flags field to ** the value passed as the third parameter. */ -void sqlite3PagerRekey(DbPage *pPage, Pgno iNew, u16 flags){ - PgHdr *pPg = (PgHdr*)pPage; - assert( (flags & PGHDR_DIRTY) && (pPg->flags & PGHDR_DIRTY) ); - assert( !subjRequiresPage(pPg) ); +void sqlite3PagerRekey(DbPage *pPg, Pgno iNew, u16 flags){ assert( pPg->pgno!=iNew ); pPg->flags = flags; sqlite3PcacheMove(pPg, iNew); From 051575cbf4c47fc472f53146e30d6ddddc9054b3 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 25 Oct 2014 12:28:25 +0000 Subject: [PATCH 022/133] Do not use virtual (and hence redundant) WHERE-clause terms to restrict the content of a automatic partial index. Show when an automatic partial index is used in the EXPLAIN QUERY PLAN output. FossilOrigin-Name: b9ad601eab1d7298d369267eb697c7fa1bc16985 --- ext/rtree/rtree6.test | 2 +- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/where.c | 4 ++++ src/whereInt.h | 1 + 5 files changed, 15 insertions(+), 10 deletions(-) diff --git a/ext/rtree/rtree6.test b/ext/rtree/rtree6.test index cec3a8da41..c9c87e8ad9 100644 --- a/ext/rtree/rtree6.test +++ b/ext/rtree/rtree6.test @@ -102,7 +102,7 @@ do_eqp_test rtree6.2.4.2 { SELECT * FROM t1,t2 WHERE v=10 and x1<10 and x2>10 } { 0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:C0E1} - 0 1 1 {SEARCH TABLE t2 USING AUTOMATIC COVERING INDEX (v=?)} + 0 1 1 {SEARCH TABLE t2 USING AUTOMATIC PARTIAL COVERING INDEX (v=?)} } do_eqp_test rtree6.2.5 { diff --git a/manifest b/manifest index e831ecb526..28fa8e1387 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sthe\sautomatic\sindex\slogic\sso\sthat\sit\screates\sa\spartial\sindex\swhen\ndoing\sso\sgives\sthe\ssame\sanswer\sfor\sless\swork. -D 2014-10-24T19:28:09.216 +C Do\snot\suse\svirtual\s(and\shence\sredundant)\sWHERE-clause\sterms\sto\srestrict\sthe\ncontent\sof\sa\sautomatic\spartial\sindex.\s\sShow\swhen\san\sautomatic\spartial\sindex\nis\sused\sin\sthe\sEXPLAIN\sQUERY\sPLAN\soutput. +D 2014-10-25T12:28:25.871 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -130,7 +130,7 @@ F ext/rtree/rtree2.test acbb3a4ce0f4fbc2c304d2b4b784cfa161856bba F ext/rtree/rtree3.test a494da55c30ee0bc9b01a91c80c81b387b22d2dc F ext/rtree/rtree4.test c8fe384f60ebd49540a5fecc990041bf452eb6e0 F ext/rtree/rtree5.test 6a510494f12454bf57ef28f45bc7764ea279431e -F ext/rtree/rtree6.test 0cfbdf27ee086bf16a3da2c6f2d5b3d6473cb27e +F ext/rtree/rtree6.test 773a90db2dce6a8353dd0d5b64bca69b29761196 F ext/rtree/rtree7.test 1fa710b9e6bf997a0c1a537b81be7bb6fded1971 F ext/rtree/rtree8.test db79c812f9e4a11f9b1f3f9934007884610a713a F ext/rtree/rtree9.test d86ebf08ff6328895613ed577dd8a2a37c472c34 @@ -302,8 +302,8 @@ F src/vtab.c cb0c194303fea276b48d7d4b6d970b5a96bde8de F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c cc0733c59bd8bf6027d28b7d24b1887f4a870215 -F src/whereInt.h 4b459cdbfc9b01f5f27673a35f9967e4dea917e8 +F src/where.c 3312adfda33a9ca85c8a380a642a1a5905398b06 +F src/whereInt.h 19279cd0664ce1d90b9ad3ef0108cb494acfe455 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test b35b4cd69fc913f90d39a575e171e1116c3a4bb7 @@ -1206,7 +1206,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 401235edf40fcd665eaf426cf5155ac6855e8537 -R def8f7d87d0f5784d9eaab2c56b2a690 +P d95d0313c447f5baeabdb17284d8606331ab7d49 +R 9499438a1fccf595ef80fbd4c31b20b8 U drh -Z cb611c4b71e8fc233ea4e1005f56a9e0 +Z cdff8327924b867b12e561826d5fe961 diff --git a/manifest.uuid b/manifest.uuid index 7417ef4fc1..8b94233ae7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d95d0313c447f5baeabdb17284d8606331ab7d49 \ No newline at end of file +b9ad601eab1d7298d369267eb697c7fa1bc16985 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 8d2a891870..2e42b8fc32 100644 --- a/src/where.c +++ b/src/where.c @@ -1612,6 +1612,7 @@ static void constructAutomaticIndex( idxCols = 0; for(pTerm=pWC->a; pTermprereq==0 + && (pTerm->wtFlags & TERM_VIRTUAL)==0 && sqlite3ExprIsTableConstant(pTerm->pExpr, pSrc->iCursor) ){ pPartial = sqlite3ExprAnd(pParse->db, pPartial, sqlite3ExprDup(pParse->db, pTerm->pExpr, 0)); @@ -1720,6 +1721,7 @@ static void constructAutomaticIndex( if( pPartial ){ iContinue = sqlite3VdbeMakeLabel(v); sqlite3ExprIfFalse(pParse, pPartial, iContinue, SQLITE_JUMPIFNULL); + pLoop->wsFlags |= WHERE_PARTIALIDX; } regRecord = sqlite3GetTempReg(pParse); sqlite3GenerateIndexKey(pParse, pIdx, pLevel->iTabCur, regRecord, 0, 0, 0, 0); @@ -2865,6 +2867,8 @@ static void explainOneScan( if( isSearch ){ zFmt = "PRIMARY KEY"; } + }else if( flags & WHERE_PARTIALIDX ){ + zFmt = "AUTOMATIC PARTIAL COVERING INDEX"; }else if( flags & WHERE_AUTO_INDEX ){ zFmt = "AUTOMATIC COVERING INDEX"; }else if( flags & WHERE_IDX_ONLY ){ diff --git a/src/whereInt.h b/src/whereInt.h index e9eb8b7dd2..fd4cfdf88a 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -459,3 +459,4 @@ struct WhereInfo { #define WHERE_AUTO_INDEX 0x00004000 /* Uses an ephemeral index */ #define WHERE_SKIPSCAN 0x00008000 /* Uses the skip-scan algorithm */ #define WHERE_UNQ_WANTED 0x00010000 /* WHERE_ONEROW would have been helpful*/ +#define WHERE_PARTIALIDX 0x00020000 /* The automatic index is partial */ From d05ab6aacf08a1fd7af581cae4b28390ecd5c44c Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 25 Oct 2014 13:42:16 +0000 Subject: [PATCH 023/133] Increase the resolution of the second parameter to the likelihood() SQL function (the probability value) so that it can handle probabilities as small as 0.00000001. Formerly, it ran out of precision at 0.001. FossilOrigin-Name: 0f08924fe0c52a85a103f67bee9809e0f8f884b0 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/resolve.c | 4 ++-- src/sqliteInt.h | 2 +- src/where.c | 2 +- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 28fa8e1387..6d8395af5e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\suse\svirtual\s(and\shence\sredundant)\sWHERE-clause\sterms\sto\srestrict\sthe\ncontent\sof\sa\sautomatic\spartial\sindex.\s\sShow\swhen\san\sautomatic\spartial\sindex\nis\sused\sin\sthe\sEXPLAIN\sQUERY\sPLAN\soutput. -D 2014-10-25T12:28:25.871 +C Increase\sthe\sresolution\sof\sthe\ssecond\sparameter\sto\sthe\slikelihood()\sSQL\nfunction\s(the\sprobability\svalue)\sso\sthat\sit\scan\shandle\sprobabilities\nas\ssmall\sas\s0.00000001.\s\sFormerly,\sit\sran\sout\sof\sprecision\sat\s0.001. +D 2014-10-25T13:42:16.126 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -225,14 +225,14 @@ F src/pragma.c 3f3e959390a10c0131676f0e307acce372777e0f F src/prepare.c 6ef0cf2f9274982988ed6b7cab1be23147e94196 F src/printf.c 090fac0f779c93c8a95089a125339686648835e4 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece -F src/resolve.c 57d5ad93913beb43ad9b8ade435a54e5fb8ccd40 +F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c 282f8f5278e0c78eb442217531172ec9e1538796 F src/sqlite.h.in 4a5e5158c189d2bcd45c7c4607c2c0eb6d25c153 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d -F src/sqliteInt.h 123b28f3552d4ffdd3e53707fe8120a069df69e4 +F src/sqliteInt.h 90519c3b3e8ee90adfce013234c4bd07275d77b5 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 961d5926e5a8fda611d385ec22c226b8635cd1cb F src/table.c 2e99ef7ef16187e17033d9398dc962ce22dab5cb @@ -302,7 +302,7 @@ F src/vtab.c cb0c194303fea276b48d7d4b6d970b5a96bde8de F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 3312adfda33a9ca85c8a380a642a1a5905398b06 +F src/where.c 4ce8c4826b7f86d080f0ed4e7a9045bb5014be77 F src/whereInt.h 19279cd0664ce1d90b9ad3ef0108cb494acfe455 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1206,7 +1206,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P d95d0313c447f5baeabdb17284d8606331ab7d49 -R 9499438a1fccf595ef80fbd4c31b20b8 +P b9ad601eab1d7298d369267eb697c7fa1bc16985 +R 06fd31d9e4c300e561fccc4cf28a702b U drh -Z cdff8327924b867b12e561826d5fe961 +Z 628a0c5751868817bd40f1740c611aef diff --git a/manifest.uuid b/manifest.uuid index 8b94233ae7..3804da23b9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b9ad601eab1d7298d369267eb697c7fa1bc16985 \ No newline at end of file +0f08924fe0c52a85a103f67bee9809e0f8f884b0 \ No newline at end of file diff --git a/src/resolve.c b/src/resolve.c index e507ccb810..8fb580b3a1 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -584,7 +584,7 @@ static int exprProbability(Expr *p){ sqlite3AtoF(p->u.zToken, &r, sqlite3Strlen30(p->u.zToken), SQLITE_UTF8); assert( r>=0.0 ); if( r>1.0 ) return -1; - return (int)(r*1000.0); + return (int)(r*134217728.0); } /* @@ -716,7 +716,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ ** EVIDENCE-OF: R-53436-40973 The likely(X) function is equivalent to ** likelihood(X,0.9375). */ /* TUNING: unlikely() probability is 0.0625. likely() is 0.9375 */ - pExpr->iTable = pDef->zName[0]=='u' ? 62 : 938; + pExpr->iTable = pDef->zName[0]=='u' ? 8388608 : 125829120; } } #ifndef SQLITE_OMIT_AUTHORIZATION diff --git a/src/sqliteInt.h b/src/sqliteInt.h index b7f992c34d..5114ccccf4 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2000,7 +2000,7 @@ struct Expr { int iTable; /* TK_COLUMN: cursor number of table holding column ** TK_REGISTER: register number ** TK_TRIGGER: 1 -> new, 0 -> old - ** EP_Unlikely: 1000 times likelihood */ + ** EP_Unlikely: 134217728 times likelihood */ ynVar iColumn; /* TK_COLUMN: column index. -1 for rowid. ** TK_VARIABLE: variable number (always >= 1). */ i16 iAgg; /* Which entry in pAggInfo->aCol[] or ->aFunc[] */ diff --git a/src/where.c b/src/where.c index 2e42b8fc32..358d508260 100644 --- a/src/where.c +++ b/src/where.c @@ -225,7 +225,7 @@ static int whereClauseInsert(WhereClause *pWC, Expr *p, u8 wtFlags){ } pTerm = &pWC->a[idx = pWC->nTerm++]; if( p && ExprHasProperty(p, EP_Unlikely) ){ - pTerm->truthProb = sqlite3LogEst(p->iTable) - 99; + pTerm->truthProb = sqlite3LogEst(p->iTable) - 270; }else{ pTerm->truthProb = 1; } From 89ca0b3878bf964c202e1d0c984b39a9eefaafb0 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 25 Oct 2014 20:36:28 +0000 Subject: [PATCH 024/133] Further modifications to new code to better handle corrupt databases. FossilOrigin-Name: 1a8cf0a043347772ac54d150d634c32845beee8b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 30 +++++++++++++++++++++++------- 3 files changed, 30 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 6c0464217b..1a5be4bf57 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Ensure\sthat\sthe\s"Any\sprior\scache\sentry\sassociated\swith\snewKey\sis\sguaranteed\snot\sto\sbe\spinned"\sguarantee\smade\sto\sxRekey\simplementations\sis\snot\sviolated. -D 2014-10-24T20:57:03.500 +C Further\smodifications\sto\snew\scode\sto\sbetter\shandle\scorrupt\sdatabases. +D 2014-10-25T20:36:28.557 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 4decfb3b97d16afdd0e5a7e5f876af1f528e8a69 +F src/btree.c 2639b89f6728f5775044704c7757c0226e071bd1 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 9dc2bd94347b878c89627000c92b0c8d97ec2919 @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 19736dd9fbbb7e252c4f8715e2277d48ac41f5bc -R c9d1872515b77329bc9fbfe4c8e91668 +P ecc3544e712041736af7c7b4f34864a1f2e30ff7 +R 17dc589efbb3f16f6448c9d9d1ae2e24 U dan -Z 931e363227b1144b3ac19bee87a82036 +Z c615066e82cbd286f5597ce396069fd7 diff --git a/manifest.uuid b/manifest.uuid index 470b2345ea..23643921ec 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ecc3544e712041736af7c7b4f34864a1f2e30ff7 \ No newline at end of file +1a8cf0a043347772ac54d150d634c32845beee8b \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 83ed66414c..46ce39abc7 100644 --- a/src/btree.c +++ b/src/btree.c @@ -6099,6 +6099,7 @@ static int pageFreeArray( ){ u8 * const aData = pPg->aData; u8 * const pEnd = &aData[pPg->pBt->usableSize]; + u8 * const pStart = &aData[pPg->hdrOffset + 8 + pPg->childPtrSize]; int nRet = 0; int i; u8 *pFree = 0; @@ -6106,12 +6107,13 @@ static int pageFreeArray( for(i=0; iaData && pCell=pStart && pCellpEnd ) return 0; }else{ pFree = pCell; szFree += sz; @@ -6854,6 +6856,18 @@ static int balance_nonroot( for(i=0; ipgno; aPgFlags[i] = apNew[i]->pDbPage->flags; + for(j=0; jnFree == - (get2byte(&apNew[0]->aData[5])-apNew[0]->cellOffset-apNew[0]->nCell*2) - ); - copyNodeContent(apNew[0], pParent, &rc); - freePage(apNew[0], &rc); + rc = defragmentPage(apNew[0]); + if( rc==SQLITE_OK ){ + assert( apNew[0]->nFree == + (get2byte(&apNew[0]->aData[5])-apNew[0]->cellOffset-apNew[0]->nCell*2) + ); + copyNodeContent(apNew[0], pParent, &rc); + freePage(apNew[0], &rc); + } }else if( ISAUTOVACUUM && !leafCorrection ){ /* Fix the pointer map entries associated with the right-child of each ** sibling page. All other pointer map entries have already been taken From 61e94c9deb9d0edff1fdaa443fe7668cbfacedb4 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 27 Oct 2014 08:02:16 +0000 Subject: [PATCH 025/133] If a free-slot is found within a page, but using that free-slot would fragment the page further and there are already at least 60 fragmented bytes, degragment the page. This matches the behaviour of the trunk. FossilOrigin-Name: 1f80f8c136ac970dcc7fb2337263dc5922e348c3 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 18 ++++++++++++++---- 3 files changed, 21 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index bafcf0fb91..a088218356 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\strunk\swith\sthis\sbranch. -D 2014-10-27T07:01:04.796 +C If\sa\sfree-slot\sis\sfound\swithin\sa\spage,\sbut\susing\sthat\sfree-slot\swould\sfragment\sthe\spage\sfurther\sand\sthere\sare\salready\sat\sleast\s60\sfragmented\sbytes,\sdegragment\sthe\spage.\sThis\smatches\sthe\sbehaviour\sof\sthe\strunk. +D 2014-10-27T08:02:16.531 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 2639b89f6728f5775044704c7757c0226e071bd1 +F src/btree.c 9790fb4df51d36861bcbb8cd0a9b41586cbae699 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 9dc2bd94347b878c89627000c92b0c8d97ec2919 @@ -1206,7 +1206,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 1a8cf0a043347772ac54d150d634c32845beee8b 0f08924fe0c52a85a103f67bee9809e0f8f884b0 -R 9858b90cbf912e211c65716d59c94e8a +P a13df3013bbac4a0d4fce5cef1376c857508c1c5 +R 05f44d14ab419727640258148977021c U dan -Z daf2b363c619dfc32da3ad287a4e25ec +Z 95c7c43e9818bc003f29ce4926da458f diff --git a/manifest.uuid b/manifest.uuid index 962b2e06bb..92680eaebb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a13df3013bbac4a0d4fce5cef1376c857508c1c5 \ No newline at end of file +1f80f8c136ac970dcc7fb2337263dc5922e348c3 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 46ce39abc7..9b14dc2b6e 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1235,8 +1235,12 @@ static int defragmentPage(MemPage *pPage){ ** pRc is non-NULL, then *pRc is set to SQLITE_CORRUPT and NULL is returned. ** Or, if corruption is detected and pRc is NULL, NULL is returned and the ** corruption goes unreported. +** +** If a slot of at least nByte bytes is found but cannot be used because +** there are already at least 60 fragmented bytes on the page, return NULL. +** In this case, if pbDefrag parameter is not NULL, set *pbDefrag to true. */ -static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){ +static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc, int *pbDefrag){ const int hdr = pPg->hdrOffset; u8 * const aData = pPg->aData; int iAddr; @@ -1255,7 +1259,10 @@ static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){ testcase( x==4 ); testcase( x==3 ); if( x<4 ){ - if( aData[hdr+7]>=60 ) return 0; + if( aData[hdr+7]>=60 ){ + if( pbDefrag ) *pbDefrag = 1; + return 0; + } /* Remove the slot from the free-list. Update the number of ** fragmented bytes within the page. */ memcpy(&aData[iAddr], &aData[pc], 2); @@ -1326,8 +1333,10 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ testcase( gap==top ); if( gap+2<=top && (data[hdr+1] || data[hdr+2]) ){ int rc = SQLITE_OK; - u8 *pSpace = pageFindSlot(pPage, nByte, &rc); + int bDefrag = 0; + u8 *pSpace = pageFindSlot(pPage, nByte, &rc, &bDefrag); if( rc ) return rc; + if( bDefrag ) goto defragment_page; if( pSpace ){ *pIdx = pSpace - data; return SQLITE_OK; @@ -1339,6 +1348,7 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ */ testcase( gap+2+nByte==top ); if( gap+2+nByte>top ){ + defragment_page: testcase( pPage->nCell==0 ); rc = defragmentPage(pPage); if( rc ) return rc; @@ -6069,7 +6079,7 @@ static int pageInsertArray( for(i=0; i Date: Mon, 27 Oct 2014 11:25:28 +0000 Subject: [PATCH 026/133] Add test file e_wal.test. FossilOrigin-Name: fc6920b5483eeeb06a474ff399a21afa51dc4859 --- manifest | 13 +-- manifest.uuid | 2 +- test/e_wal.test | 229 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 237 insertions(+), 7 deletions(-) create mode 100644 test/e_wal.test diff --git a/manifest b/manifest index 6d8395af5e..6c3a597a05 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Increase\sthe\sresolution\sof\sthe\ssecond\sparameter\sto\sthe\slikelihood()\sSQL\nfunction\s(the\sprobability\svalue)\sso\sthat\sit\scan\shandle\sprobabilities\nas\ssmall\sas\s0.00000001.\s\sFormerly,\sit\sran\sout\sof\sprecision\sat\s0.001. -D 2014-10-25T13:42:16.126 +C Add\stest\sfile\se_wal.test. +D 2014-10-27T11:25:28.528 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -466,6 +466,7 @@ F test/e_select2.test aceb80ab927d46fba5ce7586ebabf23e2bb0604f F test/e_update.test 312cb8f5ccfe41515a6bb092f8ea562a9bd54d52 F test/e_uri.test 5ae33760fb2039c61aa2d90886f1664664173585 F test/e_vacuum.test 5bfbdc21b65c0abf24398d0ba31dc88d93ca77a9 +F test/e_wal.test 0967f0b8f1dfda871dc7b9b5574198f1f4f7d69a F test/enc.test e54531cd6bf941ee6760be041dff19a104c7acea F test/enc2.test 83437a79ba1545a55fb549309175c683fb334473 F test/enc3.test 90683ad0e6ea587b9d5542ca93568af9a9858c40 @@ -1206,7 +1207,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P b9ad601eab1d7298d369267eb697c7fa1bc16985 -R 06fd31d9e4c300e561fccc4cf28a702b -U drh -Z 628a0c5751868817bd40f1740c611aef +P 0f08924fe0c52a85a103f67bee9809e0f8f884b0 +R 770a2e0b8a993a6006545c850fbc4dd8 +U dan +Z 70c394a419fcdd8591ee5c3f4c5346c4 diff --git a/manifest.uuid b/manifest.uuid index 3804da23b9..a94336a518 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0f08924fe0c52a85a103f67bee9809e0f8f884b0 \ No newline at end of file +fc6920b5483eeeb06a474ff399a21afa51dc4859 \ No newline at end of file diff --git a/test/e_wal.test b/test/e_wal.test new file mode 100644 index 0000000000..a5e074f49b --- /dev/null +++ b/test/e_wal.test @@ -0,0 +1,229 @@ +# 2011 May 06 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix e_wal + +db close +testvfs oldvfs -iversion 1 + + +# EVIDENCE-OF: R-58297-14483 WAL databases can be created, read, and +# written even if shared memory is unavailable as long as the +# locking_mode is set to EXCLUSIVE before the first attempted access. +# +# EVIDENCE-OF: R-00449-33772 This feature allows WAL databases to be +# created, read, and written by legacy VFSes that lack the "version 2" +# shared-memory methods xShmMap, xShmLock, xShmBarrier, and xShmUnmap on +# the sqlite3_io_methods object. +# +# 1.1: "create" tests. +# 1.2: "read" tests. +# 1.3: "write" tests. +# +# All three done with VFS "oldvfs", which has iVersion==1 and so does +# not support shared memory. +# +sqlite3 db test.db -vfs oldvfs +do_execsql_test 1.1.1 { + PRAGMA journal_mode = WAL; +} {delete} +do_execsql_test 1.1.2 { + PRAGMA locking_mode = EXCLUSIVE; + PRAGMA journal_mode = WAL; +} {exclusive wal} +do_execsql_test 1.1.3 { + CREATE TABLE t1(x, y); + INSERT INTO t1 VALUES(1, 2); +} {} +do_test 1.1.4 { + list [file exists test.db-shm] [file exists test.db-wal] +} {0 1} + +do_test 1.2.1 { + db close + sqlite3 db test.db -vfs oldvfs + catchsql { SELECT * FROM t1 } +} {1 {unable to open database file}} +do_test 1.2.2 { + execsql { PRAGMA locking_mode = EXCLUSIVE } + execsql { SELECT * FROM t1 } +} {1 2} +do_test 1.2.3 { + list [file exists test.db-shm] [file exists test.db-wal] +} {0 1} + +do_test 1.3.1 { + db close + sqlite3 db test.db -vfs oldvfs + catchsql { INSERT INTO t1 VALUES(3, 4) } +} {1 {unable to open database file}} +do_test 1.3.2 { + execsql { PRAGMA locking_mode = EXCLUSIVE } + execsql { INSERT INTO t1 VALUES(3, 4) } + execsql { SELECT * FROM t1 } +} {1 2 3 4} +do_test 1.3.3 { + list [file exists test.db-shm] [file exists test.db-wal] +} {0 1} + +# EVIDENCE-OF: R-31969-57825 If EXCLUSIVE locking mode is set prior to +# the first WAL-mode database access, then SQLite never attempts to call +# any of the shared-memory methods and hence no shared-memory wal-index +# is ever created. +# +db close +sqlite3 db test.db +do_execsql_test 2.1.1 { + PRAGMA locking_mode = EXCLUSIVE; + SELECT * FROM t1; +} {exclusive 1 2 3 4} +do_test 2.1.2 { + list [file exists test.db-shm] [file exists test.db-wal] +} {0 1} + +# EVIDENCE-OF: R-36328-16367 In that case, the database connection +# remains in EXCLUSIVE mode as long as the journal mode is WAL; attempts +# to change the locking mode using "PRAGMA locking_mode=NORMAL;" are +# no-ops. +# +do_execsql_test 2.2.1 { + PRAGMA locking_mode = NORMAL; + SELECT * FROM t1; +} {exclusive 1 2 3 4} +do_test 2.2.2 { + sqlite3 db2 test.db + catchsql {SELECT * FROM t1} db2 +} {1 {database is locked}} +db2 close + +# EVIDENCE-OF: R-63522-46088 The only way to change out of EXCLUSIVE +# locking mode is to first change out of WAL journal mode. +# +do_execsql_test 2.3.1 { + PRAGMA journal_mode = DELETE; + SELECT * FROM t1; +} {delete 1 2 3 4} +do_test 2.3.2 { + sqlite3 db2 test.db + catchsql {SELECT * FROM t1} db2 +} {1 {database is locked}} +do_execsql_test 2.3.3 { + PRAGMA locking_mode = NORMAL; + SELECT * FROM t1; +} {normal 1 2 3 4} +do_test 2.3.4 { + sqlite3 db2 test.db + catchsql {SELECT * FROM t1} db2 +} {0 {1 2 3 4}} +db2 close +db close + + +# EVIDENCE-OF: R-57239-11845 If NORMAL locking mode is in effect for the +# first WAL-mode database access, then the shared-memory wal-index is +# created. +# +do_test 3.0 { + sqlite3 db test.db + execsql { PRAGMA journal_mode = WAL } + db close +} {} +do_test 3.1 { + sqlite3 db test.db + execsql { SELECT * FROM t1 } + list [file exists test.db-shm] [file exists test.db-wal] +} {1 1} + +# EVIDENCE-OF: R-13779-07711 As long as exactly one connection is using +# a shared-memory wal-index, the locking mode can be changed freely +# between NORMAL and EXCLUSIVE. +# +do_execsql_test 3.2.1 { + PRAGMA locking_mode = EXCLUSIVE; + PRAGMA locking_mode = NORMAL; + PRAGMA locking_mode = EXCLUSIVE; + INSERT INTO t1 VALUES(5, 6); +} {exclusive normal exclusive} +do_test 3.2.2 { + sqlite3 db2 test.db + catchsql { SELECT * FROM t1 } db2 +} {1 {database is locked}} + +# EVIDENCE-OF: R-10993-11647 It is only when the shared-memory wal-index +# is omitted, when the locking mode is EXCLUSIVE prior to the first +# WAL-mode database access, that the locking mode is stuck in EXCLUSIVE. +# +do_execsql_test 3.2.3 { + PRAGMA locking_mode = NORMAL; + SELECT * FROM t1; +} {normal 1 2 3 4 5 6} +do_test 3.2.4 { + catchsql { SELECT * FROM t1 } db2 +} {0 {1 2 3 4 5 6}} + +do_catchsql_test 3.2.5 { + PRAGMA locking_mode = EXCLUSIVE; + INSERT INTO t1 VALUES(7, 8); +} {1 {database is locked}} + +db2 close + +# EVIDENCE-OF: R-46197-42811 This means that the underlying VFS must +# support the "version 2" shared-memory. +# +# EVIDENCE-OF: R-55316-21772 If the VFS does not support shared-memory +# methods, then the attempt to open a database that is already in WAL +# mode, or the attempt convert a database into WAL mode, will fail. +# +db close +do_test 3.4.1 { + sqlite3 db test.db -vfs oldvfs + catchsql { SELECT * FROM t1 } +} {1 {unable to open database file}} +db close +do_test 3.4.2 { + forcedelete test.db2 + sqlite3 db test.db2 -vfs oldvfs + catchsql { PRAGMA journal_mode = WAL } +} {0 delete} +db close + + +# EVIDENCE-OF: R-22428-28959 To prevent older versions of SQLite from +# trying to recover a WAL-mode database (and making matters worse) the +# database file format version numbers (bytes 18 and 19 in the database +# header) are increased from 1 to 2 in WAL mode. +# +reset_db +do_execsql_test 4.1.1 { CREATE TABLE t1(x, y) } +do_test 4.1.2 { hexio_read test.db 18 2 } {0101} +do_execsql_test 4.1.3 { PRAGMA journal_mode = wAL } {wal} +do_test 4.1.4 { hexio_read test.db 18 2 } {0202} + + +# EVIDENCE-OF: R-02535-05811 One can explicitly change out of WAL mode +# using a pragma such as this: PRAGMA journal_mode=DELETE; +# +do_execsql_test 4.2.1 { INSERT INTO t1 VALUES(1, 1); } {} +do_test 4.2.2 { file exists test.db-wal } {1} +do_execsql_test 4.2.3 { PRAGMA journal_mode = delete } {delete} +do_test 4.2.4 { file exists test.db-wal } {0} + +# EVIDENCE-OF: R-60175-02388 Deliberately changing out of WAL mode +# changes the database file format version numbers back to 1 so that +# older versions of SQLite can once again access the database file. +# +do_test 4.3 { hexio_read test.db 18 2 } {0101} + +finish_test From 8a42d1bba15c0ba571014d95b64d4d74759d39fa Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 27 Oct 2014 18:21:01 +0000 Subject: [PATCH 027/133] Fix an unused variable in btree.c:allocateSpace(). FossilOrigin-Name: 637246165a14c4808b90d0437e4d43fa5fac659e --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/btree.c | 4 +--- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 94ec16445a..3991d3d55b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Optimizations\saimed\sat\sreducing\sthe\snumber\sof\smemcpy()\soperations\srequired\sby\sbalance_nonroot(). -D 2014-10-27T14:26:54.508 +C Fix\san\sunused\svariable\sin\sbtree.c:allocateSpace(). +D 2014-10-27T18:21:01.498 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 9790fb4df51d36861bcbb8cd0a9b41586cbae699 +F src/btree.c 812c03daa8be68daf623dd0349ecb18e9f988967 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 9dc2bd94347b878c89627000c92b0c8d97ec2919 @@ -1207,7 +1207,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P fc6920b5483eeeb06a474ff399a21afa51dc4859 1f80f8c136ac970dcc7fb2337263dc5922e348c3 -R ed2304a6a2e587056109284e9717f063 -U dan -Z 986c15e36469a97fa56839a4b2a55b25 +P face33bea1ba3a6d57780655fa827226b4d2baa9 +R 10f6daecfdf5443eb311db2d487f32a3 +U drh +Z a48d68bafb210b7d63c9203bfcb49102 diff --git a/manifest.uuid b/manifest.uuid index 37f4629c87..28889facf2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -face33bea1ba3a6d57780655fa827226b4d2baa9 \ No newline at end of file +637246165a14c4808b90d0437e4d43fa5fac659e \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 9b14dc2b6e..06dfd2cd53 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1301,7 +1301,6 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ int top; /* First byte of cell content area */ int gap; /* First byte of gap between cell pointers and cell content */ int rc; /* Integer return code */ - int usableSize; /* Usable size of the page */ assert( sqlite3PagerIswriteable(pPage->pDbPage) ); assert( pPage->pBt ); @@ -1309,8 +1308,7 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ assert( nByte>=0 ); /* Minimum cell size is 4 */ assert( pPage->nFree>=nByte ); assert( pPage->nOverflow==0 ); - usableSize = pPage->pBt->usableSize; - assert( nByte < usableSize-8 ); + assert( nByte < pPage->pBt->usableSize-8 ); assert( pPage->cellOffset == hdr + 12 - 4*pPage->leaf ); gap = pPage->cellOffset + 2*pPage->nCell; From 98d94211ced2604da384d16978f783b4f166cfdb Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 27 Oct 2014 19:39:51 +0000 Subject: [PATCH 028/133] Fix a typo in the macro name of an #ifdef FossilOrigin-Name: 9646a136e69cf2583965dfc9fac5f056af4cdb62 --- manifest | 13 ++++++------- manifest.uuid | 2 +- src/ctime.c | 2 +- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index e658f3a76a..3bfe9532df 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sSQLITE_ENABLE_API_ARMOR\scompile-time\soption\sfor\sextra\sAPI\sparameter\nvalidation.\s\sEnhance\ssqlite3_stricmp(),\ssqlite3_strnicmp(),\sand\nsqlite3_uri_parameter()\sfor\simproved\sNULL\sparameter\shandling. -D 2014-10-27T18:34:07.031 +C Fix\sa\stypo\sin\sthe\smacro\sname\sof\san\s#ifdef +D 2014-10-27T19:39:51.561 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -178,7 +178,7 @@ F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 67bb05b1077e0cdaccb2e36bfcbe7a5df9ed31e8 F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 F src/complete.c c4ba6e0626bb94bc77a0861735f3382fcf7cc818 -F src/ctime.c dfa83bfebb4201d07b16534acb8a0149592c3a25 +F src/ctime.c df19848891c8a553c80e6f5a035e768280952d1a F src/date.c 57a7f9ba9f6b4d5268f5e411739066a611f99036 F src/delete.c fae81cc2eb14b75267d4f47d3cfc9ae02aae726f F src/expr.c 0391a657df4959eaf2a2fd7d77de5ebe750686ee @@ -1207,8 +1207,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 637246165a14c4808b90d0437e4d43fa5fac659e 1c220b806d56e163842e17038c3331f71861bd9c -R 78cdca5c720e64c06c4c8ef41bcc31b7 -T +closed 1c220b806d56e163842e17038c3331f71861bd9c +P ffb9d8144bbc35bf3d929e0e13a663668fff0558 +R 517f6fa1cedd5526f59aa045554cf442 U drh -Z 1739c9dd8a9fd2ca84e2afadfbbcea12 +Z d3a8b97f7ef904d82264d3ec79cf2202 diff --git a/manifest.uuid b/manifest.uuid index 2f7fda9cba..70a2bdb89a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ffb9d8144bbc35bf3d929e0e13a663668fff0558 \ No newline at end of file +9646a136e69cf2583965dfc9fac5f056af4cdb62 \ No newline at end of file diff --git a/src/ctime.c b/src/ctime.c index 36d0c266ee..59dc972d8d 100644 --- a/src/ctime.c +++ b/src/ctime.c @@ -392,7 +392,7 @@ static const char * const azCompileOpt[] = { int sqlite3_compileoption_used(const char *zOptName){ int i, n; -#ifdef SQLITE_ENABLE_API_ARMORE +#ifdef SQLITE_ENABLE_API_ARMOR if( zOptName==0 ){ (void)SQLITE_MISUSE_BKPT; return 0; From a95d8ca1fad1133a39ab65c60e7b7346f9089749 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Mon, 27 Oct 2014 19:42:02 +0000 Subject: [PATCH 029/133] Fix harmless compiler warning in an assert statement. FossilOrigin-Name: d33a1ff3aad0bfabf70a98ac338a68f82074e4fe --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/btree.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 3bfe9532df..c24ca49842 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\stypo\sin\sthe\smacro\sname\sof\san\s#ifdef -D 2014-10-27T19:39:51.561 +C Fix\sharmless\scompiler\swarning\sin\san\sassert\sstatement. +D 2014-10-27T19:42:02.875 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 7f841396adfd47507ff670a471162d2bfcda3136 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 812c03daa8be68daf623dd0349ecb18e9f988967 +F src/btree.c 0e3262c36b69bed3c5f9260313286a2e8c060ca5 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 67bb05b1077e0cdaccb2e36bfcbe7a5df9ed31e8 @@ -1207,7 +1207,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ffb9d8144bbc35bf3d929e0e13a663668fff0558 -R 517f6fa1cedd5526f59aa045554cf442 -U drh -Z d3a8b97f7ef904d82264d3ec79cf2202 +P 9646a136e69cf2583965dfc9fac5f056af4cdb62 +R c8728e917d07299237654ca84312b584 +U mistachkin +Z 269b8116d90b241ccd2319871bd381eb diff --git a/manifest.uuid b/manifest.uuid index 70a2bdb89a..3bcc889db6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9646a136e69cf2583965dfc9fac5f056af4cdb62 \ No newline at end of file +d33a1ff3aad0bfabf70a98ac338a68f82074e4fe \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 06dfd2cd53..39d1e81a3f 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1308,7 +1308,7 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ assert( nByte>=0 ); /* Minimum cell size is 4 */ assert( pPage->nFree>=nByte ); assert( pPage->nOverflow==0 ); - assert( nByte < pPage->pBt->usableSize-8 ); + assert( nByte < (int)(pPage->pBt->usableSize-8) ); assert( pPage->cellOffset == hdr + 12 - 4*pPage->leaf ); gap = pPage->cellOffset + 2*pPage->nCell; From df9c093e2c516c011ec476cf220d477eea70983c Mon Sep 17 00:00:00 2001 From: mistachkin Date: Mon, 27 Oct 2014 19:58:29 +0000 Subject: [PATCH 030/133] Fix compilation issue with MSVC due to a misplaced variable declaration. FossilOrigin-Name: 9588b345d09daaa49d24d7fb6cab732e64e5474e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/random.c | 16 +++++++++++----- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index c24ca49842..4a6bd095ba 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarning\sin\san\sassert\sstatement. -D 2014-10-27T19:42:02.875 +C Fix\scompilation\sissue\swith\sMSVC\sdue\sto\sa\smisplaced\svariable\sdeclaration. +D 2014-10-27T19:58:29.156 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -224,7 +224,7 @@ F src/pcache1.c e412cb585f777c840ddce0500eddc5c6043c2bb5 F src/pragma.c 3f3e959390a10c0131676f0e307acce372777e0f F src/prepare.c b7b7bf020bd4c962f7c8aed5a3c542c7dfe9f9c7 F src/printf.c c31012ac23e458081df4a32634b60424e0cdfaf3 -F src/random.c b8a058131851de1a37801b5587845ee73411c064 +F src/random.c f88232b90e308f58f1f8f10894d7a8a750d6f64d F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b @@ -1207,7 +1207,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 9646a136e69cf2583965dfc9fac5f056af4cdb62 -R c8728e917d07299237654ca84312b584 +P d33a1ff3aad0bfabf70a98ac338a68f82074e4fe +R ee98bc34f841b6720302924cb59e4bc2 U mistachkin -Z 269b8116d90b241ccd2319871bd381eb +Z 8119f44960013a2ca52459062580ce07 diff --git a/manifest.uuid b/manifest.uuid index 3bcc889db6..f68a004aea 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d33a1ff3aad0bfabf70a98ac338a68f82074e4fe \ No newline at end of file +9588b345d09daaa49d24d7fb6cab732e64e5474e \ No newline at end of file diff --git a/src/random.c b/src/random.c index c4241362de..bd109e71b3 100644 --- a/src/random.c +++ b/src/random.c @@ -34,10 +34,6 @@ void sqlite3_randomness(int N, void *pBuf){ unsigned char t; unsigned char *zBuf = pBuf; -#ifndef SQLITE_OMIT_AUTOINIT - if( sqlite3_initialize() ) return; -#endif - /* The "wsdPrng" macro will resolve to the pseudo-random number generator ** state vector. If writable static data is unsupported on the target, ** we have to locate the state vector at run-time. In the more common @@ -52,13 +48,23 @@ void sqlite3_randomness(int N, void *pBuf){ #endif #if SQLITE_THREADSAFE - sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_PRNG); + sqlite3_mutex *mutex; +#endif + +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return; +#endif + +#if SQLITE_THREADSAFE + mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_PRNG); sqlite3_mutex_enter(mutex); #endif if( N<=0 || pBuf==0 ){ wsdPrng.isInit = 0; +#if SQLITE_THREADSAFE sqlite3_mutex_leave(mutex); +#endif return; } From d61a18a9f1493ed6ee2b6d12d7b4ca4621cc894b Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 27 Oct 2014 20:14:02 +0000 Subject: [PATCH 031/133] Remove a small amount of unnecessary #ifdeffery from random.c. FossilOrigin-Name: 2b9340c8684bc382391e02813e960b3166f24daa --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/random.c | 4 +--- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 4a6bd095ba..31157b57d0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\scompilation\sissue\swith\sMSVC\sdue\sto\sa\smisplaced\svariable\sdeclaration. -D 2014-10-27T19:58:29.156 +C Remove\sa\ssmall\samount\sof\sunnecessary\s#ifdeffery\sfrom\srandom.c. +D 2014-10-27T20:14:02.021 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -224,7 +224,7 @@ F src/pcache1.c e412cb585f777c840ddce0500eddc5c6043c2bb5 F src/pragma.c 3f3e959390a10c0131676f0e307acce372777e0f F src/prepare.c b7b7bf020bd4c962f7c8aed5a3c542c7dfe9f9c7 F src/printf.c c31012ac23e458081df4a32634b60424e0cdfaf3 -F src/random.c f88232b90e308f58f1f8f10894d7a8a750d6f64d +F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b @@ -1207,7 +1207,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P d33a1ff3aad0bfabf70a98ac338a68f82074e4fe -R ee98bc34f841b6720302924cb59e4bc2 -U mistachkin -Z 8119f44960013a2ca52459062580ce07 +P 9588b345d09daaa49d24d7fb6cab732e64e5474e +R e26f0edef3def3003fae07f43e971e87 +U drh +Z 3e6e6f10ca8455ff968ead5f51a9fff2 diff --git a/manifest.uuid b/manifest.uuid index f68a004aea..11afc870ef 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9588b345d09daaa49d24d7fb6cab732e64e5474e \ No newline at end of file +2b9340c8684bc382391e02813e960b3166f24daa \ No newline at end of file diff --git a/src/random.c b/src/random.c index bd109e71b3..179d01bef2 100644 --- a/src/random.c +++ b/src/random.c @@ -57,14 +57,12 @@ void sqlite3_randomness(int N, void *pBuf){ #if SQLITE_THREADSAFE mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_PRNG); - sqlite3_mutex_enter(mutex); #endif + sqlite3_mutex_enter(mutex); if( N<=0 || pBuf==0 ){ wsdPrng.isInit = 0; -#if SQLITE_THREADSAFE sqlite3_mutex_leave(mutex); -#endif return; } From 2d8ad51c5b53644aa40ab276a7c7342da920d465 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Mon, 27 Oct 2014 22:06:21 +0000 Subject: [PATCH 032/133] Add special handling for static mutexes in sqlite3_mutex_alloc() when automatic calls to sqlite3_initialize() are enabled. FossilOrigin-Name: 7857d27caa845e5629d94c2e66587dc89016daca --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/mutex.c | 1 + 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 31157b57d0..ed6b523e7c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sa\ssmall\samount\sof\sunnecessary\s#ifdeffery\sfrom\srandom.c. -D 2014-10-27T20:14:02.021 +C Add\sspecial\shandling\sfor\sstatic\smutexes\sin\ssqlite3_mutex_alloc()\swhen\sautomatic\scalls\sto\ssqlite3_initialize()\sare\senabled. +D 2014-10-27T22:06:21.571 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -202,7 +202,7 @@ F src/mem2.c f1940d9e91948dd6a908fbb9ce3835c36b5d83c3 F src/mem3.c 61c9d47b792908c532ca3a62b999cf21795c6534 F src/mem5.c 61eeb90134f9a5be6c2e68d8daae7628b25953fb F src/memjournal.c 3eb2c0b51adbd869cb6a44780323f05fa904dc85 -F src/mutex.c 84a073c9a23a8d7bdd2ea832522d1730df18812c +F src/mutex.c 19bf9acba69ca2f367c3761080f8a9f0cf4670a8 F src/mutex.h 779d588e3b7756ec3ecf7d78cde1d84aba414f85 F src/mutex_noop.c f3f09fd7a2eb4287cfc799753ffc30380e7b71a1 F src/mutex_unix.c 551e2f25f0fa0ee8fd7a43f50fc3d8be00e95dde @@ -1207,7 +1207,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 9588b345d09daaa49d24d7fb6cab732e64e5474e -R e26f0edef3def3003fae07f43e971e87 -U drh -Z 3e6e6f10ca8455ff968ead5f51a9fff2 +P 2b9340c8684bc382391e02813e960b3166f24daa +R f7a6b10545dea42b1d622484e6364982 +U mistachkin +Z 6b079baac5628ec298fadd129a245092 diff --git a/manifest.uuid b/manifest.uuid index 11afc870ef..7088378432 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2b9340c8684bc382391e02813e960b3166f24daa \ No newline at end of file +7857d27caa845e5629d94c2e66587dc89016daca \ No newline at end of file diff --git a/src/mutex.c b/src/mutex.c index bad5a7c113..2b45036289 100644 --- a/src/mutex.c +++ b/src/mutex.c @@ -82,6 +82,7 @@ int sqlite3MutexEnd(void){ sqlite3_mutex *sqlite3_mutex_alloc(int id){ #ifndef SQLITE_OMIT_AUTOINIT if( id<=SQLITE_MUTEX_RECURSIVE && sqlite3_initialize() ) return 0; + if( id>SQLITE_MUTEX_RECURSIVE && sqlite3MutexInit() ) return 0; #endif return sqlite3GlobalConfig.mutex.xMutexAlloc(id); } From eefaf448db0ed620f87af06a49c1bc5fbfdb9282 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 28 Oct 2014 00:56:18 +0000 Subject: [PATCH 033/133] Correct the documentation on the maximum size of a scratch allocation. FossilOrigin-Name: 30f86eb3f9ac88f83ed9e23ea6cd1fccf68e0812 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/btree.c | 1 + src/sqlite.h.in | 12 ++++++++---- 4 files changed, 18 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index ed6b523e7c..2a931c6cb3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sspecial\shandling\sfor\sstatic\smutexes\sin\ssqlite3_mutex_alloc()\swhen\sautomatic\scalls\sto\ssqlite3_initialize()\sare\senabled. -D 2014-10-27T22:06:21.571 +C Correct\sthe\sdocumentation\son\sthe\smaximum\ssize\sof\sa\sscratch\sallocation. +D 2014-10-28T00:56:18.775 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 7f841396adfd47507ff670a471162d2bfcda3136 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 0e3262c36b69bed3c5f9260313286a2e8c060ca5 +F src/btree.c 5189881ca403938c5ceddde496b984fef9f40c5a F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 67bb05b1077e0cdaccb2e36bfcbe7a5df9ed31e8 @@ -229,7 +229,7 @@ F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c 282f8f5278e0c78eb442217531172ec9e1538796 -F src/sqlite.h.in 4a5e5158c189d2bcd45c7c4607c2c0eb6d25c153 +F src/sqlite.h.in a9f2e5a0e2472c8c7819f3a16074c14b9376608f F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 90519c3b3e8ee90adfce013234c4bd07275d77b5 @@ -1207,7 +1207,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 2b9340c8684bc382391e02813e960b3166f24daa -R f7a6b10545dea42b1d622484e6364982 -U mistachkin -Z 6b079baac5628ec298fadd129a245092 +P 7857d27caa845e5629d94c2e66587dc89016daca +R cc3ad0fbd146620828bc67546f9cb55d +U drh +Z 46e16e5efd8acf1525fc0288e922960d diff --git a/manifest.uuid b/manifest.uuid index 7088378432..bc344729d5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7857d27caa845e5629d94c2e66587dc89016daca \ No newline at end of file +30f86eb3f9ac88f83ed9e23ea6cd1fccf68e0812 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 39d1e81a3f..22b168d9e4 100644 --- a/src/btree.c +++ b/src/btree.c @@ -6649,6 +6649,7 @@ static int balance_nonroot( nMaxCells*sizeof(u8*) /* apCell */ + nMaxCells*sizeof(u16) /* szCell */ + pBt->pageSize; /* aSpace1 */ + assert( szScratch<=16896 || szScratch<=6*pBt->pageSize ); apCell = sqlite3ScratchMalloc( szScratch ); if( apCell==0 ){ rc = SQLITE_NOMEM; diff --git a/src/sqlite.h.in b/src/sqlite.h.in index f1d4e406e8..8cfe61ee1a 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -1539,10 +1539,14 @@ struct sqlite3_mem_methods { ** argument must be a multiple of 16. ** The first argument must be a pointer to an 8-byte aligned buffer ** of at least sz*N bytes of memory. -** ^SQLite will use no more than two scratch buffers per thread. So -** N should be set to twice the expected maximum number of threads. -** ^SQLite will never require a scratch buffer that is more than 6 -** times the database page size. ^If SQLite needs needs additional +** ^SQLite will not use more than two scratch buffers per thread and not +** more than one scratch buffer per thread when not performing +** a [checkpoint] in [WAL mode]. +** ^SQLite will never request a scratch buffer that is more than 6 +** times the database page size, except when performing a [checkpoint] +** in [WAL mode] when the scratch buffer request size is a small fraction +** of the size of the WAL file. +** ^If SQLite needs needs additional ** scratch memory beyond what is provided by this configuration option, then ** [sqlite3_malloc()] will be used to obtain the memory needed. ** From 916cd23b2ca6aa4716b7b407abea6d870ea4585c Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 28 Oct 2014 12:35:28 +0000 Subject: [PATCH 034/133] Bump the version number to 3.8.8 FossilOrigin-Name: 1a7e711ed10860c7985e84b97cdfff748d940b9f --- VERSION | 2 +- configure | 18 +++++++++--------- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/VERSION b/VERSION index 4351a7e3a3..d7d8e42315 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.8.7 +3.8.8 diff --git a/configure b/configure index 9b8266d812..c0f751db62 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.62 for sqlite 3.8.7. +# Generated by GNU Autoconf 2.62 for sqlite 3.8.8. # # Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, # 2002, 2003, 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc. @@ -743,8 +743,8 @@ SHELL=${CONFIG_SHELL-/bin/sh} # Identity of this package. PACKAGE_NAME='sqlite' PACKAGE_TARNAME='sqlite' -PACKAGE_VERSION='3.8.7' -PACKAGE_STRING='sqlite 3.8.7' +PACKAGE_VERSION='3.8.8' +PACKAGE_STRING='sqlite 3.8.8' PACKAGE_BUGREPORT='' # Factoring default headers for most tests. @@ -1483,7 +1483,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures sqlite 3.8.7 to adapt to many kinds of systems. +\`configure' configures sqlite 3.8.8 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1548,7 +1548,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of sqlite 3.8.7:";; + short | recursive ) echo "Configuration of sqlite 3.8.8:";; esac cat <<\_ACEOF @@ -1664,7 +1664,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -sqlite configure 3.8.7 +sqlite configure 3.8.8 generated by GNU Autoconf 2.62 Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, @@ -1678,7 +1678,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by sqlite $as_me 3.8.7, which was +It was created by sqlite $as_me 3.8.8, which was generated by GNU Autoconf 2.62. Invocation command line was $ $0 $@ @@ -14021,7 +14021,7 @@ exec 6>&1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by sqlite $as_me 3.8.7, which was +This file was extended by sqlite $as_me 3.8.8, which was generated by GNU Autoconf 2.62. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -14074,7 +14074,7 @@ Report bugs to ." _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_version="\\ -sqlite config.status 3.8.7 +sqlite config.status 3.8.8 configured by $0, generated by GNU Autoconf 2.62, with options \\"`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`\\" diff --git a/manifest b/manifest index 2a931c6cb3..27680c927a 100644 --- a/manifest +++ b/manifest @@ -1,12 +1,12 @@ -C Correct\sthe\sdocumentation\son\sthe\smaximum\ssize\sof\sa\sscratch\sallocation. -D 2014-10-28T00:56:18.775 +C Bump\sthe\sversion\snumber\sto\s3.8.8 +D 2014-10-28T12:35:28.069 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 F Makefile.msc e31dee24038965fb6269d6d61073fd6b7e331dec F Makefile.vxworks 034289efa9d591b04b1a73598623119c306cbba0 F README.md 64f270c43c38c46de749e419c22f0ae2f4499fe8 -F VERSION 53a0b870e7f16d3b06623c31d233a304c163a6af +F VERSION d846487aff892625eb8e75960234e7285f0462fe F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 F addopcodes.awk 9eb448a552d5c0185cf62c463f9c173cedae3811 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2 @@ -38,7 +38,7 @@ F autoconf/tea/win/rules.vc c511f222b80064096b705dbeb97060ee1d6b6d63 F config.guess 226d9a188c6196f3033ffc651cbc9dcee1a42977 F config.h.in 0921066a13130082764ab4ab6456f7b5bebe56de F config.sub 9ebe4c3b3dab6431ece34f16828b594fb420da55 -F configure ad59a5f48b3c59a92b5506040a22fbe3f733a9d8 x +F configure 4343c810cc772571210af75d1a8f7c2eb711d75a x F configure.ac 4cf9f60785143fa141b10962ccc885d973792e9a F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad F doc/lemon.html 334dbf6621b8fb8790297ec1abf3cfa4621709d1 @@ -1207,7 +1207,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 7857d27caa845e5629d94c2e66587dc89016daca -R cc3ad0fbd146620828bc67546f9cb55d +P 30f86eb3f9ac88f83ed9e23ea6cd1fccf68e0812 +R 241b510f33643f46040372d1284a7d7b U drh -Z 46e16e5efd8acf1525fc0288e922960d +Z b060cb081e36e4268c2624adfeebfbc8 diff --git a/manifest.uuid b/manifest.uuid index bc344729d5..4cdb1c73e1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -30f86eb3f9ac88f83ed9e23ea6cd1fccf68e0812 \ No newline at end of file +1a7e711ed10860c7985e84b97cdfff748d940b9f \ No newline at end of file From c3ef4fa88a370dc6835d6dc4c3875d9577ae34f5 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 28 Oct 2014 15:58:50 +0000 Subject: [PATCH 035/133] Trivial simplification to the automatic index logic. FossilOrigin-Name: 23073a053931de324323f631a6613086786af411 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 3 +-- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 27680c927a..4d085dbcbc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Bump\sthe\sversion\snumber\sto\s3.8.8 -D 2014-10-28T12:35:28.069 +C Trivial\ssimplification\sto\sthe\sautomatic\sindex\slogic. +D 2014-10-28T15:58:50.607 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -302,7 +302,7 @@ F src/vtab.c 2a30791bbd7926b589401bd09c3abb33de563793 F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 4ce8c4826b7f86d080f0ed4e7a9045bb5014be77 +F src/where.c 5665df88cbd2b38eb72b4b94c8892c8afb360181 F src/whereInt.h 19279cd0664ce1d90b9ad3ef0108cb494acfe455 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1207,7 +1207,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 30f86eb3f9ac88f83ed9e23ea6cd1fccf68e0812 -R 241b510f33643f46040372d1284a7d7b +P 1a7e711ed10860c7985e84b97cdfff748d940b9f +R b0011529707271df74d0fa481aad6091 U drh -Z b060cb081e36e4268c2624adfeebfbc8 +Z ab603a6740e2096395006a3b0e96e405 diff --git a/manifest.uuid b/manifest.uuid index 4cdb1c73e1..39c8a652a5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1a7e711ed10860c7985e84b97cdfff748d940b9f \ No newline at end of file +23073a053931de324323f631a6613086786af411 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 358d508260..feccf2d11d 100644 --- a/src/where.c +++ b/src/where.c @@ -1651,7 +1651,7 @@ static void constructAutomaticIndex( ** if they go out of sync. */ extraCols = pSrc->colUsed & (~idxCols | MASKBIT(BMS-1)); - mxBitCol = (pTable->nCol >= BMS-1) ? BMS-1 : pTable->nCol; + mxBitCol = MIN(BMS-1,pTable->nCol); testcase( pTable->nCol==BMS-1 ); testcase( pTable->nCol==BMS-2 ); for(i=0; icolUsed & MASKBIT(BMS-1) ){ nKeyCol += pTable->nCol - BMS + 1; } - pLoop->wsFlags |= WHERE_COLUMN_EQ | WHERE_IDX_ONLY; /* Construct the Index object to describe this index */ pIdx = sqlite3AllocateIndexObject(pParse->db, nKeyCol+1, 0, &zNotUsed); From 635e57fc4893060c6c31efa3bcd936a9720212c7 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 28 Oct 2014 16:19:18 +0000 Subject: [PATCH 036/133] Fix a faulty assert() in the DELETE code generator. FossilOrigin-Name: 95f8ebdbf87326f23cd38e561ac5632b5367a449 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/delete.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 4d085dbcbc..a6054c1a60 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Trivial\ssimplification\sto\sthe\sautomatic\sindex\slogic. -D 2014-10-28T15:58:50.607 +C Fix\sa\sfaulty\sassert()\sin\sthe\sDELETE\scode\sgenerator. +D 2014-10-28T16:19:18.944 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -180,7 +180,7 @@ F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 F src/complete.c c4ba6e0626bb94bc77a0861735f3382fcf7cc818 F src/ctime.c df19848891c8a553c80e6f5a035e768280952d1a F src/date.c 57a7f9ba9f6b4d5268f5e411739066a611f99036 -F src/delete.c fae81cc2eb14b75267d4f47d3cfc9ae02aae726f +F src/delete.c 0750b1eb4d96cd3fb2c798599a3a7c85e92f1417 F src/expr.c 0391a657df4959eaf2a2fd7d77de5ebe750686ee F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c da985ae673efef2c712caef825a5d2edb087ead7 @@ -1207,7 +1207,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 1a7e711ed10860c7985e84b97cdfff748d940b9f -R b0011529707271df74d0fa481aad6091 +P 23073a053931de324323f631a6613086786af411 +R 5f4e1ea821e16c5044076b23f1aebe8a U drh -Z ab603a6740e2096395006a3b0e96e405 +Z 478016a63ef8d014d3b6d95929bf74df diff --git a/manifest.uuid b/manifest.uuid index 39c8a652a5..49ec9c5970 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -23073a053931de324323f631a6613086786af411 \ No newline at end of file +95f8ebdbf87326f23cd38e561ac5632b5367a449 \ No newline at end of file diff --git a/src/delete.c b/src/delete.c index b97407400b..d81dd3f6b4 100644 --- a/src/delete.c +++ b/src/delete.c @@ -481,7 +481,7 @@ void sqlite3DeleteFrom( assert( nKey==nPk ); /* OP_Found will use an unpacked key */ assert( !IsVirtual(pTab) ); if( aToOpen[iDataCur-iTabCur] ){ - assert( pPk!=0 ); + assert( pPk!=0 || pTab->pSelect!=0 ); sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, addrBypass, iKey, nKey); VdbeCoverage(v); } From dd8c460081ef024abbb925029a18adb5deea3522 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 28 Oct 2014 16:50:10 +0000 Subject: [PATCH 037/133] =?UTF-8?q?Fix=20a=20crash=20that=20could=20occur?= =?UTF-8?q?=20if=20the=20WHERE=20clause=20of=20an=20UPDATE=20statement=20o?= =?UTF-8?q?n=20a=20view=20that=20does=20not=20feature=20a=20column=20named?= =?UTF-8?q?=20"rowid"=20contains=20a=20term=20such=20as=20"rowid=3D=3F".?= FossilOrigin-Name: 8523670d50004f3112b7871f11c8b8b02aab96ab --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/update.c | 4 ++-- test/trigger9.test | 33 +++++++++++++++++++++++++++++++++ 4 files changed, 44 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index a6054c1a60..8f4042bf5b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sfaulty\sassert()\sin\sthe\sDELETE\scode\sgenerator. -D 2014-10-28T16:19:18.944 +C Fix\sa\scrash\sthat\scould\soccur\sif\sthe\sWHERE\sclause\sof\san\sUPDATE\sstatement\son\sa\sview\sthat\sdoes\snot\sfeature\sa\scolumn\snamed\s"rowid"\scontains\sa\sterm\ssuch\sas\s"rowid=?". +D 2014-10-28T16:50:10.527 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -285,7 +285,7 @@ F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 6de09362b657f19ba83e5fa521ee715787ce9fee F src/tokenize.c cc9016e5007fc5e76789079616d2f26741bcc689 F src/trigger.c 25571661fdeae8c7f975ff40ffec205520a3f92f -F src/update.c 729f6f18fc27740591d085e1172cebe311144bf0 +F src/update.c 3c4ecc282accf12d39edb8d524cf089645e55a13 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 3b627daa45c7308c1e36e3dbaa3f9ce7e5c7fa73 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a @@ -1051,7 +1051,7 @@ F test/trigger5.test 619391a3e9fc194081d22cefd830d811e7badf83 F test/trigger6.test 0e411654f122552da6590f0b4e6f781048a4a9b9 F test/trigger7.test b39e6dee1debe0ff9c2ef66326668f149f07c9c4 F test/trigger8.test 30cb0530bd7c4728055420e3f739aa00412eafa4 -F test/trigger9.test 5b0789f1c5c4600961f8e68511b825b87be53e31 +F test/trigger9.test 2226ec795a33b0460ab5cf8891e9054cc7edef41 F test/triggerA.test fe5597f47ee21bacb4936dc827994ed94161e332 F test/triggerB.test 56780c031b454abac2340dbb3b71ac5c56c3d7fe F test/triggerC.test a68980c5955d62ee24be6f97129d824f199f9a4c @@ -1207,7 +1207,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 23073a053931de324323f631a6613086786af411 -R 5f4e1ea821e16c5044076b23f1aebe8a -U drh -Z 478016a63ef8d014d3b6d95929bf74df +P 95f8ebdbf87326f23cd38e561ac5632b5367a449 +R c4fade85ca4f95e680190bc9e417c1ff +U dan +Z 12d8cc683b13ea652b467efe210b5438 diff --git a/manifest.uuid b/manifest.uuid index 49ec9c5970..110e17e040 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -95f8ebdbf87326f23cd38e561ac5632b5367a449 \ No newline at end of file +8523670d50004f3112b7871f11c8b8b02aab96ab \ No newline at end of file diff --git a/src/update.c b/src/update.c index f781a60ccd..3af4017f1b 100644 --- a/src/update.c +++ b/src/update.c @@ -431,8 +431,8 @@ void sqlite3Update( /* Top of the update loop */ if( okOnePass ){ - if( aToOpen[iDataCur-iBaseCur] ){ - assert( pPk!=0 ); + if( aToOpen[iDataCur-iBaseCur] && !isView ){ + assert( pPk ); sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, labelBreak, regKey, nKey); VdbeCoverageNeverTaken(v); } diff --git a/test/trigger9.test b/test/trigger9.test index f56c8acbc5..326fa63d4c 100644 --- a/test/trigger9.test +++ b/test/trigger9.test @@ -32,6 +32,7 @@ ifcapable {!trigger} { finish_test return } +set ::testprefix trigger9 proc has_rowdata {sql} { expr {[lsearch [execsql "explain $sql"] RowData]>=0} @@ -220,4 +221,36 @@ ifcapable compound { } {2} } +reset_db +do_execsql_test 4.1 { + CREATE TABLE t1(a, b); + CREATE TABLE log(x); + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(3, 4); + CREATE VIEW v1 AS SELECT a, b FROM t1; + + CREATE TRIGGER tr1 INSTEAD OF DELETE ON v1 BEGIN + INSERT INTO log VALUES('delete'); + END; + + CREATE TRIGGER tr2 INSTEAD OF UPDATE ON v1 BEGIN + INSERT INTO log VALUES('update'); + END; + + CREATE TRIGGER tr3 INSTEAD OF INSERT ON v1 BEGIN + INSERT INTO log VALUES('insert'); + END; +} + +do_execsql_test 4.2 { + DELETE FROM v1 WHERE rowid=1; +} {} + +do_execsql_test 4.3 { + UPDATE v1 SET a=b WHERE rowid=2; +} {} + + + + finish_test From c3da667b25cd513f3552652e783990734a200992 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 28 Oct 2014 18:24:16 +0000 Subject: [PATCH 038/133] Modify the documentation for sqlite3_changes() to make it more testable. Add tests and minor fixes for the same. FossilOrigin-Name: 41cdd0c422d61533a94870cb5ad094682956d472 --- manifest | 19 +- manifest.uuid | 2 +- src/sqlite.h.in | 78 ++++---- src/vdbe.c | 1 + src/vdbeInt.h | 3 +- src/vdbeaux.c | 6 + test/e_changes.test | 441 ++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 499 insertions(+), 51 deletions(-) create mode 100644 test/e_changes.test diff --git a/manifest b/manifest index 8f4042bf5b..380d7ccd48 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\scrash\sthat\scould\soccur\sif\sthe\sWHERE\sclause\sof\san\sUPDATE\sstatement\son\sa\sview\sthat\sdoes\snot\sfeature\sa\scolumn\snamed\s"rowid"\scontains\sa\sterm\ssuch\sas\s"rowid=?". -D 2014-10-28T16:50:10.527 +C Modify\sthe\sdocumentation\sfor\ssqlite3_changes()\sto\smake\sit\smore\stestable.\sAdd\stests\sand\sminor\sfixes\sfor\sthe\ssame. +D 2014-10-28T18:24:16.387 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -229,7 +229,7 @@ F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c 282f8f5278e0c78eb442217531172ec9e1538796 -F src/sqlite.h.in a9f2e5a0e2472c8c7819f3a16074c14b9376608f +F src/sqlite.h.in 1c5624f8b21cc8261a8b048033815581347b375f F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 90519c3b3e8ee90adfce013234c4bd07275d77b5 @@ -289,11 +289,11 @@ F src/update.c 3c4ecc282accf12d39edb8d524cf089645e55a13 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 3b627daa45c7308c1e36e3dbaa3f9ce7e5c7fa73 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c 5ee15a66ce07e0482b92aa29e4dd0c5827a22d79 +F src/vdbe.c 1b7e8ccaca2a23ae2804568f34b7c645adfd332d F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 -F src/vdbeInt.h e2a060a55ee18a6ab973353a5e2ec7ee569bf787 +F src/vdbeInt.h acc36ac461f973f46ac7942f86abdd93d2f8cfbc F src/vdbeapi.c 02d8afcff710eb35e3d9e49cb677308296b00009 -F src/vdbeaux.c edbb7a9c8b2a8f7a68ac75c2475edd4040266b76 +F src/vdbeaux.c 3d6b2b412ef2193aa4729922dfc5df1efadbf6df F src/vdbeblob.c 8b5442ff0954c44b45cbabbe2e94091a2e16fdef F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f F src/vdbesort.c 975aeffa99acb0991b2f288d30294756bff41438 @@ -451,6 +451,7 @@ F test/descidx3.test 09ddbe3f5295f482d2f8b687cf6db8bad7acd9a2 F test/diskfull.test 106391384780753ea6896b7b4f005d10e9866b6e F test/distinct.test 086e70c765f172e8974e9f83b9ac5ca03c154e77 F test/distinctagg.test 1a6ef9c87a58669438fc771450d7a72577417376 +F test/e_changes.test fd66105385153dbf21fdb35eb8ef6c3e1eade579 F test/e_createtable.test c7e67b49e6cf92473c8fb30ab26143e9e2128cf7 F test/e_delete.test d5186e2f5478b659f16a2c8b66c09892823e542a F test/e_droptrigger.test 3cd080807622c13e5bbb61fc9a57bd7754da2412 @@ -1207,7 +1208,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 95f8ebdbf87326f23cd38e561ac5632b5367a449 -R c4fade85ca4f95e680190bc9e417c1ff +P 8523670d50004f3112b7871f11c8b8b02aab96ab +R 8192ca2d13d614d67e4c391f6685b170 U dan -Z 12d8cc683b13ea652b467efe210b5438 +Z 02a949e02b7399a00fe453126f032ba0 diff --git a/manifest.uuid b/manifest.uuid index 110e17e040..1803fe3792 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8523670d50004f3112b7871f11c8b8b02aab96ab \ No newline at end of file +41cdd0c422d61533a94870cb5ad094682956d472 \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 8cfe61ee1a..bb5d90254d 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -1874,47 +1874,45 @@ sqlite3_int64 sqlite3_last_insert_rowid(sqlite3*); /* ** CAPI3REF: Count The Number Of Rows Modified ** -** ^This function returns the number of database rows that were changed -** or inserted or deleted by the most recently completed SQL statement -** on the [database connection] specified by the first parameter. -** ^(Only changes that are directly specified by the [INSERT], [UPDATE], -** or [DELETE] statement are counted. Auxiliary changes caused by -** triggers or [foreign key actions] are not counted.)^ Use the -** [sqlite3_total_changes()] function to find the total number of changes -** including changes caused by triggers and foreign key actions. +** ^This function returns the number of rows modified, inserted or +** deleted by the most recently completed INSERT, UPDATE or DELETE +** statement on the database connection specified by the only parameter. +** ^Executing any other type of SQL statement does not modify the value +** returned by this function. ** -** ^Changes to a view that are simulated by an [INSTEAD OF trigger] -** are not counted. Only real table changes are counted. +** ^Only changes made directly by the INSERT, UPDATE or DELETE statement are +** considered - auxiliary changes caused by [CREATE TRIGGER | triggers], +** [foreign key actions] or [REPLACE] constraint resolution are not counted. +** +** Changes to a view that are intercepted by +** [INSTEAD OF trigger | INSTEAD OF triggers] are not counted. ^The value +** returned by sqlite3_changes() immediately after an INSERT, UPDATE or +** DELETE statement run on a view is always zero. Only changes made to real +** tables are counted. ** -** ^(A "row change" is a change to a single row of a single table -** caused by an INSERT, DELETE, or UPDATE statement. Rows that -** are changed as side effects of [REPLACE] constraint resolution, -** rollback, ABORT processing, [DROP TABLE], or by any other -** mechanisms do not count as direct row changes.)^ -** -** A "trigger context" is a scope of execution that begins and -** ends with the script of a [CREATE TRIGGER | trigger]. -** Most SQL statements are -** evaluated outside of any trigger. This is the "top level" -** trigger context. If a trigger fires from the top level, a -** new trigger context is entered for the duration of that one -** trigger. Subtriggers create subcontexts for their duration. -** -** ^Calling [sqlite3_exec()] or [sqlite3_step()] recursively does -** not create a new trigger context. -** -** ^This function returns the number of direct row changes in the -** most recent INSERT, UPDATE, or DELETE statement within the same -** trigger context. -** -** ^Thus, when called from the top level, this function returns the -** number of changes in the most recent INSERT, UPDATE, or DELETE -** that also occurred at the top level. ^(Within the body of a trigger, -** the sqlite3_changes() interface can be called to find the number of -** changes in the most recently completed INSERT, UPDATE, or DELETE -** statement within the body of the same trigger. -** However, the number returned does not include changes -** caused by subtriggers since those have their own context.)^ +** Things are more complicated if the sqlite3_changes() function is +** executed while a trigger program is running. This may happen if the +** program uses the [changes() SQL function], or if some other callback +** function invokes sqlite3_changes() directly. Essentially: +** +**
    +**
  • ^(Before entering a trigger program the value returned by +** sqlite3_changes() function is saved. After the trigger program +** has finished, the original value is restored.)^ +** +**
  • ^(Within a trigger program each INSERT, UPDATE and DELETE +** statement sets the value returned by sqlite3_changes() +** upon completion as normal. Of course, this value will not include +** any changes performed by sub-triggers, as the sqlite3_changes() +** value will be saved and restored after each sub-trigger has run.)^ +**
+** +** ^This means that if the changes() SQL function (or similar) is used +** by the first INSERT, UPDATE or DELETE statement within a trigger, it +** returns the value as set when the calling statement began executing. +** ^If it is used by the second or subsequent such statement within a trigger +** program, the value returned reflects the number of rows modified by the +** previous INSERT, UPDATE or DELETE statement within the same trigger. ** ** See also the [sqlite3_total_changes()] interface, the ** [count_changes pragma], and the [changes() SQL function]. @@ -1934,7 +1932,7 @@ int sqlite3_changes(sqlite3*); ** from all [CREATE TRIGGER | trigger] contexts and changes made by ** [foreign key actions]. However, ** the count does not include changes used to implement [REPLACE] constraints, -** do rollbacks or ABORT processing, or [DROP TABLE] processing. The +** rollbacks or [DROP TABLE] commands. The ** count does not include rows of views that fire an [INSTEAD OF trigger], ** though if the INSTEAD OF trigger makes changes of its own, those changes ** are counted.)^ diff --git a/src/vdbe.c b/src/vdbe.c index 0f9f45c456..d256c6b770 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -5423,6 +5423,7 @@ case OP_Program: { /* jump */ pFrame->pParent = p->pFrame; pFrame->lastRowid = lastRowid; pFrame->nChange = p->nChange; + pFrame->nDbChange = p->db->nChange; p->nChange = 0; p->pFrame = pFrame; p->aMem = aMem = &VdbeFrameMem(pFrame)[-1]; diff --git a/src/vdbeInt.h b/src/vdbeInt.h index bb504d64a1..623c5fdde8 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -144,7 +144,8 @@ struct VdbeFrame { int nOnceFlag; /* Number of entries in aOnceFlag */ int nChildMem; /* Number of memory cells for child frame */ int nChildCsr; /* Number of cursors for child frame */ - int nChange; /* Statement changes (Vdbe.nChanges) */ + int nChange; /* Statement changes (Vdbe.nChange) */ + int nDbChange; /* Value of db->nChange */ }; #define VdbeFrameMem(p) ((Mem *)&((u8 *)p)[ROUND8(sizeof(VdbeFrame))]) diff --git a/src/vdbeaux.c b/src/vdbeaux.c index c0018bb71c..7dfb64130d 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -1772,6 +1772,7 @@ int sqlite3VdbeFrameRestore(VdbeFrame *pFrame){ v->nCursor = pFrame->nCursor; v->db->lastRowid = pFrame->lastRowid; v->nChange = pFrame->nChange; + v->db->nChange = pFrame->nDbChange; return pFrame->pc; } @@ -2339,6 +2340,7 @@ int sqlite3VdbeHalt(Vdbe *p){ sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK); sqlite3CloseSavepoints(db); db->autoCommit = 1; + p->nChange = 0; } } } @@ -2379,6 +2381,7 @@ int sqlite3VdbeHalt(Vdbe *p){ }else if( rc!=SQLITE_OK ){ p->rc = rc; sqlite3RollbackAll(db, SQLITE_OK); + p->nChange = 0; }else{ db->nDeferredCons = 0; db->nDeferredImmCons = 0; @@ -2387,6 +2390,7 @@ int sqlite3VdbeHalt(Vdbe *p){ } }else{ sqlite3RollbackAll(db, SQLITE_OK); + p->nChange = 0; } db->nStatement = 0; }else if( eStatementOp==0 ){ @@ -2398,6 +2402,7 @@ int sqlite3VdbeHalt(Vdbe *p){ sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK); sqlite3CloseSavepoints(db); db->autoCommit = 1; + p->nChange = 0; } } @@ -2418,6 +2423,7 @@ int sqlite3VdbeHalt(Vdbe *p){ sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK); sqlite3CloseSavepoints(db); db->autoCommit = 1; + p->nChange = 0; } } diff --git a/test/e_changes.test b/test/e_changes.test new file mode 100644 index 0000000000..a77e22a2ee --- /dev/null +++ b/test/e_changes.test @@ -0,0 +1,441 @@ +# 2011 October 28 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix e_changes + +# Like [do_execsql_test], except it appends the value returned by +# [db changes] to the result of executing the SQL script. +# +proc do_changes_test {tn sql res} { + uplevel [list \ + do_test $tn "concat \[execsql {$sql}\] \[db changes\]" $res + ] +} + + +#-------------------------------------------------------------------------- +# EVIDENCE-OF: R-15996-49369 This function returns the number of rows +# modified, inserted or deleted by the most recently completed INSERT, +# UPDATE or DELETE statement on the database connection specified by the +# only parameter. +# +do_execsql_test 1.0 { + CREATE TABLE t1(a, b); + CREATE TABLE t2(x, y, PRIMARY KEY(x, y)) WITHOUT ROWID; + CREATE INDEX i1 ON t1(a); + CREATE INDEX i2 ON t2(y); +} +foreach {tn schema} { + 1 { + CREATE TABLE t1(a, b); + CREATE INDEX i1 ON t1(b); + } + 2 { + CREATE TABLE t1(a, b, PRIMARY KEY(a, b)) WITHOUT ROWID; + CREATE INDEX i1 ON t1(b); + } +} { + reset_db + execsql $schema + + # Insert 1 row. + do_changes_test 1.$tn.1 { INSERT INTO t1 VALUES(0, 0) } 1 + + # Insert 10 rows. + do_changes_test 1.$tn.2 { + WITH rows(i, j) AS ( + SELECT 1, 1 UNION ALL SELECT i+1, j+i FROM rows WHERE i<10 + ) + INSERT INTO t1 SELECT * FROM rows + } 10 + + # Modify 5 rows. + do_changes_test 1.$tn.3 { + UPDATE t1 SET b=b+1 WHERE a<5; + } 5 + + # Delete 4 rows + do_changes_test 1.$tn.4 { + DELETE FROM t1 WHERE a>6 + } 4 + + # Check the "on the database connecton specified" part of hte + # requirement - changes made by other connections do not show up in + # the return value of sqlite3_changes(). + do_test 1.$tn.5 { + sqlite3 db2 test.db + execsql { INSERT INTO t1 VALUES(-1, -1) } db2 + db2 changes + } 1 + do_test 1.$tn.6 { + db changes + } 4 + db2 close + + # Test that statements that modify no rows because they hit UNIQUE + # constraints set the sqlite3_changes() value to 0. Regardless of + # whether or not they are executed inside an explicit transaction. + # + # 1.$tn.8-9: outside of a transaction + # 1.$tn.10-12: inside a transaction + # + do_changes_test 1.$tn.7 { + CREATE UNIQUE INDEX i2 ON t1(a); + } 4 + do_catchsql_test 1.$tn.8 { + INSERT INTO t1 VALUES('a', 0), ('b', 0), ('c', 0), (0, 11); + } {1 {UNIQUE constraint failed: t1.a}} + do_test 1.$tn.9 { db changes } 0 + do_catchsql_test 1.$tn.10 { + BEGIN; + INSERT INTO t1 VALUES('a', 0), ('b', 0), ('c', 0), (0, 11); + } {1 {UNIQUE constraint failed: t1.a}} + do_test 1.$tn.11 { db changes } 0 + do_changes_test 1.$tn.12 COMMIT 0 + +} + + +#-------------------------------------------------------------------------- +# EVIDENCE-OF: R-44877-05564 Executing any other type of SQL statement +# does not modify the value returned by this function. +# +reset_db +do_changes_test 2.1 { CREATE TABLE t1(x) } 0 +do_changes_test 2.2 { + WITH d(y) AS (SELECT 1 UNION ALL SELECT y+1 FROM d WHERE y<47) + INSERT INTO t1 SELECT y FROM d; +} 47 + +# The statement above set changes() to 47. Check that none of the following +# modify this. +do_changes_test 2.3 { SELECT count(x) FROM t1 } {47 47} +do_changes_test 2.4 { DROP TABLE t1 } 47 +do_changes_test 2.5 { CREATE TABLE t1(x) } 47 +do_changes_test 2.6 { ALTER TABLE t1 ADD COLUMN b } 47 + + +#-------------------------------------------------------------------------- +# EVIDENCE-OF: R-53938-27527 Only changes made directly by the INSERT, +# UPDATE or DELETE statement are considered - auxiliary changes caused +# by triggers, foreign key actions or REPLACE constraint resolution are +# not counted. +# +# 3.1.*: triggers +# 3.2.*: foreign key actions +# 3.3.*: replace constraints +# +reset_db +do_execsql_test 3.1.0 { + CREATE TABLE log(x); + CREATE TABLE p1(one PRIMARY KEY, two); + + CREATE TRIGGER tr_ai AFTER INSERT ON p1 BEGIN + INSERT INTO log VALUES('insert'); + END; + CREATE TRIGGER tr_bd BEFORE DELETE ON p1 BEGIN + INSERT INTO log VALUES('delete'); + END; + CREATE TRIGGER tr_au AFTER UPDATE ON p1 BEGIN + INSERT INTO log VALUES('update'); + END; + +} + +do_changes_test 3.1.1 { + INSERT INTO p1 VALUES('a', 'A'), ('b', 'B'), ('c', 'C'); +} 3 +do_changes_test 3.1.2 { + UPDATE p1 SET two = two||two; +} 3 +do_changes_test 3.1.3 { + DELETE FROM p1 WHERE one IN ('a', 'c'); +} 2 +do_execsql_test 3.1.4 { + -- None of the inserts on table log were counted. + SELECT count(*) FROM log +} 8 + +do_execsql_test 3.2.0 { + DELETE FROM p1; + INSERT INTO p1 VALUES('a', 'A'), ('b', 'B'), ('c', 'C'); + + CREATE TABLE c1(a, b, FOREIGN KEY(a) REFERENCES p1 ON DELETE SET NULL); + CREATE TABLE c2(a, b, FOREIGN KEY(a) REFERENCES p1 ON DELETE SET DEFAULT); + CREATE TABLE c3(a, b, FOREIGN KEY(a) REFERENCES p1 ON DELETE CASCADE); + INSERT INTO c1 VALUES('a', 'aaa'); + INSERT INTO c2 VALUES('b', 'bbb'); + INSERT INTO c3 VALUES('c', 'ccc'); + + INSERT INTO p1 VALUES('d', 'D'), ('e', 'E'), ('f', 'F'); + CREATE TABLE c4(a, b, FOREIGN KEY(a) REFERENCES p1 ON UPDATE SET NULL); + CREATE TABLE c5(a, b, FOREIGN KEY(a) REFERENCES p1 ON UPDATE SET DEFAULT); + CREATE TABLE c6(a, b, FOREIGN KEY(a) REFERENCES p1 ON UPDATE CASCADE); + INSERT INTO c4 VALUES('d', 'aaa'); + INSERT INTO c5 VALUES('e', 'bbb'); + INSERT INTO c6 VALUES('f', 'ccc'); + + PRAGMA foreign_keys = ON; +} + +do_changes_test 3.2.1 { DELETE FROM p1 WHERE one = 'a' } 1 +do_changes_test 3.2.2 { DELETE FROM p1 WHERE one = 'b' } 1 +do_changes_test 3.2.3 { DELETE FROM p1 WHERE one = 'c' } 1 +do_execsql_test 3.2.4 { + SELECT * FROM c1; + SELECT * FROM c2; + SELECT * FROM c3; +} {{} aaa {} bbb} + +do_changes_test 3.2.5 { UPDATE p1 SET one = 'g' WHERE one = 'd' } 1 +do_changes_test 3.2.6 { UPDATE p1 SET one = 'h' WHERE one = 'e' } 1 +do_changes_test 3.2.7 { UPDATE p1 SET one = 'i' WHERE one = 'f' } 1 +do_execsql_test 3.2.8 { + SELECT * FROM c4; + SELECT * FROM c5; + SELECT * FROM c6; +} {{} aaa {} bbb i ccc} + +do_execsql_test 3.3.0 { + CREATE TABLE r1(a UNIQUE, b UNIQUE); + INSERT INTO r1 VALUES('i', 'i'); + INSERT INTO r1 VALUES('ii', 'ii'); + INSERT INTO r1 VALUES('iii', 'iii'); + INSERT INTO r1 VALUES('iv', 'iv'); + INSERT INTO r1 VALUES('v', 'v'); + INSERT INTO r1 VALUES('vi', 'vi'); + INSERT INTO r1 VALUES('vii', 'vii'); +} + +do_changes_test 3.3.1 { INSERT OR REPLACE INTO r1 VALUES('i', 1) } 1 +do_changes_test 3.3.2 { INSERT OR REPLACE INTO r1 VALUES('iv', 'v') } 1 +do_changes_test 3.3.3 { UPDATE OR REPLACE r1 SET b='v' WHERE a='iii' } 1 +do_changes_test 3.3.4 { UPDATE OR REPLACE r1 SET b='vi',a='vii' WHERE a='ii' } 1 +do_execsql_test 3.3.5 { + SELECT * FROM r1 ORDER BY a; +} {i 1 iii v vii vi} + + +#-------------------------------------------------------------------------- +# EVIDENCE-OF: R-09813-48563 The value returned by sqlite3_changes() +# immediately after an INSERT, UPDATE or DELETE statement run on a view +# is always zero. +# +reset_db +do_execsql_test 4.1 { + CREATE TABLE log(log); + CREATE TABLE t1(x, y); + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(3, 4); + INSERT INTO t1 VALUES(5, 6); + + CREATE VIEW v1 AS SELECT * FROM t1; + CREATE TRIGGER v1_i INSTEAD OF INSERT ON v1 BEGIN + INSERT INTO log VALUES('insert'); + END; + CREATE TRIGGER v1_u INSTEAD OF UPDATE ON v1 BEGIN + INSERT INTO log VALUES('update'), ('update'); + END; + CREATE TRIGGER v1_d INSTEAD OF DELETE ON v1 BEGIN + INSERT INTO log VALUES('delete'), ('delete'), ('delete'); + END; +} + +do_changes_test 4.2.1 { INSERT INTO t1 SELECT * FROM t1 } 3 +do_changes_test 4.2.2 { INSERT INTO v1 VALUES(1, 2) } 0 + +do_changes_test 4.3.1 { INSERT INTO t1 SELECT * FROM t1 } 6 +do_changes_test 4.3.2 { UPDATE v1 SET y='xyz' WHERE x=1 } 0 + +do_changes_test 4.4.1 { INSERT INTO t1 SELECT * FROM t1 } 12 +do_changes_test 4.4.2 { DELETE FROM v1 WHERE x=5 } 0 + + +#-------------------------------------------------------------------------- +# EVIDENCE-OF: R-32918-61474 Before entering a trigger program the value +# returned by sqlite3_changes() function is saved. After the trigger +# program has finished, the original value is restored. +# +reset_db +db func my_changes my_changes +set ::changes [list] +proc my_changes {x} { + set res [db changes] + lappend ::changes $x $res + return $res +} + +do_execsql_test 5.1.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + CREATE TABLE t2(x); + INSERT INTO t1 VALUES(1, NULL); + INSERT INTO t1 VALUES(2, NULL); + INSERT INTO t1 VALUES(3, NULL); + CREATE TRIGGER AFTER UPDATE ON t1 BEGIN + INSERT INTO t2 VALUES('a'), ('b'), ('c'); + SELECT my_changes('trigger'); + END; +} + +do_execsql_test 5.1.1 { + INSERT INTO t2 VALUES('a'), ('b'); + UPDATE t1 SET b = my_changes('update'); + SELECT * FROM t1; +} {1 2 2 2 3 2} + +# Value is being restored to "2" when the trigger program exits. +do_test 5.1.2 { + set ::changes +} {update 2 trigger 3 update 2 trigger 3 update 2 trigger 3} + + +reset_db +do_execsql_test 5.2.0 { + CREATE TABLE t1(a, b); + CREATE TABLE log(x); + INSERT INTO t1 VALUES(1, 0); + INSERT INTO t1 VALUES(2, 0); + INSERT INTO t1 VALUES(3, 0); + CREATE TRIGGER t1_a_u AFTER UPDATE ON t1 BEGIN + INSERT INTO log VALUES(old.b || ' -> ' || new.b || ' c = ' || changes() ); + END; + CREATE TABLE t2(a); + INSERT INTO t2 VALUES(1), (2), (3); + UPDATE t1 SET b = changes(); +} +do_execsql_test 5.2.1 { + SELECT * FROM t1; +} {1 3 2 3 3 3} +do_execsql_test 5.2.2 { + SELECT * FROM log; +} {{0 -> 3 c = 3} {0 -> 3 c = 3} {0 -> 3 c = 3}} + + +#-------------------------------------------------------------------------- +# EVIDENCE-OF: R-17146-37073 Within a trigger program each INSERT, +# UPDATE and DELETE statement sets the value returned by +# sqlite3_changes() upon completion as normal. Of course, this value +# will not include any changes performed by sub-triggers, as the +# sqlite3_changes() value will be saved and restored after each +# sub-trigger has run. +reset_db +do_execsql_test 6.0 { + + CREATE TABLE t1(a, b); + CREATE TABLE t2(a, b); + CREATE TABLE t3(a, b); + CREATE TABLE log(x); + + CREATE TRIGGER t1_i BEFORE INSERT ON t1 BEGIN + INSERT INTO t2 VALUES(new.a, new.b), (new.a, new.b); + INSERT INTO log VALUES('t2->' || changes()); + END; + + CREATE TRIGGER t2_i AFTER INSERT ON t2 BEGIN + INSERT INTO t3 VALUES(new.a, new.b), (new.a, new.b), (new.a, new.b); + INSERT INTO log VALUES('t3->' || changes()); + END; + + CREATE TRIGGER t1_u AFTER UPDATE ON t1 BEGIN + UPDATE t2 SET b=new.b WHERE a=old.a; + INSERT INTO log VALUES('t2->' || changes()); + END; + + CREATE TRIGGER t2_u BEFORE UPDATE ON t2 BEGIN + UPDATE t3 SET b=new.b WHERE a=old.a; + INSERT INTO log VALUES('t3->' || changes()); + END; + + CREATE TRIGGER t1_d AFTER DELETE ON t1 BEGIN + DELETE FROM t2 WHERE a=old.a AND b=old.b; + INSERT INTO log VALUES('t2->' || changes()); + END; + + CREATE TRIGGER t2_d BEFORE DELETE ON t2 BEGIN + DELETE FROM t3 WHERE a=old.a AND b=old.b; + INSERT INTO log VALUES('t3->' || changes()); + END; +} + +do_changes_test 6.1 { + INSERT INTO t1 VALUES('+', 'o'); + SELECT * FROM log; +} {t3->3 t3->3 t2->2 1} + +do_changes_test 6.2 { + DELETE FROM log; + UPDATE t1 SET b='*'; + SELECT * FROM log; +} {t3->6 t3->6 t2->2 1} + +do_changes_test 6.3 { + DELETE FROM log; + DELETE FROM t1; + SELECT * FROM log; +} {t3->6 t3->0 t2->2 1} + + +#-------------------------------------------------------------------------- +# EVIDENCE-OF: R-43399-09409 This means that if the changes() SQL +# function (or similar) is used by the first INSERT, UPDATE or DELETE +# statement within a trigger, it returns the value as set when the +# calling statement began executing. +# +# EVIDENCE-OF: R-53215-27584 If it is used by the second or subsequent +# such statement within a trigger program, the value returned reflects +# the number of rows modified by the previous INSERT, UPDATE or DELETE +# statement within the same trigger. +# +reset_db +do_execsql_test 7.1 { + CREATE TABLE q1(t); + CREATE TABLE q2(u, v); + CREATE TABLE q3(w); + + CREATE TRIGGER q2_insert BEFORE INSERT ON q2 BEGIN + + /* changes() returns value from previous I/U/D in callers context */ + INSERT INTO q1 VALUES('1:' || changes()); + + /* changes() returns value of previous I/U/D in this context */ + INSERT INTO q3 VALUES(changes()), (2), (3); + INSERT INTO q1 VALUES('2:' || changes()); + INSERT INTO q3 VALUES(changes() + 3), (changes()+4); + SELECT 'this does not affect things!'; + INSERT INTO q1 VALUES('3:' || changes()); + UPDATE q3 SET w = w+10 WHERE w%2; + INSERT INTO q1 VALUES('4:' || changes()); + DELETE FROM q3; + INSERT INTO q1 VALUES('5:' || changes()); + END; +} + +do_execsql_test 7.2 { + INSERT INTO q2 VALUES('x', 'y'); + SELECT * FROM q1; +} { + 1:0 2:3 3:2 4:3 5:5 +} + +do_execsql_test 7.3 { + DELETE FROM q1; + INSERT INTO q2 VALUES('x', 'y'); + SELECT * FROM q1; +} { + 1:5 2:3 3:2 4:3 5:5 +} + + + +finish_test From 4f41b7dec5a99cc72148cf6d0dd66b890cf82619 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 28 Oct 2014 20:35:18 +0000 Subject: [PATCH 039/133] Update the documentation on the sqlite3_randomness() interface to conform to enhancements associated with the SQLITE_ENABLE_API_ARMOR change. FossilOrigin-Name: 96e9917c350dfe2069b87860bbb961424ff1105a --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/sqlite.h.in | 11 ++++++----- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 380d7ccd48..4c55bebcf9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Modify\sthe\sdocumentation\sfor\ssqlite3_changes()\sto\smake\sit\smore\stestable.\sAdd\stests\sand\sminor\sfixes\sfor\sthe\ssame. -D 2014-10-28T18:24:16.387 +C Update\sthe\sdocumentation\son\sthe\ssqlite3_randomness()\sinterface\sto\sconform\nto\senhancements\sassociated\swith\sthe\sSQLITE_ENABLE_API_ARMOR\schange. +D 2014-10-28T20:35:18.499 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -229,7 +229,7 @@ F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c 282f8f5278e0c78eb442217531172ec9e1538796 -F src/sqlite.h.in 1c5624f8b21cc8261a8b048033815581347b375f +F src/sqlite.h.in 9c8090268644fba8d2b5b594b9d577b3da23905c F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 90519c3b3e8ee90adfce013234c4bd07275d77b5 @@ -1208,7 +1208,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 8523670d50004f3112b7871f11c8b8b02aab96ab -R 8192ca2d13d614d67e4c391f6685b170 -U dan -Z 02a949e02b7399a00fe453126f032ba0 +P 41cdd0c422d61533a94870cb5ad094682956d472 +R d8b973f48f438632851d26e03df40631 +U drh +Z d1783aa6143d26d5a0a1c09bffad4d99 diff --git a/manifest.uuid b/manifest.uuid index 1803fe3792..6cff793b96 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -41cdd0c422d61533a94870cb5ad094682956d472 \ No newline at end of file +96e9917c350dfe2069b87860bbb961424ff1105a \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index bb5d90254d..20835794e8 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -2417,13 +2417,14 @@ sqlite3_int64 sqlite3_memory_highwater(int resetFlag); ** applications to access the same PRNG for other purposes. ** ** ^A call to this routine stores N bytes of randomness into buffer P. -** ^If N is less than one, then P can be a NULL pointer. +** ^The P parameter can be a NULL pointer. ** ** ^If this routine has not been previously called or if the previous -** call had N less than one, then the PRNG is seeded using randomness -** obtained from the xRandomness method of the default [sqlite3_vfs] object. -** ^If the previous call to this routine had an N of 1 or more then -** the pseudo-randomness is generated +** call had N less than one or a NULL pointer for P, then the PRNG is +** seeded using randomness obtained from the xRandomness method of +** the default [sqlite3_vfs] object. +** ^If the previous call to this routine had an N of 1 or more and a +** non-NULL P then the pseudo-randomness is generated ** internally and without recourse to the [sqlite3_vfs] xRandomness ** method. */ From aa55563d7eed0ba2e6ab0afd5625b00e30931b1d Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 28 Oct 2014 20:49:59 +0000 Subject: [PATCH 040/133] Add new test file e_totalchanges.test, containing tests of the sqlite3_total_changes() interface. FossilOrigin-Name: f84af4adcc34d7a4c72027bf5b038a1a45a4c307 --- manifest | 15 +-- manifest.uuid | 2 +- src/sqlite.h.in | 25 ++--- test/e_totalchanges.test | 213 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 233 insertions(+), 22 deletions(-) create mode 100644 test/e_totalchanges.test diff --git a/manifest b/manifest index 4c55bebcf9..cbbdd001c6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sthe\sdocumentation\son\sthe\ssqlite3_randomness()\sinterface\sto\sconform\nto\senhancements\sassociated\swith\sthe\sSQLITE_ENABLE_API_ARMOR\schange. -D 2014-10-28T20:35:18.499 +C Add\snew\stest\sfile\se_totalchanges.test,\scontaining\stests\sof\sthe\ssqlite3_total_changes()\sinterface. +D 2014-10-28T20:49:59.213 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -229,7 +229,7 @@ F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c 282f8f5278e0c78eb442217531172ec9e1538796 -F src/sqlite.h.in 9c8090268644fba8d2b5b594b9d577b3da23905c +F src/sqlite.h.in 737b7dd0f3f81fe183646d22828b39f85ef3e68c F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 90519c3b3e8ee90adfce013234c4bd07275d77b5 @@ -464,6 +464,7 @@ F test/e_reindex.test 396b7b4f0a66863b4e95116a67d93b227193e589 F test/e_resolve.test dcce9308fb13b934ce29591105d031d3e14fbba6 F test/e_select.test 52692ff3849541e828ad4661fe3773a9b8711763 F test/e_select2.test aceb80ab927d46fba5ce7586ebabf23e2bb0604f +F test/e_totalchanges.test b12ee5809d3e63aeb83238dd501a7bca7fd72c10 F test/e_update.test 312cb8f5ccfe41515a6bb092f8ea562a9bd54d52 F test/e_uri.test 5ae33760fb2039c61aa2d90886f1664664173585 F test/e_vacuum.test 5bfbdc21b65c0abf24398d0ba31dc88d93ca77a9 @@ -1208,7 +1209,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 41cdd0c422d61533a94870cb5ad094682956d472 -R d8b973f48f438632851d26e03df40631 -U drh -Z d1783aa6143d26d5a0a1c09bffad4d99 +P 96e9917c350dfe2069b87860bbb961424ff1105a +R ff91cb59544b367ee28dfb499ecb1ca4 +U dan +Z ba0579c9ba051395a01d0b2f40d05365 diff --git a/manifest.uuid b/manifest.uuid index 6cff793b96..6aa58547c5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -96e9917c350dfe2069b87860bbb961424ff1105a \ No newline at end of file +f84af4adcc34d7a4c72027bf5b038a1a45a4c307 \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 20835794e8..b4081f2a02 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -1926,20 +1926,17 @@ int sqlite3_changes(sqlite3*); /* ** CAPI3REF: Total Number Of Rows Modified ** -** ^This function returns the number of row changes caused by [INSERT], -** [UPDATE] or [DELETE] statements since the [database connection] was opened. -** ^(The count returned by sqlite3_total_changes() includes all changes -** from all [CREATE TRIGGER | trigger] contexts and changes made by -** [foreign key actions]. However, -** the count does not include changes used to implement [REPLACE] constraints, -** rollbacks or [DROP TABLE] commands. The -** count does not include rows of views that fire an [INSTEAD OF trigger], -** though if the INSTEAD OF trigger makes changes of its own, those changes -** are counted.)^ -** ^The sqlite3_total_changes() function counts the changes as soon as -** the statement that makes them is completed (when the statement handle -** is passed to [sqlite3_reset()] or [sqlite3_finalize()]). -** +** ^This function returns the total number of rows inserted, modified or +** deleted by all [INSERT], [UPDATE] or [DELETE] statements completed +** since the database connection was opened, including those executed as +** part of trigger programs. ^Executing any other type of SQL statement +** does not affect the value returned by sqlite3_total_changes(). +** +** ^Changes made as part of [foreign key actions] are included in the +** count, but those made as part of REPLACE constraint resolution are +** not. ^Changes to a view that are intercepted by INSTEAD OF triggers +** are not counted. +** ** See also the [sqlite3_changes()] interface, the ** [count_changes pragma], and the [total_changes() SQL function]. ** diff --git a/test/e_totalchanges.test b/test/e_totalchanges.test new file mode 100644 index 0000000000..ee163c914f --- /dev/null +++ b/test/e_totalchanges.test @@ -0,0 +1,213 @@ +# 2011 May 06 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix e_totalchanges + +# Like [do_execsql_test], except it appends the value returned by +# [db total_changes] to the result of executing the SQL script. +# +proc do_tc_test {tn sql res} { + uplevel [list \ + do_test $tn "concat \[execsql {$sql}\] \[db total_changes\]" $res + ] +} + +do_execsql_test 1.0 { + CREATE TABLE t1(a, b); + CREATE INDEX t1_b ON t1(b); + CREATE TABLE t2(x, y, PRIMARY KEY(x, y)) WITHOUT ROWID; + CREATE INDEX t2_y ON t2(y); +} + + +#-------------------------------------------------------------------------- +# EVIDENCE-OF: R-65438-26258 This function returns the total number of +# rows inserted, modified or deleted by all INSERT, UPDATE or DELETE +# statements completed since the database connection was opened, +# including those executed as part of trigger programs. +# +# 1.1.*: different types of I/U/D statements, +# 1.2.*: trigger programs. +# +do_tc_test 1.1.1 { + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(3, 4); + UPDATE t1 SET a = a+1; + DELETE FROM t1; +} {6} +do_tc_test 1.1.2 { + DELETE FROM t1 +} {6} + +do_tc_test 1.1.3 { + WITH data(a,b) AS ( + SELECT 0, 0 UNION ALL SELECT a+1, b+1 FROM data WHERE a<99 + ) + INSERT INTO t1 SELECT * FROM data; +} {106} + +do_tc_test 1.1.4 { + INSERT INTO t2 SELECT * FROM t1 WHERE a<50; + UPDATE t2 SET y=y+1; +} {206} + +do_tc_test 1.1.5 { + DELETE FROM t2 WHERE y<=25 +} {231} + +do_execsql_test 1.2.1 { + DELETE FROM t1; + DELETE FROM t2; +} +sqlite3 db test.db ; # To reset total_changes +do_tc_test 1.2.2 { + CREATE TABLE log(detail); + CREATE TRIGGER t1_after_insert AFTER INSERT ON t1 BEGIN + INSERT INTO log VALUES('inserted into t1'); + END; + + CREATE TRIGGER t1_before_delete BEFORE DELETE ON t1 BEGIN + INSERT INTO log VALUES('deleting from t1'); + INSERT INTO log VALUES('here we go!'); + END; + + CREATE TRIGGER t1_after_update AFTER UPDATE ON t1 BEGIN + INSERT INTO log VALUES('update'); + DELETE FROM log; + END; + + INSERT INTO t1 VALUES('a', 'b'); -- 1 + 1 + UPDATE t1 SET b='c'; -- 1 + 1 + 2 + DELETE FROM t1; -- 1 + 1 + 1 +} {9} + +#-------------------------------------------------------------------------- +# EVIDENCE-OF: R-61766-15253 Executing any other type of SQL statement +# does not affect the value returned by sqlite3_total_changes(). +do_tc_test 2.1 { + INSERT INTO t1 VALUES(1, 2), (3, 4); + INSERT INTO t2 VALUES(1, 2), (3, 4); +} {15} +do_tc_test 2.2 { + SELECT count(*) FROM t1; +} {2 15} +do_tc_test 2.3 { + CREATE TABLE t4(a, b); + ALTER TABLE t4 ADD COLUMN c; + CREATE INDEX i4 ON t4(c); + ALTER TABLE t4 RENAME TO t5; + ANALYZE; + BEGIN; + DROP TABLE t2; + ROLLBACK; + VACUUM; +} {15} + + +#-------------------------------------------------------------------------- +# EVIDENCE-OF: R-36043-10590 Changes made as part of foreign key +# actions are included in the count, but those made as part of REPLACE +# constraint resolution are not. +# +# 3.1.*: foreign key actions +# 3.2.*: REPLACE constraints. +# +sqlite3 db test.db ; # To reset total_changes +do_tc_test 3.1.1 { + CREATE TABLE p1(c PRIMARY KEY, d); + CREATE TABLE c1(a, b, FOREIGN KEY(a) REFERENCES p1 ON DELETE SET NULL); + CREATE TABLE c2(a, b, FOREIGN KEY(a) REFERENCES p1 ON DELETE CASCADE); + CREATE TABLE c3(a, b, FOREIGN KEY(a) REFERENCES p1 ON DELETE SET DEFAULT); + + INSERT INTO p1 VALUES(1, 'one'); + INSERT INTO p1 VALUES(2, 'two'); + INSERT INTO p1 VALUES(3, 'three'); + INSERT INTO p1 VALUES(4, 'four'); + + INSERT INTO c1 VALUES(1, 'i'); + INSERT INTO c2 VALUES(2, 'ii'); + INSERT INTO c3 VALUES(3, 'iii'); + PRAGMA foreign_keys = ON; +} {7} + +do_tc_test 3.1.2 { DELETE FROM p1 WHERE c=1; } {9} +do_tc_test 3.1.3 { DELETE FROM p1 WHERE c=2; } {11} +do_tc_test 3.1.4 { DELETE FROM p1 WHERE c=3; } {13} +do_tc_test 3.1.5 { DELETE FROM p1 WHERE c=4; } {14} ; # only 1 this time. + +sqlite3 db test.db ; # To reset total_changes +do_tc_test 3.1.6 { + DROP TABLE c1; + DROP TABLE c2; + DROP TABLE c3; + CREATE TABLE c1(a, b, FOREIGN KEY(a) REFERENCES p1 ON UPDATE SET NULL); + CREATE TABLE c2(a, b, FOREIGN KEY(a) REFERENCES p1 ON UPDATE CASCADE); + CREATE TABLE c3(a, b, FOREIGN KEY(a) REFERENCES p1 ON UPDATE SET DEFAULT); + + INSERT INTO p1 VALUES(1, 'one'); + INSERT INTO p1 VALUES(2, 'two'); + INSERT INTO p1 VALUES(3, 'three'); + INSERT INTO p1 VALUES(4, 'four'); + + INSERT INTO c1 VALUES(1, 'i'); + INSERT INTO c2 VALUES(2, 'ii'); + INSERT INTO c3 VALUES(3, 'iii'); + PRAGMA foreign_keys = ON; +} {7} + +do_tc_test 3.1.7 { UPDATE p1 SET c=c+4 WHERE c=1; } {9} +do_tc_test 3.1.8 { UPDATE p1 SET c=c+4 WHERE c=2; } {11} +do_tc_test 3.1.9 { UPDATE p1 SET c=c+4 WHERE c=3; } {13} +do_tc_test 3.1.10 { UPDATE p1 SET c=c+4 WHERE c=4; } {14} ; # only 1 this time. + +sqlite3 db test.db ; # To reset total_changes +do_tc_test 3.2.1 { + CREATE TABLE t3(a UNIQUE, b UNIQUE); + INSERT INTO t3 VALUES('one', 'one'); + INSERT INTO t3 VALUES('two', 'two'); + INSERT OR REPLACE INTO t3 VALUES('one', 'two'); +} {3} + +do_tc_test 3.2.2 { + INSERT INTO t3 VALUES('three', 'one'); + UPDATE OR REPLACE t3 SET b='two' WHERE b='one'; + SELECT * FROM t3; +} {three two 5} + +#-------------------------------------------------------------------------- +# EVIDENCE-OF: R-54872-08741 Changes to a view that are intercepted by +# INSTEAD OF triggers are not counted. +# +sqlite3 db test.db ; # To reset total_changes +do_tc_test 4.1 { + CREATE TABLE t6(x); + CREATE VIEW v1 AS SELECT * FROM t6; + CREATE TRIGGER v1_tr1 INSTEAD OF INSERT ON v1 BEGIN + SELECT 'no-op'; + END; + + INSERT INTO v1 VALUES('a'); + INSERT INTO v1 VALUES('b'); +} {0} +do_tc_test 4.2 { + CREATE TRIGGER v1_tr2 INSTEAD OF INSERT ON v1 BEGIN + INSERT INTO t6 VALUES(new.x); + END; + + INSERT INTO v1 VALUES('c'); + INSERT INTO v1 VALUES('d'); +} {2} + + +finish_test From 22e8d833f06835a9ea2cdf7e7d2dd55f79102ce0 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 29 Oct 2014 00:58:38 +0000 Subject: [PATCH 041/133] In the OP_Column opcode, when extracting a field that is past the end of a short record (because the row was originally inserted prior to ALTER TABLE ADD COLUMN) then make sure the output register is fully NULL and does not contain leftover flags (such as MEM_Ephem) from its previous use. Fix for ticket [43107840f1c02]. FossilOrigin-Name: 24780f8ddc1683fc62180e6961dc6bfe1168f4df --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/vdbe.c | 2 +- test/update.test | 14 ++++++++++++++ 4 files changed, 24 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index cbbdd001c6..209baf8beb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\snew\stest\sfile\se_totalchanges.test,\scontaining\stests\sof\sthe\ssqlite3_total_changes()\sinterface. -D 2014-10-28T20:49:59.213 +C In\sthe\sOP_Column\sopcode,\swhen\sextracting\sa\sfield\sthat\sis\spast\sthe\send\sof\na\sshort\srecord\s(because\sthe\srow\swas\soriginally\sinserted\sprior\sto\sALTER\sTABLE\nADD\sCOLUMN)\sthen\smake\ssure\sthe\soutput\sregister\sis\sfully\sNULL\sand\sdoes\snot\ncontain\sleftover\sflags\s(such\sas\sMEM_Ephem)\sfrom\sits\sprevious\suse.\nFix\sfor\sticket\s[43107840f1c02]. +D 2014-10-29T00:58:38.898 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/update.c 3c4ecc282accf12d39edb8d524cf089645e55a13 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 3b627daa45c7308c1e36e3dbaa3f9ce7e5c7fa73 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c 1b7e8ccaca2a23ae2804568f34b7c645adfd332d +F src/vdbe.c 49e659bc165e99b28492004b440e22146dc898ab F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 F src/vdbeInt.h acc36ac461f973f46ac7942f86abdd93d2f8cfbc F src/vdbeapi.c 02d8afcff710eb35e3d9e49cb677308296b00009 @@ -1067,7 +1067,7 @@ F test/unique.test 93f8b2ef5ea51b9495f8d6493429b1fd0f465264 F test/unique2.test 41e7f83c6827605991160a31380148a9fc5f1339 F test/unixexcl.test cd6c765f75e50e8e2c2ba763149e5d340ea19825 F test/unordered.test ca7adce0419e4ca0c50f039885e76ed2c531eda8 -F test/update.test 1b6c488a8f993d090b7ee9ad0e234faa161b3aeb +F test/update.test 6c68446b8a0a33d522a7c72b320934596a2d7d32 F test/uri.test 23662b7b61958b0f0e47082de7d06341ccf85d5b F test/userauth01.test e740a2697a7b40d7c5003a7d7edaee16acd349a9 F test/utf16align.test 54cd35a27c005a9b6e7815d887718780b6a462ae @@ -1209,7 +1209,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 96e9917c350dfe2069b87860bbb961424ff1105a -R ff91cb59544b367ee28dfb499ecb1ca4 -U dan -Z ba0579c9ba051395a01d0b2f40d05365 +P f84af4adcc34d7a4c72027bf5b038a1a45a4c307 +R c2870e402bb1970025fb678b8c13d710 +U drh +Z 3a55744e14b55dcc32cb71f070af972c diff --git a/manifest.uuid b/manifest.uuid index 6aa58547c5..ea9bca8740 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f84af4adcc34d7a4c72027bf5b038a1a45a4c307 \ No newline at end of file +24780f8ddc1683fc62180e6961dc6bfe1168f4df \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index d256c6b770..c17bfdfe14 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2441,7 +2441,7 @@ case OP_Column: { if( pOp->p4type==P4_MEM ){ sqlite3VdbeMemShallowCopy(pDest, pOp->p4.pMem, MEM_Static); }else{ - MemSetTypeFlag(pDest, MEM_Null); + sqlite3VdbeMemSetNull(pDest); } goto op_column_out; } diff --git a/test/update.test b/test/update.test index e67b0efddc..d7baf6e702 100644 --- a/test/update.test +++ b/test/update.test @@ -604,5 +604,19 @@ do_test update-14.4 { } ;# ifcapable {trigger} +# Ticket [https://www.sqlite.org/src/tktview/43107840f1c02] on 2014-10-29 +# An assertion fault on UPDATE +# +do_execsql_test update-15.1 { + CREATE TABLE t15(a INTEGER PRIMARY KEY, b); + INSERT INTO t15(a,b) VALUES(10,'abc'),(20,'def'),(30,'ghi'); + ALTER TABLE t15 ADD COLUMN c; + CREATE INDEX t15c ON t15(c); + INSERT INTO t15(a,b) + VALUES(5,'zyx'),(15,'wvu'),(25,'tsr'),(35,'qpo'); + UPDATE t15 SET c=printf("y%d",a) WHERE c IS NULL; + SELECT a,b,c,'|' FROM t15 ORDER BY a; +} {5 zyx y5 | 10 abc y10 | 15 wvu y15 | 20 def y20 | 25 tsr y25 | 30 ghi y30 | 35 qpo y35 |} + finish_test From 62ecc28caf3c45769fe4a7f8aa9e631a2280cf1d Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 29 Oct 2014 01:07:21 +0000 Subject: [PATCH 042/133] Call fsync() right after ftruncate() when in journal_mode=TRUNCATE and when synchronous=FULL in order to ensure that transactions are durable across a power loss that happens moments after the commit. Proposed fix for [https://bugzilla.mozilla.org/show_bug.cgi?id=1072773]. This is a cherrypick of [3e922208b68563489]. FossilOrigin-Name: a8f9bd1e890434fcffa02fcd2baf8b0fb5d4c9dd --- manifest | 19 ++++++++++--------- manifest.uuid | 2 +- src/pager.c | 8 ++++++++ 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index e210943d15..a156aa4f0e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Version\s3.8.7 -D 2014-10-17T11:24:17.839 +C Call\sfsync()\sright\safter\sftruncate()\swhen\sin\sjournal_mode=TRUNCATE\sand\nwhen\ssynchronous=FULL\sin\sorder\sto\sensure\sthat\stransactions\sare\sdurable\nacross\sa\spower\sloss\sthat\shappens\smoments\safter\sthe\scommit.\s\sProposed\nfix\sfor\s[https://bugzilla.mozilla.org/show_bug.cgi?id=1072773].\nThis\sis\sa\scherrypick\sof\s[3e922208b68563489]. +D 2014-10-29T01:07:21.172 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -215,7 +215,7 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c fb587121840f690101336879adfa6d0b2cd0e8c7 F src/os_win.c a019caaae2bcbbc0cc4c39af6e7d7e43d8426053 F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 -F src/pager.c a171cf9dd09c6cb162b262c328d4dfd198e04f80 +F src/pager.c a98547ad9b1b5dbbc5e7d1c520be041b5d2c0926 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 F src/parse.y 5dfead8aed90cb0c7c1115898ee2266804daff45 F src/pcache.c 4121a0571c18581ee9f82f086d5e2030051ebd6a @@ -1204,10 +1204,11 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 06c576c152c4013080c255cbbeb45bf2e298be9f -R 76bb09f8a8f8bdebeb708f5bb39aa31b -T +bgcolor * #d0c0ff -T +sym-release * -T +sym-version-3.8.7 * +P e4ab094f8afce0817f4074e823fabe59fc29ebb4 +Q +3e922208b68563489c7766abb9afb4885113e7b8 +R dac8a0df55bcef8314fa6cd2a33fc123 +T *branch * branch-3.8.7 +T *sym-branch-3.8.7 * +T -sym-trunk * U drh -Z b2f516d1147acb0e1bf1c700327ee52e +Z 1e440e6f18a01c290161fa6c4a1f516b diff --git a/manifest.uuid b/manifest.uuid index f4f611fe44..bf1f77f03a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e4ab094f8afce0817f4074e823fabe59fc29ebb4 \ No newline at end of file +a8f9bd1e890434fcffa02fcd2baf8b0fb5d4c9dd \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index d3a36ef484..d840a39a15 100644 --- a/src/pager.c +++ b/src/pager.c @@ -1941,6 +1941,14 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){ rc = SQLITE_OK; }else{ rc = sqlite3OsTruncate(pPager->jfd, 0); + if( rc==SQLITE_OK && pPager->fullSync ){ + /* Make sure the new file size is written into the inode right away. + ** Otherwise the journal might resurrect following a power loss and + ** cause the last transaction to roll back. See + ** https://bugzilla.mozilla.org/show_bug.cgi?id=1072773 + */ + rc = sqlite3OsSync(pPager->jfd, pPager->syncFlags); + } } pPager->journalOff = 0; }else if( pPager->journalMode==PAGER_JOURNALMODE_PERSIST From 739383d25dc714bf968a67e6f5aae52cc0033e2e Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 29 Oct 2014 01:13:58 +0000 Subject: [PATCH 043/133] In the OP_Column opcode, when extracting a field that is past the end of a short record (because the row was originally inserted prior to ALTER TABLE ADD COLUMN) then make sure the output register is fully NULL and does not contain leftover flags (such as MEM_Ephem) from its previous use. Fix for ticket [43107840f1c02]. This is a cherrypick of check-in [24780f8ddc1683fc]. FossilOrigin-Name: 304ea6ba6f4cf40a76d32d37af73a253f493ba47 --- manifest | 19 ++++++++----------- manifest.uuid | 2 +- src/vdbe.c | 2 +- test/update.test | 14 ++++++++++++++ 4 files changed, 24 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index a156aa4f0e..d38979f110 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Call\sfsync()\sright\safter\sftruncate()\swhen\sin\sjournal_mode=TRUNCATE\sand\nwhen\ssynchronous=FULL\sin\sorder\sto\sensure\sthat\stransactions\sare\sdurable\nacross\sa\spower\sloss\sthat\shappens\smoments\safter\sthe\scommit.\s\sProposed\nfix\sfor\s[https://bugzilla.mozilla.org/show_bug.cgi?id=1072773].\nThis\sis\sa\scherrypick\sof\s[3e922208b68563489]. -D 2014-10-29T01:07:21.172 +C In\sthe\sOP_Column\sopcode,\swhen\sextracting\sa\sfield\sthat\sis\spast\sthe\send\sof\na\sshort\srecord\s(because\sthe\srow\swas\soriginally\sinserted\sprior\sto\sALTER\sTABLE\nADD\sCOLUMN)\sthen\smake\ssure\sthe\soutput\sregister\sis\sfully\sNULL\sand\sdoes\snot\ncontain\sleftover\sflags\s(such\sas\sMEM_Ephem)\sfrom\sits\sprevious\suse.\nFix\sfor\sticket\s[43107840f1c02].\s\sThis\sis\sa\scherrypick\sof\ncheck-in\s[24780f8ddc1683fc]. +D 2014-10-29T01:13:58.092 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/update.c 729f6f18fc27740591d085e1172cebe311144bf0 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 4006c01772bd8d8ac4306d523bbcee41d3e392d8 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c 5ee15a66ce07e0482b92aa29e4dd0c5827a22d79 +F src/vdbe.c 5a1afb571853ddb911d698ac996bc4fd8ddf1eed F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 F src/vdbeInt.h e2a060a55ee18a6ab973353a5e2ec7ee569bf787 F src/vdbeapi.c 37a6c6ae284a97bcace365f2f0a225680c0499d9 @@ -1062,7 +1062,7 @@ F test/unique.test 93f8b2ef5ea51b9495f8d6493429b1fd0f465264 F test/unique2.test 41e7f83c6827605991160a31380148a9fc5f1339 F test/unixexcl.test cd6c765f75e50e8e2c2ba763149e5d340ea19825 F test/unordered.test ca7adce0419e4ca0c50f039885e76ed2c531eda8 -F test/update.test 1b6c488a8f993d090b7ee9ad0e234faa161b3aeb +F test/update.test 6c68446b8a0a33d522a7c72b320934596a2d7d32 F test/uri.test 23662b7b61958b0f0e47082de7d06341ccf85d5b F test/userauth01.test e740a2697a7b40d7c5003a7d7edaee16acd349a9 F test/utf16align.test 54cd35a27c005a9b6e7815d887718780b6a462ae @@ -1204,11 +1204,8 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P e4ab094f8afce0817f4074e823fabe59fc29ebb4 -Q +3e922208b68563489c7766abb9afb4885113e7b8 -R dac8a0df55bcef8314fa6cd2a33fc123 -T *branch * branch-3.8.7 -T *sym-branch-3.8.7 * -T -sym-trunk * +P a8f9bd1e890434fcffa02fcd2baf8b0fb5d4c9dd +Q +24780f8ddc1683fc62180e6961dc6bfe1168f4df +R d684ab06f9952cfb8bd4bb795d206cf9 U drh -Z 1e440e6f18a01c290161fa6c4a1f516b +Z 6a68e4e50056aa43b6faddb91db24dac diff --git a/manifest.uuid b/manifest.uuid index bf1f77f03a..e62a923d4d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a8f9bd1e890434fcffa02fcd2baf8b0fb5d4c9dd \ No newline at end of file +304ea6ba6f4cf40a76d32d37af73a253f493ba47 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 0f9f45c456..88fadb023e 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2441,7 +2441,7 @@ case OP_Column: { if( pOp->p4type==P4_MEM ){ sqlite3VdbeMemShallowCopy(pDest, pOp->p4.pMem, MEM_Static); }else{ - MemSetTypeFlag(pDest, MEM_Null); + sqlite3VdbeMemSetNull(pDest); } goto op_column_out; } diff --git a/test/update.test b/test/update.test index e67b0efddc..d7baf6e702 100644 --- a/test/update.test +++ b/test/update.test @@ -604,5 +604,19 @@ do_test update-14.4 { } ;# ifcapable {trigger} +# Ticket [https://www.sqlite.org/src/tktview/43107840f1c02] on 2014-10-29 +# An assertion fault on UPDATE +# +do_execsql_test update-15.1 { + CREATE TABLE t15(a INTEGER PRIMARY KEY, b); + INSERT INTO t15(a,b) VALUES(10,'abc'),(20,'def'),(30,'ghi'); + ALTER TABLE t15 ADD COLUMN c; + CREATE INDEX t15c ON t15(c); + INSERT INTO t15(a,b) + VALUES(5,'zyx'),(15,'wvu'),(25,'tsr'),(35,'qpo'); + UPDATE t15 SET c=printf("y%d",a) WHERE c IS NULL; + SELECT a,b,c,'|' FROM t15 ORDER BY a; +} {5 zyx y5 | 10 abc y10 | 15 wvu y15 | 20 def y20 | 25 tsr y25 | 30 ghi y30 | 35 qpo y35 |} + finish_test From bba8206be97e8b09ee9c135e94c55db10d4d19c0 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 29 Oct 2014 01:18:03 +0000 Subject: [PATCH 044/133] Disable the use of strchrnul() unless specifically enabled by compile-time options. Cherrypick of check-in [e580470db77d6da9] FossilOrigin-Name: 837368adfe859c41b347d3124d5b3fdf790eec03 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/printf.c | 6 +----- 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index d38979f110..f5c3788a32 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\sOP_Column\sopcode,\swhen\sextracting\sa\sfield\sthat\sis\spast\sthe\send\sof\na\sshort\srecord\s(because\sthe\srow\swas\soriginally\sinserted\sprior\sto\sALTER\sTABLE\nADD\sCOLUMN)\sthen\smake\ssure\sthe\soutput\sregister\sis\sfully\sNULL\sand\sdoes\snot\ncontain\sleftover\sflags\s(such\sas\sMEM_Ephem)\sfrom\sits\sprevious\suse.\nFix\sfor\sticket\s[43107840f1c02].\s\sThis\sis\sa\scherrypick\sof\ncheck-in\s[24780f8ddc1683fc]. -D 2014-10-29T01:13:58.092 +C Disable\sthe\suse\sof\sstrchrnul()\sunless\sspecifically\senabled\sby\scompile-time\noptions.\s\sCherrypick\sof\scheck-in\s[e580470db77d6da9] +D 2014-10-29T01:18:03.185 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -223,7 +223,7 @@ F src/pcache.h 9b559127b83f84ff76d735c8262f04853be0c59a F src/pcache1.c e412cb585f777c840ddce0500eddc5c6043c2bb5 F src/pragma.c 3f3e959390a10c0131676f0e307acce372777e0f F src/prepare.c 6ef0cf2f9274982988ed6b7cab1be23147e94196 -F src/printf.c 6b79bbd063dcbadca4cf617a4cde255bcc13ea64 +F src/printf.c 090fac0f779c93c8a95089a125339686648835e4 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c a3466128b52a86c466e47ac1a19e2174f7b5cf89 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e @@ -1204,8 +1204,8 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P a8f9bd1e890434fcffa02fcd2baf8b0fb5d4c9dd -Q +24780f8ddc1683fc62180e6961dc6bfe1168f4df -R d684ab06f9952cfb8bd4bb795d206cf9 +P 304ea6ba6f4cf40a76d32d37af73a253f493ba47 +Q +e580470db77d6da970c755102790e603fb26b3c6 +R 87a2c5d1dcd7231208aa0589a51fed61 U drh -Z 6a68e4e50056aa43b6faddb91db24dac +Z b2af8bf5eda5ca802506e90d5ed9925a diff --git a/manifest.uuid b/manifest.uuid index e62a923d4d..3e0b305a64 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -304ea6ba6f4cf40a76d32d37af73a253f493ba47 \ No newline at end of file +837368adfe859c41b347d3124d5b3fdf790eec03 \ No newline at end of file diff --git a/src/printf.c b/src/printf.c index c0b3c70f6b..1df287fbb6 100644 --- a/src/printf.c +++ b/src/printf.c @@ -21,11 +21,7 @@ ** the glibc version so the glibc version is definitely preferred. */ #if !defined(HAVE_STRCHRNUL) -# if defined(linux) -# define HAVE_STRCHRNUL 1 -# else -# define HAVE_STRCHRNUL 0 -# endif +# define HAVE_STRCHRNUL 0 #endif From a26b9a77275e5ef76e2ba5ac0c480e70f5abeda4 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 29 Oct 2014 01:26:25 +0000 Subject: [PATCH 045/133] Fix problems with running UPDATE and DELETE against a VIEW and referencing the rowid in the WHERE clause. This is a cherrypick of [95f8ebdbf87326f2] and [8523670d50004f3]. FossilOrigin-Name: cc33e846c8509419f0a1fbfb286807b4c137788d --- manifest | 19 ++++++++++--------- manifest.uuid | 2 +- src/delete.c | 2 +- src/update.c | 4 ++-- test/trigger9.test | 33 +++++++++++++++++++++++++++++++++ 5 files changed, 47 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index f5c3788a32..be89f4c86a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Disable\sthe\suse\sof\sstrchrnul()\sunless\sspecifically\senabled\sby\scompile-time\noptions.\s\sCherrypick\sof\scheck-in\s[e580470db77d6da9] -D 2014-10-29T01:18:03.185 +C Fix\sproblems\swith\srunning\sUPDATE\sand\sDELETE\sagainst\sa\sVIEW\sand\sreferencing\nthe\srowid\sin\sthe\sWHERE\sclause.\s\sThis\sis\sa\scherrypick\sof\s[95f8ebdbf87326f2]\nand\s[8523670d50004f3]. +D 2014-10-29T01:26:25.633 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -180,7 +180,7 @@ F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 F src/complete.c 535183afb3c75628b78ce82612931ac7cdf26f14 F src/ctime.c bb434068b5308a857b181c2d204a320ff0d6c638 F src/date.c 57a7f9ba9f6b4d5268f5e411739066a611f99036 -F src/delete.c fae81cc2eb14b75267d4f47d3cfc9ae02aae726f +F src/delete.c 0750b1eb4d96cd3fb2c798599a3a7c85e92f1417 F src/expr.c fc204d08af06437ddaffe5a1b1f1f6f9e1a55d6d F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c da985ae673efef2c712caef825a5d2edb087ead7 @@ -285,7 +285,7 @@ F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 6de09362b657f19ba83e5fa521ee715787ce9fee F src/tokenize.c cc9016e5007fc5e76789079616d2f26741bcc689 F src/trigger.c 25571661fdeae8c7f975ff40ffec205520a3f92f -F src/update.c 729f6f18fc27740591d085e1172cebe311144bf0 +F src/update.c 3c4ecc282accf12d39edb8d524cf089645e55a13 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 4006c01772bd8d8ac4306d523bbcee41d3e392d8 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a @@ -1048,7 +1048,7 @@ F test/trigger5.test 619391a3e9fc194081d22cefd830d811e7badf83 F test/trigger6.test 0e411654f122552da6590f0b4e6f781048a4a9b9 F test/trigger7.test b39e6dee1debe0ff9c2ef66326668f149f07c9c4 F test/trigger8.test 30cb0530bd7c4728055420e3f739aa00412eafa4 -F test/trigger9.test 5b0789f1c5c4600961f8e68511b825b87be53e31 +F test/trigger9.test 2226ec795a33b0460ab5cf8891e9054cc7edef41 F test/triggerA.test fe5597f47ee21bacb4936dc827994ed94161e332 F test/triggerB.test 56780c031b454abac2340dbb3b71ac5c56c3d7fe F test/triggerC.test a68980c5955d62ee24be6f97129d824f199f9a4c @@ -1204,8 +1204,9 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 304ea6ba6f4cf40a76d32d37af73a253f493ba47 -Q +e580470db77d6da970c755102790e603fb26b3c6 -R 87a2c5d1dcd7231208aa0589a51fed61 +P 837368adfe859c41b347d3124d5b3fdf790eec03 +Q +8523670d50004f3112b7871f11c8b8b02aab96ab +Q +95f8ebdbf87326f23cd38e561ac5632b5367a449 +R b3ee8c71d158066739429e7ae8387325 U drh -Z b2af8bf5eda5ca802506e90d5ed9925a +Z 624aac6d2575ef133925a49cc2066eb1 diff --git a/manifest.uuid b/manifest.uuid index 3e0b305a64..99e8b826f5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -837368adfe859c41b347d3124d5b3fdf790eec03 \ No newline at end of file +cc33e846c8509419f0a1fbfb286807b4c137788d \ No newline at end of file diff --git a/src/delete.c b/src/delete.c index b97407400b..d81dd3f6b4 100644 --- a/src/delete.c +++ b/src/delete.c @@ -481,7 +481,7 @@ void sqlite3DeleteFrom( assert( nKey==nPk ); /* OP_Found will use an unpacked key */ assert( !IsVirtual(pTab) ); if( aToOpen[iDataCur-iTabCur] ){ - assert( pPk!=0 ); + assert( pPk!=0 || pTab->pSelect!=0 ); sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, addrBypass, iKey, nKey); VdbeCoverage(v); } diff --git a/src/update.c b/src/update.c index f781a60ccd..3af4017f1b 100644 --- a/src/update.c +++ b/src/update.c @@ -431,8 +431,8 @@ void sqlite3Update( /* Top of the update loop */ if( okOnePass ){ - if( aToOpen[iDataCur-iBaseCur] ){ - assert( pPk!=0 ); + if( aToOpen[iDataCur-iBaseCur] && !isView ){ + assert( pPk ); sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, labelBreak, regKey, nKey); VdbeCoverageNeverTaken(v); } diff --git a/test/trigger9.test b/test/trigger9.test index f56c8acbc5..326fa63d4c 100644 --- a/test/trigger9.test +++ b/test/trigger9.test @@ -32,6 +32,7 @@ ifcapable {!trigger} { finish_test return } +set ::testprefix trigger9 proc has_rowdata {sql} { expr {[lsearch [execsql "explain $sql"] RowData]>=0} @@ -220,4 +221,36 @@ ifcapable compound { } {2} } +reset_db +do_execsql_test 4.1 { + CREATE TABLE t1(a, b); + CREATE TABLE log(x); + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(3, 4); + CREATE VIEW v1 AS SELECT a, b FROM t1; + + CREATE TRIGGER tr1 INSTEAD OF DELETE ON v1 BEGIN + INSERT INTO log VALUES('delete'); + END; + + CREATE TRIGGER tr2 INSTEAD OF UPDATE ON v1 BEGIN + INSERT INTO log VALUES('update'); + END; + + CREATE TRIGGER tr3 INSTEAD OF INSERT ON v1 BEGIN + INSERT INTO log VALUES('insert'); + END; +} + +do_execsql_test 4.2 { + DELETE FROM v1 WHERE rowid=1; +} {} + +do_execsql_test 4.3 { + UPDATE v1 SET a=b WHERE rowid=2; +} {} + + + + finish_test From 35404253f680a8d6939a09d24ad4286ade7b5e8f Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 29 Oct 2014 01:27:43 +0000 Subject: [PATCH 046/133] Increase the version number to 3.8.7.1 FossilOrigin-Name: 83afe23e553e802c0947c80d0ffdd120423e7c52 --- VERSION | 2 +- configure | 18 +++++++++--------- manifest | 16 +++++++--------- manifest.uuid | 2 +- 4 files changed, 18 insertions(+), 20 deletions(-) diff --git a/VERSION b/VERSION index 4351a7e3a3..f53c1ed56b 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.8.7 +3.8.7.1 diff --git a/configure b/configure index 9b8266d812..d253f21a7d 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.62 for sqlite 3.8.7. +# Generated by GNU Autoconf 2.62 for sqlite 3.8.7.1. # # Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, # 2002, 2003, 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc. @@ -743,8 +743,8 @@ SHELL=${CONFIG_SHELL-/bin/sh} # Identity of this package. PACKAGE_NAME='sqlite' PACKAGE_TARNAME='sqlite' -PACKAGE_VERSION='3.8.7' -PACKAGE_STRING='sqlite 3.8.7' +PACKAGE_VERSION='3.8.7.1' +PACKAGE_STRING='sqlite 3.8.7.1' PACKAGE_BUGREPORT='' # Factoring default headers for most tests. @@ -1483,7 +1483,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures sqlite 3.8.7 to adapt to many kinds of systems. +\`configure' configures sqlite 3.8.7.1 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1548,7 +1548,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of sqlite 3.8.7:";; + short | recursive ) echo "Configuration of sqlite 3.8.7.1:";; esac cat <<\_ACEOF @@ -1664,7 +1664,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -sqlite configure 3.8.7 +sqlite configure 3.8.7.1 generated by GNU Autoconf 2.62 Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, @@ -1678,7 +1678,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by sqlite $as_me 3.8.7, which was +It was created by sqlite $as_me 3.8.7.1, which was generated by GNU Autoconf 2.62. Invocation command line was $ $0 $@ @@ -14021,7 +14021,7 @@ exec 6>&1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by sqlite $as_me 3.8.7, which was +This file was extended by sqlite $as_me 3.8.7.1, which was generated by GNU Autoconf 2.62. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -14074,7 +14074,7 @@ Report bugs to ." _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_version="\\ -sqlite config.status 3.8.7 +sqlite config.status 3.8.7.1 configured by $0, generated by GNU Autoconf 2.62, with options \\"`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`\\" diff --git a/manifest b/manifest index be89f4c86a..916213d465 100644 --- a/manifest +++ b/manifest @@ -1,12 +1,12 @@ -C Fix\sproblems\swith\srunning\sUPDATE\sand\sDELETE\sagainst\sa\sVIEW\sand\sreferencing\nthe\srowid\sin\sthe\sWHERE\sclause.\s\sThis\sis\sa\scherrypick\sof\s[95f8ebdbf87326f2]\nand\s[8523670d50004f3]. -D 2014-10-29T01:26:25.633 +C Increase\sthe\sversion\snumber\sto\s3.8.7.1 +D 2014-10-29T01:27:43.048 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 F Makefile.msc e31dee24038965fb6269d6d61073fd6b7e331dec F Makefile.vxworks 034289efa9d591b04b1a73598623119c306cbba0 F README.md 64f270c43c38c46de749e419c22f0ae2f4499fe8 -F VERSION 53a0b870e7f16d3b06623c31d233a304c163a6af +F VERSION 5cc0baaee7eeee3238f0f7b5871398d17f79d0cd F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 F addopcodes.awk 9eb448a552d5c0185cf62c463f9c173cedae3811 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2 @@ -38,7 +38,7 @@ F autoconf/tea/win/rules.vc c511f222b80064096b705dbeb97060ee1d6b6d63 F config.guess 226d9a188c6196f3033ffc651cbc9dcee1a42977 F config.h.in 0921066a13130082764ab4ab6456f7b5bebe56de F config.sub 9ebe4c3b3dab6431ece34f16828b594fb420da55 -F configure ad59a5f48b3c59a92b5506040a22fbe3f733a9d8 x +F configure 56fe985cf0e59cd594f9b929099d0be40260e667 x F configure.ac 4cf9f60785143fa141b10962ccc885d973792e9a F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad F doc/lemon.html 334dbf6621b8fb8790297ec1abf3cfa4621709d1 @@ -1204,9 +1204,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 837368adfe859c41b347d3124d5b3fdf790eec03 -Q +8523670d50004f3112b7871f11c8b8b02aab96ab -Q +95f8ebdbf87326f23cd38e561ac5632b5367a449 -R b3ee8c71d158066739429e7ae8387325 +P cc33e846c8509419f0a1fbfb286807b4c137788d +R 5381895dceb9bc7ad4d425f399561cb7 U drh -Z 624aac6d2575ef133925a49cc2066eb1 +Z 906d83e1887c766ebb0692623bc0209a diff --git a/manifest.uuid b/manifest.uuid index 99e8b826f5..abe23b5708 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cc33e846c8509419f0a1fbfb286807b4c137788d \ No newline at end of file +83afe23e553e802c0947c80d0ffdd120423e7c52 \ No newline at end of file From 7abfe9d0c62966783f878c94ca07ee4f7a557b72 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 29 Oct 2014 13:59:56 +0000 Subject: [PATCH 047/133] Version 3.8.7.1 FossilOrigin-Name: 3b7b72c4685aa5cf5e675c2c47ebec10d9704221 --- manifest | 11 +++++++---- manifest.uuid | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/manifest b/manifest index 916213d465..8688f8549f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Increase\sthe\sversion\snumber\sto\s3.8.7.1 -D 2014-10-29T01:27:43.048 +C Version\s3.8.7.1 +D 2014-10-29T13:59:56.070 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1204,7 +1204,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P cc33e846c8509419f0a1fbfb286807b4c137788d +P 83afe23e553e802c0947c80d0ffdd120423e7c52 R 5381895dceb9bc7ad4d425f399561cb7 +T +bgcolor * #d0c0ff +T +sym-release * +T +sym-version-3.8.7.1 * U drh -Z 906d83e1887c766ebb0692623bc0209a +Z e5ce942eb65e60b92fcaefcb5b84039b diff --git a/manifest.uuid b/manifest.uuid index abe23b5708..1787d659c6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -83afe23e553e802c0947c80d0ffdd120423e7c52 \ No newline at end of file +3b7b72c4685aa5cf5e675c2c47ebec10d9704221 \ No newline at end of file From af8f513f9d9414b8aae11f9470e4a8789e7cf74e Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 29 Oct 2014 18:20:18 +0000 Subject: [PATCH 048/133] Fix the %c format character in sqlite3VXPrintf() so that it correctly handles precisions larger than 70. FossilOrigin-Name: 08a27440f19b7fc884464832e6105af1bf008172 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/printf.c | 33 +++++++++++++++++++-------------- src/sqliteInt.h | 2 +- test/printf2.test | 20 ++++++++++++++++++++ 5 files changed, 49 insertions(+), 24 deletions(-) diff --git a/manifest b/manifest index 209baf8beb..c8c7eafdd4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\sOP_Column\sopcode,\swhen\sextracting\sa\sfield\sthat\sis\spast\sthe\send\sof\na\sshort\srecord\s(because\sthe\srow\swas\soriginally\sinserted\sprior\sto\sALTER\sTABLE\nADD\sCOLUMN)\sthen\smake\ssure\sthe\soutput\sregister\sis\sfully\sNULL\sand\sdoes\snot\ncontain\sleftover\sflags\s(such\sas\sMEM_Ephem)\sfrom\sits\sprevious\suse.\nFix\sfor\sticket\s[43107840f1c02]. -D 2014-10-29T00:58:38.898 +C Fix\sthe\s%c\sformat\scharacter\sin\ssqlite3VXPrintf()\sso\sthat\sit\scorrectly\nhandles\sprecisions\slarger\sthan\s70. +D 2014-10-29T18:20:18.932 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -223,7 +223,7 @@ F src/pcache.h 9b559127b83f84ff76d735c8262f04853be0c59a F src/pcache1.c e412cb585f777c840ddce0500eddc5c6043c2bb5 F src/pragma.c 3f3e959390a10c0131676f0e307acce372777e0f F src/prepare.c b7b7bf020bd4c962f7c8aed5a3c542c7dfe9f9c7 -F src/printf.c c31012ac23e458081df4a32634b60424e0cdfaf3 +F src/printf.c 10a2493593c8e4a538915cd3674bd7a67f70c488 F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e @@ -232,7 +232,7 @@ F src/shell.c 282f8f5278e0c78eb442217531172ec9e1538796 F src/sqlite.h.in 737b7dd0f3f81fe183646d22828b39f85ef3e68c F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d -F src/sqliteInt.h 90519c3b3e8ee90adfce013234c4bd07275d77b5 +F src/sqliteInt.h 4f86ac648ea398c1bb3db036062934cde257ea23 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 81712116e826b0089bb221b018929536b2b5406f F src/table.c f142bba7903e93ca8d113a5b8877a108ad1a27dc @@ -772,7 +772,7 @@ F test/permutations.test cef25f5e8499a15846eccd06785f17f4180407ab F test/pragma.test 19d0241a007bcdd77fc2606ec60fc60357e7fc8b F test/pragma2.test aea7b3d82c76034a2df2b38a13745172ddc0bc13 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 -F test/printf2.test bed79b4c3e5da08ba88ad637c0bf62586843cfb1 +F test/printf2.test b4acd4bf8734243257f01ddefa17c4fb090acc8a F test/progress.test a282973d1d17f08071bc58a77d6b80f2a81c354d F test/ptrchng.test ef1aa72d6cf35a2bbd0869a649b744e9d84977fc F test/queryonly.test 5f653159e0f552f0552d43259890c1089391dcca @@ -1209,7 +1209,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f84af4adcc34d7a4c72027bf5b038a1a45a4c307 -R c2870e402bb1970025fb678b8c13d710 +P 24780f8ddc1683fc62180e6961dc6bfe1168f4df +R c12283d048765bdbe52bad70443f70c1 U drh -Z 3a55744e14b55dcc32cb71f070af972c +Z 8aeb3112f50d033e0580c4400e41dd66 diff --git a/manifest.uuid b/manifest.uuid index ea9bca8740..bd48a08b43 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -24780f8ddc1683fc62180e6961dc6bfe1168f4df \ No newline at end of file +08a27440f19b7fc884464832e6105af1bf008172 \ No newline at end of file diff --git a/src/printf.c b/src/printf.c index f000da7fcc..85d237f0e7 100644 --- a/src/printf.c +++ b/src/printf.c @@ -212,7 +212,7 @@ void sqlite3VXPrintf( const et_info *infop; /* Pointer to the appropriate info structure */ char *zOut; /* Rendering buffer */ int nOut; /* Size of the rendering buffer */ - char *zExtra; /* Malloced memory used by some conversion */ + char *zExtra = 0; /* Malloced memory used by some conversion */ #ifndef SQLITE_OMIT_FLOATING_POINT int exp, e2; /* exponent of real numbers */ int nsd; /* Number of significant digits returned */ @@ -336,7 +336,6 @@ void sqlite3VXPrintf( break; } } - zExtra = 0; /* ** At this point, variables are initialized as follows: @@ -627,13 +626,16 @@ void sqlite3VXPrintf( }else{ c = va_arg(ap,int); } - buf[0] = (char)c; - if( precision>=0 ){ - for(idx=1; idx1 ){ + width -= precision-1; + if( width>1 && !flag_leftjustify ){ + sqlite3AppendChar(pAccum, width-1, ' '); + width = 0; + } + sqlite3AppendChar(pAccum, precision-1, c); } + length = 1; + buf[0] = c; bufpt = buf; break; case etSTRING: @@ -734,11 +736,14 @@ void sqlite3VXPrintf( ** the output. */ width -= length; - if( width>0 && !flag_leftjustify ) sqlite3AppendSpace(pAccum, width); + if( width>0 && !flag_leftjustify ) sqlite3AppendChar(pAccum, width, ' '); sqlite3StrAccumAppend(pAccum, bufpt, length); - if( width>0 && flag_leftjustify ) sqlite3AppendSpace(pAccum, width); + if( width>0 && flag_leftjustify ) sqlite3AppendChar(pAccum, width, ' '); - if( zExtra ) sqlite3_free(zExtra); + if( zExtra ){ + sqlite3_free(zExtra); + zExtra = 0; + } }/* End for loop over the format string */ } /* End of function */ @@ -791,11 +796,11 @@ static int sqlite3StrAccumEnlarge(StrAccum *p, int N){ } /* -** Append N space characters to the given string buffer. +** Append N copies of character c to the given string buffer. */ -void sqlite3AppendSpace(StrAccum *p, int N){ +void sqlite3AppendChar(StrAccum *p, int N, char c){ if( p->nChar+N >= p->nAlloc && (N = sqlite3StrAccumEnlarge(p, N))<=0 ) return; - while( (N--)>0 ) p->zText[p->nChar++] = ' '; + while( (N--)>0 ) p->zText[p->nChar++] = c; } /* diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 5114ccccf4..f4785411d5 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3543,7 +3543,7 @@ int sqlite3OpenTempDatabase(Parse *); void sqlite3StrAccumInit(StrAccum*, char*, int, int); void sqlite3StrAccumAppend(StrAccum*,const char*,int); void sqlite3StrAccumAppendAll(StrAccum*,const char*); -void sqlite3AppendSpace(StrAccum*,int); +void sqlite3AppendChar(StrAccum*,int,char); char *sqlite3StrAccumFinish(StrAccum*); void sqlite3StrAccumReset(StrAccum*); void sqlite3SelectDestInit(SelectDest*,int,int); diff --git a/test/printf2.test b/test/printf2.test index 4cb1783bfb..21deeb779d 100644 --- a/test/printf2.test +++ b/test/printf2.test @@ -95,5 +95,25 @@ do_execsql_test printf2-2.3 { SELECT printf('%s=(%d/%g/%s)',a) FROM t1 ORDER BY a; } {-1=(0/0/) 1=(0/0/) 1.5=(0/0/) abc=(0/0/)} +# The precision of the %c conversion causes the character to repeat. +# +do_execsql_test printf2-3.1 { + SELECT printf('|%110.100c|','*'); +} {{| ****************************************************************************************************|}} +do_execsql_test printf2-3.2 { + SELECT printf('|%-110.100c|','*'); +} {{|**************************************************************************************************** |}} +do_execsql_test printf2-3.3 { + SELECT printf('|%9.8c|%-9.8c|','*','*'); +} {{| ********|******** |}} +do_execsql_test printf2-3.4 { + SELECT printf('|%8.8c|%-8.8c|','*','*'); +} {|********|********|} +do_execsql_test printf2-3.5 { + SELECT printf('|%7.8c|%-7.8c|','*','*'); +} {|********|********|} + + + finish_test From ba0f99941848816581ac12bd1d5f6d03216f5338 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 30 Oct 2014 20:48:44 +0000 Subject: [PATCH 049/133] Tweaks to comments in btree.c. Minor code changes to enhance testability. FossilOrigin-Name: c7d9aa3a1ce63e27ec94295601bc89fecf1e4977 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 20 ++++++++++---------- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index c8c7eafdd4..c8c02f1b16 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\s%c\sformat\scharacter\sin\ssqlite3VXPrintf()\sso\sthat\sit\scorrectly\nhandles\sprecisions\slarger\sthan\s70. -D 2014-10-29T18:20:18.932 +C Tweaks\sto\scomments\sin\sbtree.c.\s\sMinor\scode\schanges\sto\senhance\stestability. +D 2014-10-30T20:48:44.305 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 7f841396adfd47507ff670a471162d2bfcda3136 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 5189881ca403938c5ceddde496b984fef9f40c5a +F src/btree.c 8d955d8ef15dd724ea5ef1cb65c17151beaff1e0 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 67bb05b1077e0cdaccb2e36bfcbe7a5df9ed31e8 @@ -1209,7 +1209,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 24780f8ddc1683fc62180e6961dc6bfe1168f4df -R c12283d048765bdbe52bad70443f70c1 +P 08a27440f19b7fc884464832e6105af1bf008172 +R 984c5e022c7b2ce03eb5bed97ff43b89 U drh -Z 8aeb3112f50d033e0580c4400e41dd66 +Z 333c45d7dfd79a88b32c00dfbc4028a5 diff --git a/manifest.uuid b/manifest.uuid index bd48a08b43..8964ddfef6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -08a27440f19b7fc884464832e6105af1bf008172 \ No newline at end of file +c7d9aa3a1ce63e27ec94295601bc89fecf1e4977 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 22b168d9e4..e56cdf81ed 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1231,10 +1231,8 @@ static int defragmentPage(MemPage *pPage){ ** ** If no suitable space can be found on the free-list, return NULL. ** -** This function may detect corruption within pPg. If it does and argument -** pRc is non-NULL, then *pRc is set to SQLITE_CORRUPT and NULL is returned. -** Or, if corruption is detected and pRc is NULL, NULL is returned and the -** corruption goes unreported. +** This function may detect corruption within pPg. If corruption is +** detected then *pRc is set to SQLITE_CORRUPT and NULL is returned. ** ** If a slot of at least nByte bytes is found but cannot be used because ** there are already at least 60 fragmented bytes on the page, return NULL. @@ -1250,7 +1248,7 @@ static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc, int *pbDefrag){ for(iAddr=hdr+1; (pc = get2byte(&aData[iAddr]))>0; iAddr=pc){ int size; /* Size of the free slot */ if( pc>usableSize-4 || pc usableSize ){ - if( pRc ) *pRc = SQLITE_CORRUPT_BKPT; + *pRc = SQLITE_CORRUPT_BKPT; return 0; }else{ /* The slot remains on the free-list. Reduce its size to account @@ -6076,8 +6074,9 @@ static int pageInsertArray( assert( CORRUPT_DB || pPg->hdrOffset==0 ); /* Never called on page 1 */ for(i=0; i Date: Thu, 30 Oct 2014 23:14:56 +0000 Subject: [PATCH 050/133] Improvements to the wording of some comments. Reinstate an assert() that is only true for non-corrupt database files by adding an "|| CORRUPT_DB" term. FossilOrigin-Name: 67adb44838f98805f86aecca634d9a3b07370b9e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 27 ++++++++++++--------------- 3 files changed, 19 insertions(+), 22 deletions(-) diff --git a/manifest b/manifest index c8c02f1b16..98a273f2e0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Tweaks\sto\scomments\sin\sbtree.c.\s\sMinor\scode\schanges\sto\senhance\stestability. -D 2014-10-30T20:48:44.305 +C Improvements\sto\sthe\swording\sof\ssome\scomments.\s\sReinstate\san\sassert()\sthat\nis\sonly\strue\sfor\snon-corrupt\sdatabase\sfiles\sby\sadding\san\s"||\sCORRUPT_DB"\sterm. +D 2014-10-30T23:14:56.756 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 7f841396adfd47507ff670a471162d2bfcda3136 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 8d955d8ef15dd724ea5ef1cb65c17151beaff1e0 +F src/btree.c 8f7ea96935c3b1db94439204965a6ec1392f7977 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 67bb05b1077e0cdaccb2e36bfcbe7a5df9ed31e8 @@ -1209,7 +1209,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 08a27440f19b7fc884464832e6105af1bf008172 -R 984c5e022c7b2ce03eb5bed97ff43b89 +P c7d9aa3a1ce63e27ec94295601bc89fecf1e4977 +R 230a6673a34b4786200f7ae54e553a63 U drh -Z 333c45d7dfd79a88b32c00dfbc4028a5 +Z 52f8e66a5796212d2c4dbab1cb7cef14 diff --git a/manifest.uuid b/manifest.uuid index 8964ddfef6..fd375ba77a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c7d9aa3a1ce63e27ec94295601bc89fecf1e4977 \ No newline at end of file +67adb44838f98805f86aecca634d9a3b07370b9e \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index e56cdf81ed..2c3cbb4adc 100644 --- a/src/btree.c +++ b/src/btree.c @@ -6526,7 +6526,7 @@ static int balance_nonroot( u8 *apDiv[NB-1]; /* Divider cells in pParent */ int cntNew[NB+2]; /* Index in aCell[] of cell after i-th page */ int cntOld[NB+2]; /* Old index in aCell[] after i-th page */ - int szNew[NB+2]; /* Combined size of cells place on i-th page */ + int szNew[NB+2]; /* Combined size of cells placed on i-th page */ u8 **apCell = 0; /* All cells begin balanced */ u16 *szCell; /* Local size of all cells in apCell[] */ u8 *aSpace1; /* Space for copies of dividers cells */ @@ -6765,9 +6765,10 @@ static int balance_nonroot( /* ** The packing computed by the previous block is biased toward the siblings - ** on the left side. The left siblings are always nearly full, while the - ** right-most sibling might be nearly empty. This block of code attempts - ** to adjust the packing of siblings to get a better balance. + ** on the left side (siblings with smaller keys). The left siblings are + ** always nearly full, while the right-most sibling might be nearly empty. + ** The next block of code attempts to adjust the packing of siblings to + ** get a better balance. ** ** This adjustment is more than an optimization. The packing above might ** be so out of balance as to be illegal. For example, the right-most @@ -6796,18 +6797,14 @@ static int balance_nonroot( szNew[i-1] = szLeft; } - /* Either we found one or more cells (cntnew[0])>0) or pPage is - ** a virtual root page. A virtual root page is when the real root - ** page is page 1 and we are the only child of that page. - ** - ** UPDATE: The assert() below is not necessarily true if the database - ** file is corrupt. The corruption will be detected and reported later - ** in this procedure so there is no need to act upon it now. + /* Sanity check: For a non-corrupt database file one of the follwing + ** must be true: + ** (1) We found one or more cells (cntNew[0])>0), or + ** (2) pPage is a virtual root page. A virtual root page is when + ** the real root page is page 1 and we are the only child of + ** that page. */ -#if 0 - assert( cntNew[0]>0 || (pParent->pgno==1 && pParent->nCell==0) ); -#endif - + assert( cntNew[0]>0 || (pParent->pgno==1 && pParent->nCell==0) || CORRUPT_DB); TRACE(("BALANCE: old: %d(nc=%d) %d(nc=%d) %d(nc=%d)\n", apOld[0]->pgno, apOld[0]->nCell, nOld>=2 ? apOld[1]->pgno : 0, nOld>=2 ? apOld[1]->nCell : 0, From 00fe08af82c5e66a69abfbf687d9d99d46d8ad47 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 31 Oct 2014 00:05:23 +0000 Subject: [PATCH 051/133] In the balance_nonroot() routine, protect the values in aPgno[] array from change during the page sort, so that aPgno[] can be used to avoid unnecessary pointer-map updates for auto_vacuum databases. FossilOrigin-Name: 69c3924fe834a78d4a8d86833626bf5f68e33a3a --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 9 +++++---- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 98a273f2e0..b3f68e9802 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improvements\sto\sthe\swording\sof\ssome\scomments.\s\sReinstate\san\sassert()\sthat\nis\sonly\strue\sfor\snon-corrupt\sdatabase\sfiles\sby\sadding\san\s"||\sCORRUPT_DB"\sterm. -D 2014-10-30T23:14:56.756 +C In\sthe\sbalance_nonroot()\sroutine,\sprotect\sthe\svalues\sin\saPgno[]\sarray\sfrom\nchange\sduring\sthe\spage\ssort,\sso\sthat\saPgno[]\scan\sbe\sused\sto\savoid\sunnecessary\npointer-map\supdates\sfor\sauto_vacuum\sdatabases. +D 2014-10-31T00:05:23.983 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 7f841396adfd47507ff670a471162d2bfcda3136 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 8f7ea96935c3b1db94439204965a6ec1392f7977 +F src/btree.c 88c87803b334807da3150245987139baacea4e6e F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 67bb05b1077e0cdaccb2e36bfcbe7a5df9ed31e8 @@ -1209,7 +1209,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P c7d9aa3a1ce63e27ec94295601bc89fecf1e4977 -R 230a6673a34b4786200f7ae54e553a63 +P 67adb44838f98805f86aecca634d9a3b07370b9e +R 882446b4b0e231e5d89ec5bd9f22a940 U drh -Z 52f8e66a5796212d2c4dbab1cb7cef14 +Z 021277b745352cb31d525498ae57f1ac diff --git a/manifest.uuid b/manifest.uuid index fd375ba77a..03860ab39c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -67adb44838f98805f86aecca634d9a3b07370b9e \ No newline at end of file +69c3924fe834a78d4a8d86833626bf5f68e33a3a \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 2c3cbb4adc..05dbd8cec7 100644 --- a/src/btree.c +++ b/src/btree.c @@ -6533,6 +6533,7 @@ static int balance_nonroot( Pgno pgno; /* Temp var to store a page number in */ u8 abDone[NB+2]; /* True after i'th new page is populated */ Pgno aPgno[NB+2]; /* Page numbers of new pages before shuffling */ + Pgno aPgOrder[NB+2]; /* Copy of aPgno[] used for sorting pages */ u16 aPgFlags[NB+2]; /* flags field of new pages before shuffling */ memset(abDone, 0, sizeof(abDone)); @@ -6859,7 +6860,7 @@ static int balance_nonroot( ** for large insertions and deletions. */ for(i=0; ipgno; + aPgOrder[i] = aPgno[i] = apNew[i]->pgno; aPgFlags[i] = apNew[i]->pDbPage->flags; for(j=0; ji ){ sqlite3PagerRekey(apNew[iBest]->pDbPage, pBt->nPage+iBest+1, 0); From 768f29002e24301bb601e5eefcf41b1a0904b78f Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 31 Oct 2014 02:51:41 +0000 Subject: [PATCH 052/133] Remove an unnecessary branch from balance_nonroot(). FossilOrigin-Name: 9fc7c88e3f5221883aa6eafbf8af3be94db0c299 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 23 ++++++++++++----------- 3 files changed, 19 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index b3f68e9802..2496dc7f00 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\sbalance_nonroot()\sroutine,\sprotect\sthe\svalues\sin\saPgno[]\sarray\sfrom\nchange\sduring\sthe\spage\ssort,\sso\sthat\saPgno[]\scan\sbe\sused\sto\savoid\sunnecessary\npointer-map\supdates\sfor\sauto_vacuum\sdatabases. -D 2014-10-31T00:05:23.983 +C Remove\san\sunnecessary\sbranch\sfrom\sbalance_nonroot(). +D 2014-10-31T02:51:41.215 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 7f841396adfd47507ff670a471162d2bfcda3136 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 88c87803b334807da3150245987139baacea4e6e +F src/btree.c 7031b8cec28b4ba853090da021c6c456952f1f92 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 67bb05b1077e0cdaccb2e36bfcbe7a5df9ed31e8 @@ -1209,7 +1209,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 67adb44838f98805f86aecca634d9a3b07370b9e -R 882446b4b0e231e5d89ec5bd9f22a940 +P 69c3924fe834a78d4a8d86833626bf5f68e33a3a +R 85b30d88694d9feacf26ae3f80208908 U drh -Z 021277b745352cb31d525498ae57f1ac +Z 7bc5c33b6cb165c738435440761728db diff --git a/manifest.uuid b/manifest.uuid index 03860ab39c..d59874b337 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -69c3924fe834a78d4a8d86833626bf5f68e33a3a \ No newline at end of file +9fc7c88e3f5221883aa6eafbf8af3be94db0c299 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 05dbd8cec7..f21fce2a51 100644 --- a/src/btree.c +++ b/src/btree.c @@ -7078,19 +7078,20 @@ static int balance_nonroot( ** sets all pointer-map entries corresponding to database image pages ** for which the pointer is stored within the content being copied. ** - ** The second assert below verifies that the child page is defragmented - ** (it must be, as it was just reconstructed using assemblePage()). This - ** is important if the parent page happens to be page 1 of the database - ** image. */ + ** It is critical that the child page be defragmented before being + ** copied into the parent, because if the parent is page 1 then it will + ** by smaller than the child due to the database header, and so all the + ** free space needs to be up front. + */ assert( nNew==1 ); rc = defragmentPage(apNew[0]); - if( rc==SQLITE_OK ){ - assert( apNew[0]->nFree == - (get2byte(&apNew[0]->aData[5])-apNew[0]->cellOffset-apNew[0]->nCell*2) - ); - copyNodeContent(apNew[0], pParent, &rc); - freePage(apNew[0], &rc); - } + testcase( rc!=SQLITE_OK ); + assert( apNew[0]->nFree == + (get2byte(&apNew[0]->aData[5])-apNew[0]->cellOffset-apNew[0]->nCell*2) + || rc!=SQLITE_OK + ); + copyNodeContent(apNew[0], pParent, &rc); + freePage(apNew[0], &rc); }else if( ISAUTOVACUUM && !leafCorrection ){ /* Fix the pointer map entries associated with the right-child of each ** sibling page. All other pointer map entries have already been taken From 9c0153457af66ca84768c08472f7424f7e6711d0 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 31 Oct 2014 10:31:59 +0000 Subject: [PATCH 053/133] Add the "varint.c" utility program in the tool directory. FossilOrigin-Name: ea5d56be5fe14934e4dbe9c17d46b058f487a231 --- manifest | 11 +++-- manifest.uuid | 2 +- tool/varint.c | 123 ++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 130 insertions(+), 6 deletions(-) create mode 100644 tool/varint.c diff --git a/manifest b/manifest index 2496dc7f00..2c4f7bf41f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sunnecessary\sbranch\sfrom\sbalance_nonroot(). -D 2014-10-31T02:51:41.215 +C Add\sthe\s"varint.c"\sutility\sprogram\sin\sthe\stool\sdirectory. +D 2014-10-31T10:31:59.052 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1204,12 +1204,13 @@ F tool/stack_usage.tcl f8e71b92cdb099a147dad572375595eae55eca43 F tool/symbols-mingw.sh 4dbcea7e74768305384c9fd2ed2b41bbf9f0414d F tool/symbols.sh fec58532668296d7c7dc48be9c87f75ccdb5814f F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 +F tool/varint.c 5d94cb5003db9dbbcbcc5df08d66f16071aee003 F tool/vdbe-compress.tcl 5926c71f9c12d2ab73ef35c29376e756eb68361c F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 69c3924fe834a78d4a8d86833626bf5f68e33a3a -R 85b30d88694d9feacf26ae3f80208908 +P 9fc7c88e3f5221883aa6eafbf8af3be94db0c299 +R 932c8189456e662bc00860a65a53ebcf U drh -Z 7bc5c33b6cb165c738435440761728db +Z 653e5ed37198e17f610045be3bfdae23 diff --git a/manifest.uuid b/manifest.uuid index d59874b337..337c7347f8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9fc7c88e3f5221883aa6eafbf8af3be94db0c299 \ No newline at end of file +ea5d56be5fe14934e4dbe9c17d46b058f487a231 \ No newline at end of file diff --git a/tool/varint.c b/tool/varint.c new file mode 100644 index 0000000000..f4a51118b4 --- /dev/null +++ b/tool/varint.c @@ -0,0 +1,123 @@ +/* +** A utility program to translate SQLite varints into decimal and decimal +** integers into varints. +*/ +#include +#include +#include + +#if defined(_MSC_VER) || defined(__BORLANDC__) + typedef __int64 i64; + typedef unsigned __int64 u64; +#else + typedef long long int i64; + typedef unsigned long long int u64; +#endif + +static int hexValue(char c){ + if( c>='0' && c<='9' ) return c - '0'; + if( c>='a' && c<='f' ) return c - 'a' + 10; + if( c>='A' && c<='F' ) return c - 'A' + 10; + return -1; +} + +static char toHex(unsigned char c){ + return "0123456789abcdef"[c&0xf]; +} + +static int putVarint(unsigned char *p, u64 v){ + int i, j, n; + unsigned char buf[10]; + if( v & (((u64)0xff000000)<<32) ){ + p[8] = (unsigned char)v; + v >>= 8; + for(i=7; i>=0; i--){ + p[i] = (unsigned char)((v & 0x7f) | 0x80); + v >>= 7; + } + return 9; + } + n = 0; + do{ + buf[n++] = (unsigned char)((v & 0x7f) | 0x80); + v >>= 7; + }while( v!=0 ); + buf[0] &= 0x7f; + for(i=0, j=n-1; j>=0; j--, i++){ + p[i] = buf[j]; + } + return n; +} + + +int main(int argc, char **argv){ + int i; + u64 x; + u64 uX = 0; + i64 iX; + int n; + unsigned char zHex[20]; + + if( argc==1 ){ + fprintf(stderr, + "Usage:\n" + " %s HH HH HH ... Convert varint to decimal\n" + " %s DDDDD Convert decimal to varint\n" + " Add '+' or '-' before DDDDD to disambiguate.\n", + argv[0], argv[0]); + exit(1); + } + if( argc>2 + || (strlen(argv[1])==2 && hexValue(argv[1][0])>=0 && hexValue(argv[1][1])>=0) + ){ + /* Hex to decimal */ + for(i=1; i'9' ){ + fprintf(stderr, "Not a decimal number: %s", argv[1]); + exit(1); + } + uX = uX*10 + z[0] - '0'; + z++; + } + if( sign<0 ){ + memcpy(&iX, &uX, 8); + iX = -iX; + memcpy(&uX, &iX, 8); + } + } + n = putVarint(zHex, uX); + printf("%lld =", (i64)uX); + for(i=0; i>4), toHex(zHex[i]&0x0f)); + } + printf("\n"); + return 0; +} From bec021b9fca398564dd2eccabe10d8af8fa6fc41 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 31 Oct 2014 12:22:00 +0000 Subject: [PATCH 054/133] Simplify the math slightly, and reduce by one the number of loop iterations, for the loop in balance_nonroot() that moves cells between pages. FossilOrigin-Name: 2e838db82e533598b3cb00011c04fc0d5a896895 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 6 ++++-- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 2c4f7bf41f..e6108baa44 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s"varint.c"\sutility\sprogram\sin\sthe\stool\sdirectory. -D 2014-10-31T10:31:59.052 +C Simplify\sthe\smath\sslightly,\sand\sreduce\sby\sone\sthe\snumber\sof\sloop\siterations,\nfor\sthe\sloop\sin\sbalance_nonroot()\sthat\smoves\scells\sbetween\spages. +D 2014-10-31T12:22:00.360 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 7f841396adfd47507ff670a471162d2bfcda3136 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 7031b8cec28b4ba853090da021c6c456952f1f92 +F src/btree.c 0f294d5e1eca7a912037d1992c904e28959a5aa5 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 67bb05b1077e0cdaccb2e36bfcbe7a5df9ed31e8 @@ -1210,7 +1210,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 9fc7c88e3f5221883aa6eafbf8af3be94db0c299 -R 932c8189456e662bc00860a65a53ebcf +P ea5d56be5fe14934e4dbe9c17d46b058f487a231 +R 5dd41c1bf29a3c6eca71e97060df99dc U drh -Z 653e5ed37198e17f610045be3bfdae23 +Z 53accc48044e0d3a03ff14711aee89d8 diff --git a/manifest.uuid b/manifest.uuid index 337c7347f8..000bb51e9a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ea5d56be5fe14934e4dbe9c17d46b058f487a231 \ No newline at end of file +2e838db82e533598b3cb00011c04fc0d5a896895 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index f21fce2a51..4176d2c8bd 100644 --- a/src/btree.c +++ b/src/btree.c @@ -7036,8 +7036,10 @@ static int balance_nonroot( ** ** If neither of the above apply, the page is safe to update. */ - for(i=0; i=nNew ? i-nNew : nNew-1-i); + for(i=1-nNew; i=0 && iPg=cntNew[iPg-1] || abDone[iPg-1]) && (cntNew[iPg]>=cntOld[iPg] || abDone[iPg+1]) From d836d42383d93c549826510747b0b6a1bb55c55b Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 31 Oct 2014 14:26:36 +0000 Subject: [PATCH 055/133] Simplify the logic in the cell redistribution loop of balance_nonroot(). Enhance and clarify comments and add assert() statements for additional verification of correctness. FossilOrigin-Name: a07078b60007e88adea67bec5f0caf91f707ad78 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 39 ++++++++++++++++++++++++++++----------- 3 files changed, 35 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index e6108baa44..5aa7ed2f46 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Simplify\sthe\smath\sslightly,\sand\sreduce\sby\sone\sthe\snumber\sof\sloop\siterations,\nfor\sthe\sloop\sin\sbalance_nonroot()\sthat\smoves\scells\sbetween\spages. -D 2014-10-31T12:22:00.360 +C Simplify\sthe\slogic\sin\sthe\scell\sredistribution\sloop\sof\sbalance_nonroot().\nEnhance\sand\sclarify\scomments\sand\sadd\sassert()\sstatements\sfor\sadditional\nverification\sof\scorrectness. +D 2014-10-31T14:26:36.417 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 7f841396adfd47507ff670a471162d2bfcda3136 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 0f294d5e1eca7a912037d1992c904e28959a5aa5 +F src/btree.c 61d96c2edacc5267fae6e477c24c774cd540d7f0 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 67bb05b1077e0cdaccb2e36bfcbe7a5df9ed31e8 @@ -1210,7 +1210,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ea5d56be5fe14934e4dbe9c17d46b058f487a231 -R 5dd41c1bf29a3c6eca71e97060df99dc +P 2e838db82e533598b3cb00011c04fc0d5a896895 +R 6da135d51be6a673f0389acc80c0dfb3 U drh -Z 53accc48044e0d3a03ff14711aee89d8 +Z 452523febd435385cb08d146713a8a8f diff --git a/manifest.uuid b/manifest.uuid index 000bb51e9a..4ff01299eb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2e838db82e533598b3cb00011c04fc0d5a896895 \ No newline at end of file +a07078b60007e88adea67bec5f0caf91f707ad78 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 4176d2c8bd..9300a6a54f 100644 --- a/src/btree.c +++ b/src/btree.c @@ -7026,28 +7026,43 @@ static int balance_nonroot( ** is important, as this code needs to avoid disrupting any page from which ** cells may still to be read. In practice, this means: ** - ** 1) If cells are to be removed from the start of the page and shifted - ** to the left-hand sibling, it is not safe to update the page until - ** the left-hand sibling (apNew[i-1]) has already been updated. + ** (1) If cells are moving left (from apNew[iPg] to apNew[iPg-1]) + ** then it is not safe to update page apNew[iPg] until after + ** the left-hand sibling apNew[iPg-1] has been updated. ** - ** 2) If cells are to be removed from the end of the page and shifted - ** to the right-hand sibling, it is not safe to update the page until - ** the right-hand sibling (apNew[i+1]) has already been updated. + ** (2) If cells are moving right (from apNew[iPg] to apNew[iPg+1]) + ** then it is not safe to update page apNew[iPg] until after + ** the right-hand sibling apNew[iPg+1] has been updated. ** ** If neither of the above apply, the page is safe to update. + ** + ** The iPg value in the following loop starts at nNew-1 goes down + ** to 0, then back up to nNew-1 again, thus making two passes over + ** the pages. On the initial downward pass, only condition (1) above + ** needs to be tested because (2) will always be true from the previous + ** step. On the upward pass, both conditions are always true, so the + ** upwards pass simply processes pages that were missed on the downward + ** pass. */ for(i=1-nNew; i=0 && iPg=cntNew[iPg-1] || abDone[iPg-1]) - && (cntNew[iPg]>=cntOld[iPg] || abDone[iPg+1]) + if( abDone[iPg] ) continue; /* Skip pages already processed */ + if( i>=0 /* On the upwards pass, or... */ + || cntOld[iPg-1]>=cntNew[iPg-1] /* Condition (1) is true */ ){ int iNew; int iOld; int nNewCell; + /* Verify condition (1): If cells are moving left, update iPg + ** only after iPg-1 has already been updated. */ + assert( iPg==0 || cntOld[iPg-1]>=cntNew[iPg-1] || abDone[iPg-1] ); + + /* Verify condition (2): If cells are moving right, update iPg + ** only after iPg+1 has already been updated. */ + assert( cntNew[iPg]>=cntOld[iPg] || abDone[iPg+1] ); + if( iPg==0 ){ iNew = iOld = 0; nNewCell = cntNew[0]; @@ -7058,12 +7073,14 @@ static int balance_nonroot( } editPage(apNew[iPg], iOld, iNew, nNewCell, apCell, szCell); - abDone[iPg] = 1; + abDone[iPg]++; apNew[iPg]->nFree = usableSpace-szNew[iPg]; assert( apNew[iPg]->nOverflow==0 ); assert( apNew[iPg]->nCell==nNewCell ); } } + + /* All pages have been processed exactly once */ assert( memcmp(abDone, "\01\01\01\01\01", nNew)==0 ); assert( nOld>0 ); From 0fb5daed34ba6f16d761b8ad02b944ec59cf5c03 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 31 Oct 2014 14:46:51 +0000 Subject: [PATCH 056/133] Change the command-line shell man-page to use the ".tr" troff directive instead of ".cc" for escaping the initial "." characters in the ".help" output. FossilOrigin-Name: 67f0d469da28c023200239a1f3d0c6cef9ef0e45 --- manifest | 12 ++++---- manifest.uuid | 2 +- sqlite3.1 | 83 ++++++++++++++++++++++++++++----------------------- 3 files changed, 52 insertions(+), 45 deletions(-) diff --git a/manifest b/manifest index 5aa7ed2f46..ce0614fbee 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Simplify\sthe\slogic\sin\sthe\scell\sredistribution\sloop\sof\sbalance_nonroot().\nEnhance\sand\sclarify\scomments\sand\sadd\sassert()\sstatements\sfor\sadditional\nverification\sof\scorrectness. -D 2014-10-31T14:26:36.417 +C Change\sthe\scommand-line\sshell\sman-page\sto\suse\sthe\s".tr"\stroff\sdirective\ninstead\sof\s".cc"\sfor\sescaping\sthe\sinitial\s"."\scharacters\sin\sthe\s".help"\noutput. +D 2014-10-31T14:46:51.896 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -163,7 +163,7 @@ F mptest/mptest.c 499a74af4be293b7c1c7c3d40f332b67227dd739 F mptest/multiwrite01.test 499ad0310da8dff8e8f98d2e272fc2a8aa741b2e F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b -F sqlite3.1 3d8b83c91651f53472ca17599dae3457b8b89494 +F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c ba266a779bc7ce10e52e59e7d3dc79fa342e8fdb F src/analyze.c afbcca663c3f3625340b8e30d440cd7a97ded6bc @@ -1210,7 +1210,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 2e838db82e533598b3cb00011c04fc0d5a896895 -R 6da135d51be6a673f0389acc80c0dfb3 +P a07078b60007e88adea67bec5f0caf91f707ad78 +R f904ce78baced6aeb6bcb91c631714db U drh -Z 452523febd435385cb08d146713a8a8f +Z c463256d5d45e3bce4a212ca4dd9f3e4 diff --git a/manifest.uuid b/manifest.uuid index 4ff01299eb..4735237141 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a07078b60007e88adea67bec5f0caf91f707ad78 \ No newline at end of file +67f0d469da28c023200239a1f3d0c6cef9ef0e45 \ No newline at end of file diff --git a/sqlite3.1 b/sqlite3.1 index 02df7f44c7..80353b0eec 100644 --- a/sqlite3.1 +++ b/sqlite3.1 @@ -2,7 +2,7 @@ .\" First parameter, NAME, should be all caps .\" Second parameter, SECTION, should be 1-8, maybe w/ subsection .\" other parameters are allowed: see man(7), man(1) -.TH SQLITE3 1 "Mon Jan 31 11:14:00 2014" +.TH SQLITE3 1 "Fri Oct 31 10:41:31 EDT 2014" .\" Please adjust this date whenever revising the manpage. .\" .\" Some roff macros, for reference: @@ -49,7 +49,7 @@ a table named "memos" and insert a couple of records into that table: $ .B sqlite3 mydata.db .br -SQLite version 3.8.3 +SQLite version 3.8.8 .br Enter ".help" for instructions .br @@ -107,26 +107,29 @@ the '.help' command. For example: sqlite> .B .help .nf -.cc | -.backup ?DB? FILE Backup DB (default "main") to FILE -.bail ON|OFF Stop after hitting an error. Default OFF -.databases List names and files of attached databases -.dump ?TABLE? ... Dump the database in an SQL text format +.tr %. +%backup ?DB? FILE Backup DB (default "main") to FILE +%bail on|off Stop after hitting an error. Default OFF +%clone NEWDB Clone data into NEWDB from the existing database +%databases List names and files of attached databases +%dump ?TABLE? ... Dump the database in an SQL text format If TABLE specified, only dump tables matching LIKE pattern TABLE. -.echo ON|OFF Turn command echo on or off -.exit Exit this program -.explain ?ON|OFF? Turn output mode suitable for EXPLAIN on or off. +%echo on|off Turn command echo on or off +%eqp on|off Enable or disable automatic EXPLAIN QUERY PLAN +%exit Exit this program +%explain ?on|off? Turn output mode suitable for EXPLAIN on or off. With no args, it turns EXPLAIN on. -.header(s) ON|OFF Turn display of headers on or off -.help Show this message -.import FILE TABLE Import data from FILE into TABLE -.indices ?TABLE? Show names of all indices +%fullschema Show schema and the content of sqlite_stat tables +%headers on|off Turn display of headers on or off +%help Show this message +%import FILE TABLE Import data from FILE into TABLE +%indices ?TABLE? Show names of all indices If TABLE specified, only show indices for tables matching LIKE pattern TABLE. -.load FILE ?ENTRY? Load an extension library -.log FILE|off Turn logging on or off. FILE can be stderr/stdout -.mode MODE ?TABLE? Set output mode where MODE is one of: +%load FILE ?ENTRY? Load an extension library +%log FILE|off Turn logging on or off. FILE can be stderr/stdout +%mode MODE ?TABLE? Set output mode where MODE is one of: csv Comma-separated values column Left-aligned columns. (See .width) html HTML code @@ -135,31 +138,35 @@ sqlite> list Values delimited by .separator string tabs Tab-separated values tcl TCL list elements -.nullvalue STRING Use STRING in place of NULL values -.open ?FILENAME? Close existing database and reopen FILENAME -.output FILENAME Send output to FILENAME -.output stdout Send output to the screen -.print STRING... Print literal STRING -.prompt MAIN CONTINUE Replace the standard prompts -.quit Exit this program -.read FILENAME Execute SQL in FILENAME -.restore ?DB? FILE Restore content of DB (default "main") from FILE -.schema ?TABLE? Show the CREATE statements +%nullvalue STRING Use STRING in place of NULL values +%once FILENAME Output for the next SQL command only to FILENAME +%open ?FILENAME? Close existing database and reopen FILENAME +%output ?FILENAME? Send output to FILENAME or stdout +%print STRING... Print literal STRING +%prompt MAIN CONTINUE Replace the standard prompts +%quit Exit this program +%read FILENAME Execute SQL in FILENAME +%restore ?DB? FILE Restore content of DB (default "main") from FILE +%save FILE Write in-memory database into FILE +%schema ?TABLE? Show the CREATE statements If TABLE specified, only show tables matching LIKE pattern TABLE. -.separator STRING Change separator used by output mode and .import -.show Show the current values for various settings -.stats ON|OFF Turn stats on or off -.tables ?TABLE? List names of tables +%separator STRING ?NL? Change separator used by output mode and .import + NL is the end-of-line mark for CSV +%shell CMD ARGS... Run CMD ARGS... in a system shell +%show Show the current values for various settings +%stats on|off Turn stats on or off +%system CMD ARGS... Run CMD ARGS... in a system shell +%tables ?TABLE? List names of tables If TABLE specified, only list tables matching LIKE pattern TABLE. -.timeout MS Try opening locked tables for MS milliseconds -.trace FILE|off Output each SQL statement as it is run -.vfsname ?AUX? Print the name of the VFS stack -.width NUM1 NUM2 ... Set column widths for "column" mode -.timer ON|OFF Turn the CPU timer measurement on or off +%timeout MS Try opening locked tables for MS milliseconds +%timer on|off Turn SQL timer on or off +%trace FILE|off Output each SQL statement as it is run +%vfsname ?AUX? Print the name of the VFS stack +%width NUM1 NUM2 ... Set column widths for "column" mode + Negative values right-justify sqlite> -|cc . .sp .fi .SH OPTIONS @@ -269,7 +276,7 @@ o If the -init option is present, the specified file is processed. o All other command line options are processed. .SH SEE ALSO -http://www.sqlite.org/ +http://www.sqlite.org/cli.html .br The sqlite3-doc package. .SH AUTHOR From 04489b6dce1b28fffb9afa524b0eb854a400da21 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 31 Oct 2014 20:11:32 +0000 Subject: [PATCH 057/133] Add the experimental sqlite3_stmt_scanstatus() API. FossilOrigin-Name: 6a9bab34aeb6a01b612211a28c140de60a3e883c --- manifest | 34 ++++++++++++--------- manifest.uuid | 2 +- src/sqlite.h.in | 36 ++++++++++++++++++++++ src/tclsqlite.c | 42 +++++++++++++++++++++++++ src/test1.c | 73 ++++++++++++++++++++++++++++++++++++++++++++ src/vdbe.c | 44 ++++++++++++++++++++++++-- src/vdbe.h | 16 ++++++++++ src/vdbeInt.h | 4 +++ src/vdbeapi.c | 37 ++++++++++++++++++++++ src/vdbeaux.c | 13 ++++++++ src/where.c | 55 +++++++++++++++++++++++++-------- test/scanstatus.test | 69 +++++++++++++++++++++++++++++++++++++++++ 12 files changed, 394 insertions(+), 31 deletions(-) create mode 100644 test/scanstatus.test diff --git a/manifest b/manifest index ce0614fbee..39eaf23ca4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\scommand-line\sshell\sman-page\sto\suse\sthe\s".tr"\stroff\sdirective\ninstead\sof\s".cc"\sfor\sescaping\sthe\sinitial\s"."\scharacters\sin\sthe\s".help"\noutput. -D 2014-10-31T14:46:51.896 +C Add\sthe\sexperimental\ssqlite3_stmt_scanstatus()\sAPI. +D 2014-10-31T20:11:32.562 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -229,15 +229,15 @@ F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c 282f8f5278e0c78eb442217531172ec9e1538796 -F src/sqlite.h.in 737b7dd0f3f81fe183646d22828b39f85ef3e68c +F src/sqlite.h.in 6e90cdb404e4fa8c0eb149ca79c11c6608a97ece F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 4f86ac648ea398c1bb3db036062934cde257ea23 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 81712116e826b0089bb221b018929536b2b5406f F src/table.c f142bba7903e93ca8d113a5b8877a108ad1a27dc -F src/tclsqlite.c c67d310c833046cccc192125d64ad422ab882684 -F src/test1.c 63d4b1707c4052cf9c05c1cbb4a62666d70a0b48 +F src/tclsqlite.c 7cdd4dd3c2a4183483feca260070d73d6e22cd47 +F src/test1.c b53f4da2f386efa5c248716cd4bdc8344a87e952 F src/test2.c 98049e51a17dc62606a99a9eb95ee477f9996712 F src/test3.c 1c0e5d6f080b8e33c1ce8b3078e7013fdbcd560c F src/test4.c 9b32d22f5f150abe23c1830e2057c4037c45b3df @@ -289,11 +289,11 @@ F src/update.c 3c4ecc282accf12d39edb8d524cf089645e55a13 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 3b627daa45c7308c1e36e3dbaa3f9ce7e5c7fa73 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c 49e659bc165e99b28492004b440e22146dc898ab -F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 -F src/vdbeInt.h acc36ac461f973f46ac7942f86abdd93d2f8cfbc -F src/vdbeapi.c 02d8afcff710eb35e3d9e49cb677308296b00009 -F src/vdbeaux.c 3d6b2b412ef2193aa4729922dfc5df1efadbf6df +F src/vdbe.c 80329be857cff3c2d6e8c1f594ac689166c44ae7 +F src/vdbe.h f8e5388173dbf8408da4ae1dcbf75911caf2253d +F src/vdbeInt.h 284b2294c188474daa9b5ce2bd0200560cb0940d +F src/vdbeapi.c 2afa2e162f290879ca9c51e8795a377035f1662a +F src/vdbeaux.c 1e2561eeb94749f6b8c0a1eb02b5fecb645ed48c F src/vdbeblob.c 8b5442ff0954c44b45cbabbe2e94091a2e16fdef F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f F src/vdbesort.c 975aeffa99acb0991b2f288d30294756bff41438 @@ -302,7 +302,7 @@ F src/vtab.c 2a30791bbd7926b589401bd09c3abb33de563793 F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 5665df88cbd2b38eb72b4b94c8892c8afb360181 +F src/where.c 344beac082f8f4f69ec7c4a669d74294475e5abe F src/whereInt.h 19279cd0664ce1d90b9ad3ef0108cb494acfe455 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -801,6 +801,7 @@ F test/savepoint4.test c8f8159ade6d2acd9128be61e1230f1c1edc6cc0 F test/savepoint5.test 0735db177e0ebbaedc39812c8d065075d563c4fd F test/savepoint6.test f41279c5e137139fa5c21485773332c7adb98cd7 F test/savepoint7.test fbf319a7b2dda089ec5be30a424a0e95f121d423 +F test/scanstatus.test a62dad3311fc51eab1d6977b082738bd2148fe07 F test/schema.test 8f7999be894260f151adf15c2c7540f1c6d6a481 F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5 F test/schema3.test 1bc1008e1f8cb5654b248c55f27249366eb7ed38 @@ -1210,7 +1211,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P a07078b60007e88adea67bec5f0caf91f707ad78 -R f904ce78baced6aeb6bcb91c631714db -U drh -Z c463256d5d45e3bce4a212ca4dd9f3e4 +P 67f0d469da28c023200239a1f3d0c6cef9ef0e45 +R 4c76f3e7b97f67c1dd65bda3a6e96149 +T *branch * scanstatus +T *sym-scanstatus * +T -sym-trunk * +U dan +Z 4994ca943ee260eeca8f493b4f59766a diff --git a/manifest.uuid b/manifest.uuid index 4735237141..b7984cac38 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -67f0d469da28c023200239a1f3d0c6cef9ef0e45 \ No newline at end of file +6a9bab34aeb6a01b612211a28c140de60a3e883c \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index b4081f2a02..ba20cb9890 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -7407,6 +7407,42 @@ int sqlite3_vtab_on_conflict(sqlite3 *); #define SQLITE_REPLACE 5 +/* +** Return status data for a single loop within query pStmt. +** +** Parameter "idx" identifies the specific loop to retrieve statistics for. +** Loops are numbered starting from zero. If idx is out of range - less than +** zero or greater than or equal to the total number of loops used to implement +** the statement - a non-zero value is returned. In this case the final value +** of all five output parameters is undefined. Otherwise, if idx is in range, +** the output parameters are populated and zero returned. +** +** Statistics may not be available for all loops in all statements. In cases +** where there exist loops with no available statistics, this function ignores +** them completely. +** +** This API is only available if the library is built with pre-processor +** symbol SQLITE_ENABLE_STMT_SCANSTATUS defined. +*/ +SQLITE_EXPERIMENTAL int sqlite3_stmt_scanstatus( + sqlite3_stmt *pStmt, + int idx, /* Index of loop to report on */ + sqlite3_int64 *pnLoop, /* OUT: Number of times loop was run */ + sqlite3_int64 *pnVisit, /* OUT: Number of rows visited (all loops) */ + sqlite3_int64 *pnEst, /* OUT: Number of rows estimated (per loop) */ + const char **pzName, /* OUT: Object name (table or index) */ + const char **pzExplain /* OUT: EQP string */ +); + + +/* +** Zero all sqlite3_stmt_scanstatus() related event counters. +** +** This API is only available if the library is built with pre-processor +** symbol SQLITE_ENABLE_STMT_SCANSTATUS defined. +*/ +SQLITE_EXPERIMENTAL void sqlite3_stmt_scanstatus_reset(sqlite3_stmt*); + /* ** Undo the hack that converts floating point types to integer for diff --git a/src/tclsqlite.c b/src/tclsqlite.c index 756d0daa5a..bff4a92421 100644 --- a/src/tclsqlite.c +++ b/src/tclsqlite.c @@ -3641,6 +3641,45 @@ static int db_use_legacy_prepare_cmd( Tcl_ResetResult(interp); return TCL_OK; } + +/* +** Tclcmd: db_last_stmt_ptr DB +** +** If the statement cache associated with database DB is not empty, +** return the text representation of the most recently used statement +** handle. +*/ +static int db_last_stmt_ptr( + ClientData cd, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + extern int sqlite3TestMakePointerStr(Tcl_Interp*, char*, void*); + Tcl_CmdInfo cmdInfo; + SqliteDb *pDb; + sqlite3_stmt *pStmt = 0; + char zBuf[100]; + + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB"); + return TCL_ERROR; + } + + if( !Tcl_GetCommandInfo(interp, Tcl_GetString(objv[1]), &cmdInfo) ){ + Tcl_AppendResult(interp, "no such db: ", Tcl_GetString(objv[1]), (char*)0); + return TCL_ERROR; + } + pDb = (SqliteDb*)cmdInfo.objClientData; + + if( pDb->stmtList ) pStmt = pDb->stmtList->pStmt; + if( sqlite3TestMakePointerStr(interp, zBuf, pStmt) ){ + return TCL_ERROR; + } + Tcl_SetResult(interp, zBuf, TCL_VOLATILE); + + return TCL_OK; +} #endif /* @@ -3760,6 +3799,9 @@ static void init_all(Tcl_Interp *interp){ Tcl_CreateObjCommand( interp, "db_use_legacy_prepare", db_use_legacy_prepare_cmd, 0, 0 ); + Tcl_CreateObjCommand( + interp, "db_last_stmt_ptr", db_last_stmt_ptr, 0, 0 + ); #ifdef SQLITE_SSE Sqlitetestsse_Init(interp); diff --git a/src/test1.c b/src/test1.c index 802aa99b5b..72e70513bc 100644 --- a/src/test1.c +++ b/src/test1.c @@ -2301,6 +2301,75 @@ static int test_stmt_status( return TCL_OK; } +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS +/* +** Usage: sqlite3_stmt_scanstatus STMT IDX +*/ +static int test_stmt_scanstatus( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + sqlite3_stmt *pStmt; /* First argument */ + int idx; /* Second argument */ + + const char *zName; + const char *zExplain; + sqlite3_int64 nLoop; + sqlite3_int64 nVisit; + sqlite3_int64 nEst; + int res; + + if( objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "STMT IDX"); + return TCL_ERROR; + } + if( getStmtPointer(interp, Tcl_GetString(objv[1]), &pStmt) ) return TCL_ERROR; + if( Tcl_GetIntFromObj(interp, objv[2], &idx) ) return TCL_ERROR; + + res = sqlite3_stmt_scanstatus( + pStmt, idx, &nLoop, &nVisit, &nEst, &zName, &zExplain + ); + if( res==0 ){ + Tcl_Obj *pRet = Tcl_NewObj(); + Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj("nLoop", -1)); + Tcl_ListObjAppendElement(0, pRet, Tcl_NewWideIntObj(nLoop)); + Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj("nVisit", -1)); + Tcl_ListObjAppendElement(0, pRet, Tcl_NewWideIntObj(nVisit)); + Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj("nEst", -1)); + Tcl_ListObjAppendElement(0, pRet, Tcl_NewWideIntObj(nEst)); + Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj("zName", -1)); + Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zName, -1)); + Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj("zExplain", -1)); + Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zExplain, -1)); + Tcl_SetObjResult(interp, pRet); + }else{ + Tcl_ResetResult(interp); + } + return TCL_OK; +} + +/* +** Usage: sqlite3_stmt_scanstatus_reset STMT +*/ +static int test_stmt_scanstatus_reset( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + sqlite3_stmt *pStmt; /* First argument */ + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "STMT"); + return TCL_ERROR; + } + if( getStmtPointer(interp, Tcl_GetString(objv[1]), &pStmt) ) return TCL_ERROR; + sqlite3_stmt_scanstatus_reset(pStmt); + return TCL_OK; +} +#endif + /* ** Usage: sqlite3_next_stmt DB STMT ** @@ -6868,6 +6937,10 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ { "sqlite3_user_change", test_user_change, 0 }, { "sqlite3_user_delete", test_user_delete, 0 }, #endif +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + { "sqlite3_stmt_scanstatus", test_stmt_scanstatus, 0 }, + { "sqlite3_stmt_scanstatus_reset", test_stmt_scanstatus_reset, 0 }, +#endif }; static int bitmask_size = sizeof(Bitmask)*8; diff --git a/src/vdbe.c b/src/vdbe.c index c17bfdfe14..a92c0509b3 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -167,6 +167,18 @@ int sqlite3_found_count = 0; /* Return true if the cursor was opened using the OP_OpenSorter opcode. */ #define isSorter(x) ((x)->pSorter!=0) +/* +** The first argument passed to the IncrementExplainCounter() macro must +** be a non-NULL pointer to an object of type VdbeCursor. The second +** argument must be either "nLoop" or "nVisit" (without the double-quotes). +*/ +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS +# define IncrementExplainCounter(pCsr, counter) \ + if( (pCsr)->pExplain ) (pCsr)->pExplain->counter++ +#else +# define IncrementExplainCounter(pCsr, counter) +#endif + /* ** Allocate VdbeCursor number iCur. Return a pointer to it. Return NULL ** if we run out of memory. @@ -3556,6 +3568,7 @@ case OP_SeekGT: { /* jump, in3 */ #ifdef SQLITE_DEBUG pC->seekOp = pOp->opcode; #endif + IncrementExplainCounter(pC, nLoop); if( pC->isTable ){ /* The input value in P3 might be of any type: integer, real, string, ** blob, or NULL. But it needs to be an integer before we can do @@ -3664,6 +3677,8 @@ case OP_SeekGT: { /* jump, in3 */ VdbeBranchTaken(res!=0,2); if( res ){ pc = pOp->p2 - 1; + }else{ + IncrementExplainCounter(pC, nVisit); } break; } @@ -4449,6 +4464,8 @@ case OP_Last: { /* jump */ res = 0; assert( pCrsr!=0 ); rc = sqlite3BtreeLast(pCrsr, &res); + IncrementExplainCounter(pC, nLoop); + if( res==0 ) IncrementExplainCounter(pC, nVisit); pC->nullRow = (u8)res; pC->deferredMoveto = 0; pC->cacheStatus = CACHE_STALE; @@ -4488,9 +4505,9 @@ case OP_Sort: { /* jump */ ** ** The next use of the Rowid or Column or Next instruction for P1 ** will refer to the first entry in the database table or index. -** If the table or index is empty and P2>0, then jump immediately to P2. -** If P2 is 0 or if the table or index is not empty, fall through -** to the following instruction. +** If the table or index is empty, jump immediately to P2. +** If the table or index is not empty, fall through to the following +** instruction. ** ** This opcode leaves the cursor configured to move in forward order, ** from the beginning toward the end. In other words, the cursor is @@ -4518,11 +4535,14 @@ case OP_Rewind: { /* jump */ pC->deferredMoveto = 0; pC->cacheStatus = CACHE_STALE; } + IncrementExplainCounter(pC, nLoop); pC->nullRow = (u8)res; assert( pOp->p2>0 && pOp->p2nOp ); VdbeBranchTaken(res!=0,2); if( res ){ pc = pOp->p2 - 1; + }else{ + IncrementExplainCounter(pC, nVisit); } break; } @@ -4633,6 +4653,7 @@ next_tail: pC->cacheStatus = CACHE_STALE; VdbeBranchTaken(res==0,2); if( res==0 ){ + IncrementExplainCounter(pC, nVisit); pC->nullRow = 0; pc = pOp->p2 - 1; p->aCounter[pOp->p5]++; @@ -6055,7 +6076,10 @@ case OP_VFilter: { /* jump */ VdbeBranchTaken(res!=0,2); if( res ){ pc = pOp->p2 - 1; + }else{ + IncrementExplainCounter(pCur, nVisit); } + IncrementExplainCounter(pCur, nLoop); } pCur->nullRow = 0; @@ -6148,6 +6172,7 @@ case OP_VNext: { /* jump */ if( !res ){ /* If there is data, jump to P2 */ pc = pOp->p2 - 1; + IncrementExplainCounter(pCur, nVisit); } goto check_for_interrupt; } @@ -6347,6 +6372,19 @@ case OP_Init: { /* jump */ break; } +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS +case OP_Explain: { + ExplainArg *pArg; + VdbeCursor *pCur; + if( pOp->p4type==P4_EXPLAIN && (pArg = pOp->p4.pExplain)->iCsr>=0 ){ + pArg = pOp->p4.pExplain; + pCur = p->apCsr[pArg->iCsr]; + pCur->pExplain = pArg; + } + break; +} +#endif + /* Opcode: Noop * * * * * ** diff --git a/src/vdbe.h b/src/vdbe.h index f975f95543..1b470441be 100644 --- a/src/vdbe.h +++ b/src/vdbe.h @@ -25,6 +25,7 @@ ** of this structure. */ typedef struct Vdbe Vdbe; +typedef struct ExplainArg ExplainArg; /* ** The names of the following types declared in vdbeInt.h are required @@ -59,6 +60,7 @@ struct VdbeOp { KeyInfo *pKeyInfo; /* Used when p4type is P4_KEYINFO */ int *ai; /* Used when p4type is P4_INTARRAY */ SubProgram *pProgram; /* Used when p4type is P4_SUBPROGRAM */ + ExplainArg *pExplain; /* Used when p4type is P4_EXPLAIN */ int (*xAdvance)(BtCursor *, int *); } p4; #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS @@ -100,6 +102,19 @@ struct VdbeOpList { }; typedef struct VdbeOpList VdbeOpList; +/* +** Structure used as the P4 parameter for OP_Explain opcodes. +*/ +struct ExplainArg { + int iCsr; /* Cursor number this applies to */ + i64 nLoop; /* Number of times loop has run */ + i64 nVisit; /* Total number of rows visited */ + i64 nEst; /* Estimated number of rows per scan */ + const char *zName; /* Name of table/index being scanned */ + const char *zExplain; /* EQP text for this loop */ +}; + + /* ** Allowed values of VdbeOp.p4type */ @@ -119,6 +134,7 @@ typedef struct VdbeOpList VdbeOpList; #define P4_INTARRAY (-15) /* P4 is a vector of 32-bit integers */ #define P4_SUBPROGRAM (-18) /* P4 is a pointer to a SubProgram structure */ #define P4_ADVANCE (-19) /* P4 is a pointer to BtreeNext() or BtreePrev() */ +#define P4_EXPLAIN (-20) /* P4 is a pointer to an instance of ExplainArg */ /* Error message codes for OP_Halt */ #define P5_ConstraintNotNull 1 diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 623c5fdde8..839108f6bc 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -83,6 +83,7 @@ struct VdbeCursor { i64 seqCount; /* Sequence counter */ i64 movetoTarget; /* Argument to the deferred sqlite3BtreeMoveto() */ VdbeSorter *pSorter; /* Sorter object for OP_SorterOpen cursors */ + ExplainArg *pExplain; /* Object to store seek/visit counts (may be NULL) */ /* Cached information about the header for the data record that the ** cursor is currently pointing to. Only valid if cacheStatus matches @@ -291,6 +292,7 @@ struct Explain { char zBase[100]; /* Initial space */ }; + /* A bitfield type for use inside of structures. Always follow with :N where ** N is the number of bits. */ @@ -368,6 +370,8 @@ struct Vdbe { int nOnceFlag; /* Size of array aOnceFlag[] */ u8 *aOnceFlag; /* Flags for OP_Once */ AuxData *pAuxData; /* Linked list of auxdata allocations */ + ExplainArg **apExplain; /* Array of pointers to P4_EXPLAIN p4 values */ + int nExplain; /* Number of entries in array apExplain */ }; /* diff --git a/src/vdbeapi.c b/src/vdbeapi.c index daf7b101d2..b09faa1fa5 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -1475,3 +1475,40 @@ int sqlite3_stmt_status(sqlite3_stmt *pStmt, int op, int resetFlag){ if( resetFlag ) pVdbe->aCounter[op] = 0; return (int)v; } + +/* +** Return status data for a single loop within query pStmt. +*/ +int sqlite3_stmt_scanstatus( + sqlite3_stmt *pStmt, + int idx, /* Index of loop to report on */ + sqlite3_int64 *pnLoop, /* OUT: Number of times loop was run */ + sqlite3_int64 *pnVisit, /* OUT: Number of rows visited (all loops) */ + sqlite3_int64 *pnEst, /* OUT: Number of rows estimated (per loop) */ + const char **pzName, /* OUT: Object name (table or index) */ + const char **pzExplain /* OUT: EQP string */ +){ + Vdbe *p = (Vdbe*)pStmt; + ExplainArg *pExplain; + if( idx<0 || idx>=p->nExplain ) return 1; + pExplain = p->apExplain[idx]; + if( pnLoop ) *pnLoop = pExplain->nLoop; + if( pnVisit ) *pnVisit = pExplain->nVisit; + if( pnEst ) *pnEst = pExplain->nEst; + if( *pzName ) *pzName = pExplain->zName; + if( *pzExplain ) *pzExplain = pExplain->zExplain; + return 0; +} + +/* +** Zero all counters associated with the sqlite3_stmt_scanstatus() data. +*/ +void sqlite3_stmt_scanstatus_reset(sqlite3_stmt *pStmt){ + Vdbe *p = (Vdbe*)pStmt; + int i; + for(i=0; inExplain; i++){ + p->apExplain[i]->nLoop = 0; + p->apExplain[i]->nVisit = 0; + } +} + diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 7dfb64130d..a8223890bd 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -672,6 +672,7 @@ static void freeP4(sqlite3 *db, int p4type, void *p4){ if( p4 ){ assert( db ); switch( p4type ){ + case P4_EXPLAIN: case P4_REAL: case P4_INT64: case P4_DYNAMIC: @@ -820,6 +821,13 @@ void sqlite3VdbeChangeP4(Vdbe *p, int addr, const char *zP4, int n){ pOp->p4type = P4_VTAB; sqlite3VtabLock((VTable *)zP4); assert( ((VTable *)zP4)->db==p->db ); + }else if( n==P4_EXPLAIN ){ + pOp->p4.p = (void*)zP4; + pOp->p4type = P4_EXPLAIN; + p->apExplain = (ExplainArg**)sqlite3DbReallocOrFree( + p->db, p->apExplain, sizeof(ExplainArg*) * p->nExplain + 1 + ); + if( p->apExplain ) p->apExplain[p->nExplain++] = (ExplainArg*)zP4; }else if( n<0 ){ pOp->p4.p = (void*)zP4; pOp->p4type = (signed char)n; @@ -1103,6 +1111,10 @@ static char *displayP4(Op *pOp, char *zTemp, int nTemp){ zTemp[0] = 0; break; } + case P4_EXPLAIN: { + zP4 = (char*)pOp->p4.pExplain->zExplain; + break; + } default: { zP4 = pOp->p4.z; if( zP4==0 ){ @@ -2685,6 +2697,7 @@ void sqlite3VdbeClearObject(sqlite3 *db, Vdbe *p){ sqlite3DbFree(db, p->aColName); sqlite3DbFree(db, p->zSql); sqlite3DbFree(db, p->pFree); + sqlite3DbFree(db, p->apExplain); } /* diff --git a/src/where.c b/src/where.c index feccf2d11d..3565327b52 100644 --- a/src/where.c +++ b/src/where.c @@ -2820,7 +2820,7 @@ static void explainOneScan( int iFrom, /* Value for "from" column of output */ u16 wctrlFlags /* Flags passed to sqlite3WhereBegin() */ ){ -#ifndef SQLITE_DEBUG +#if !defined(SQLITE_DEBUG) && !defined(SQLITE_ENABLE_STMT_SCANSTATUS) if( pParse->explain==2 ) #endif { @@ -2831,20 +2831,38 @@ static void explainOneScan( int isSearch; /* True for a SEARCH. False for SCAN. */ WhereLoop *pLoop; /* The controlling WhereLoop object */ u32 flags; /* Flags that describe this loop */ - char *zMsg; /* Text to add to EQP output */ StrAccum str; /* EQP output string */ char zBuf[100]; /* Initial space for EQP output string */ + const char *zObj; + ExplainArg *pExplain; + i64 nEstRow; /* Estimated rows per scan of pLevel */ pLoop = pLevel->pWLoop; flags = pLoop->wsFlags; if( (flags&WHERE_MULTI_OR) || (wctrlFlags&WHERE_ONETABLE_ONLY) ) return; + sqlite3StrAccumInit(&str, zBuf, sizeof(zBuf), SQLITE_MAX_LENGTH); + str.db = db; + + /* Reserve space at the start of the buffer managed by the StrAccum + ** object for *pExplain. */ + assert( sizeof(*pExplain)<=sizeof(zBuf) ); + str.nChar = sizeof(*pExplain); + + /* Append the object (table or index) name to the buffer. */ + if( pItem->pSelect ){ + zObj = ""; + }else if( (flags & (WHERE_IPK|WHERE_VIRTUALTABLE))==0 ){ + zObj = pLoop->u.btree.pIndex->zName; + }else{ + zObj = pItem->zName; + } + sqlite3StrAccumAppend(&str, zObj, sqlite3Strlen30(zObj)+1); + + /* Append the EQP text to the buffer */ isSearch = (flags&(WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))!=0 || ((flags&WHERE_VIRTUALTABLE)==0 && (pLoop->u.btree.nEq>0)) || (wctrlFlags&(WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX)); - - sqlite3StrAccumInit(&str, zBuf, sizeof(zBuf), SQLITE_MAX_LENGTH); - str.db = db; sqlite3StrAccumAppendAll(&str, isSearch ? "SEARCH" : "SCAN"); if( pItem->pSelect ){ sqlite3XPrintf(&str, 0, " SUBQUERY %d", pItem->iSelectId); @@ -2901,15 +2919,28 @@ static void explainOneScan( pLoop->u.vtab.idxNum, pLoop->u.vtab.idxStr); } #endif + nEstRow = pLoop->nOut>=10 ? sqlite3LogEstToInt(pLoop->nOut) : 1; #ifdef SQLITE_EXPLAIN_ESTIMATED_ROWS - if( pLoop->nOut>=10 ){ - sqlite3XPrintf(&str, 0, " (~%llu rows)", sqlite3LogEstToInt(pLoop->nOut)); - }else{ - sqlite3StrAccumAppend(&str, " (~1 row)", 9); - } + sqlite3XPrintf(&str, 0, " (~%llu rows)", nEstRow); #endif - zMsg = sqlite3StrAccumFinish(&str); - sqlite3VdbeAddOp4(v, OP_Explain, iId, iLevel, iFrom, zMsg, P4_DYNAMIC); + pExplain = (ExplainArg*)sqlite3StrAccumFinish(&str); + assert( pExplain || db->mallocFailed ); + if( pExplain ){ + memset(pExplain, 0, sizeof(*pExplain)); + if( pLoop->wsFlags & WHERE_INDEXED ){ + pExplain->iCsr = pLevel->iIdxCur; + }else if( pItem->pSelect==0 ){ + pExplain->iCsr = pLevel->iTabCur; + }else{ + pExplain->iCsr = -1; + } + pExplain->nEst = nEstRow; + pExplain->zName = (const char*)&pExplain[1]; + pExplain->zExplain = &pExplain->zName[sqlite3Strlen30(pExplain->zName)+1]; + sqlite3VdbeAddOp4(v, OP_Explain, + iId, iLevel, iFrom, (char*)pExplain,P4_EXPLAIN + ); + } } } #else diff --git a/test/scanstatus.test b/test/scanstatus.test new file mode 100644 index 0000000000..69a92dcb6a --- /dev/null +++ b/test/scanstatus.test @@ -0,0 +1,69 @@ +# 2014 November 1 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix scanstatus + +do_execsql_test 1.0 { + CREATE TABLE t1(a, b); + CREATE TABLE t2(x, y); + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(3, 4); + INSERT INTO t2 VALUES('a', 'b'); + INSERT INTO t2 VALUES('c', 'd'); + INSERT INTO t2 VALUES('e', 'f'); +} + +proc do_scanstatus_test {tn res} { + set stmt [db_last_stmt_ptr db] + set idx 0 + set ret [list] + while {1} { + set r [sqlite3_stmt_scanstatus $stmt $idx] + if {[llength $r]==0} break + lappend ret {*}$r + incr idx + } + + uplevel [list do_test $tn [list set {} $ret] [list {*}$res]] +} + +do_execsql_test 1.1 { SELECT count(*) FROM t1, t2; } 6 +do_scanstatus_test 1.2 { + nLoop 1 nVisit 2 nEst 1048576 zName t1 zExplain {SCAN TABLE t1} + nLoop 2 nVisit 6 nEst 1048576 zName t2 zExplain {SCAN TABLE t2} +} + +do_execsql_test 1.3 { + ANALYZE; + SELECT count(*) FROM t1, t2; +} 6 +do_scanstatus_test 1.4 { + nLoop 1 nVisit 2 nEst 2 zName t1 zExplain {SCAN TABLE t1} + nLoop 2 nVisit 6 nEst 3 zName t2 zExplain {SCAN TABLE t2} +} + +do_execsql_test 1.5 { + ANALYZE; + SELECT count(*) FROM t1, t2 WHERE t2.rowid>1; +} 4 +do_scanstatus_test 1.6 { + nLoop 1 nVisit 2 nEst 2 zName t2 zExplain + {SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid>?)} + nLoop 2 nVisit 4 nEst 2 zName t1 zExplain {SCAN TABLE t1} +} + + + + +finish_test From 89e71646df320b5ff107f618169227c5c7cac8eb Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 1 Nov 2014 18:08:04 +0000 Subject: [PATCH 058/133] Minor fixes and documentation improvements for sqlite3_stmt_scanstatus(). FossilOrigin-Name: 8d8cc9608d30bb65fffcfe488e904411cbbc7f41 --- manifest | 23 ++++---- manifest.uuid | 2 +- src/sqlite.h.in | 24 +++++++- src/vdbe.c | 3 + src/where.c | 40 +++++++++---- src/whereInt.h | 1 + test/scanstatus.test | 138 ++++++++++++++++++++++++++++++++++++++++++- 7 files changed, 200 insertions(+), 31 deletions(-) diff --git a/manifest b/manifest index 39eaf23ca4..8582d6b5e8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sexperimental\ssqlite3_stmt_scanstatus()\sAPI. -D 2014-10-31T20:11:32.562 +C Minor\sfixes\sand\sdocumentation\simprovements\sfor\ssqlite3_stmt_scanstatus(). +D 2014-11-01T18:08:04.130 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -229,7 +229,7 @@ F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c 282f8f5278e0c78eb442217531172ec9e1538796 -F src/sqlite.h.in 6e90cdb404e4fa8c0eb149ca79c11c6608a97ece +F src/sqlite.h.in aeba29025ba5fc721a11c1b81ed8745f93029590 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 4f86ac648ea398c1bb3db036062934cde257ea23 @@ -289,7 +289,7 @@ F src/update.c 3c4ecc282accf12d39edb8d524cf089645e55a13 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 3b627daa45c7308c1e36e3dbaa3f9ce7e5c7fa73 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c 80329be857cff3c2d6e8c1f594ac689166c44ae7 +F src/vdbe.c 5240bad20ce0b707afee211f0c21a862b54b1e0b F src/vdbe.h f8e5388173dbf8408da4ae1dcbf75911caf2253d F src/vdbeInt.h 284b2294c188474daa9b5ce2bd0200560cb0940d F src/vdbeapi.c 2afa2e162f290879ca9c51e8795a377035f1662a @@ -302,8 +302,8 @@ F src/vtab.c 2a30791bbd7926b589401bd09c3abb33de563793 F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 344beac082f8f4f69ec7c4a669d74294475e5abe -F src/whereInt.h 19279cd0664ce1d90b9ad3ef0108cb494acfe455 +F src/where.c 636ca646c8f2f53fd66f9ed5c773e84cb0c4b762 +F src/whereInt.h 99d324a8f921d7a40c605a8b197350c3cb18977d F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test b35b4cd69fc913f90d39a575e171e1116c3a4bb7 @@ -801,7 +801,7 @@ F test/savepoint4.test c8f8159ade6d2acd9128be61e1230f1c1edc6cc0 F test/savepoint5.test 0735db177e0ebbaedc39812c8d065075d563c4fd F test/savepoint6.test f41279c5e137139fa5c21485773332c7adb98cd7 F test/savepoint7.test fbf319a7b2dda089ec5be30a424a0e95f121d423 -F test/scanstatus.test a62dad3311fc51eab1d6977b082738bd2148fe07 +F test/scanstatus.test b4b1780bad243e1576329d05597b99f06886e4d2 F test/schema.test 8f7999be894260f151adf15c2c7540f1c6d6a481 F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5 F test/schema3.test 1bc1008e1f8cb5654b248c55f27249366eb7ed38 @@ -1211,10 +1211,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 67f0d469da28c023200239a1f3d0c6cef9ef0e45 -R 4c76f3e7b97f67c1dd65bda3a6e96149 -T *branch * scanstatus -T *sym-scanstatus * -T -sym-trunk * +P 6a9bab34aeb6a01b612211a28c140de60a3e883c +R d964f269915d57cf95d9198c6ba22665 U dan -Z 4994ca943ee260eeca8f493b4f59766a +Z 082decdd6298ff1a2330407f8d406dfb diff --git a/manifest.uuid b/manifest.uuid index b7984cac38..7325586095 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6a9bab34aeb6a01b612211a28c140de60a3e883c \ No newline at end of file +8d8cc9608d30bb65fffcfe488e904411cbbc7f41 \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index ba20cb9890..51383426d9 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -7408,6 +7408,8 @@ int sqlite3_vtab_on_conflict(sqlite3 *); /* +** CAPI3REF: Prepared Statement Scan Statuses +** ** Return status data for a single loop within query pStmt. ** ** Parameter "idx" identifies the specific loop to retrieve statistics for. @@ -7415,7 +7417,25 @@ int sqlite3_vtab_on_conflict(sqlite3 *); ** zero or greater than or equal to the total number of loops used to implement ** the statement - a non-zero value is returned. In this case the final value ** of all five output parameters is undefined. Otherwise, if idx is in range, -** the output parameters are populated and zero returned. +** zero is returned and the output parameters set as follows: +** +**
    +**
  • (*pnLoop) is set to the total number of times the loop has been run. +**
  • (*pnVisit) is set to the total number of rows visited by the loop. +**
  • (*pnEst) is set to the estimate of the number of rows visited +** by each run of the loop used by the SQL optimizer. Ideally, this +** value should be close to (*pnVisit)/(*pnLoop). +**
  • (*pzName) is set to point to a nul-terminated string containing the +** name of the index of table used by this loop. +**
  • (*pzExplain) is set to point to a nul-terminated string containing +** same text that would be returned for this loop by an EXPLAIN +** QUERY PLAN command. +**
+** +** Output parameters *pzName and *pzExplain are set to point to buffers +** managed by the statement object. Both of these pointers may be invalidated +** by any API call on the same statement object, including an sqlite3_step() +** sqlite3_bind_*() call. ** ** Statistics may not be available for all loops in all statements. In cases ** where there exist loops with no available statistics, this function ignores @@ -7436,6 +7456,8 @@ SQLITE_EXPERIMENTAL int sqlite3_stmt_scanstatus( /* +** CAPI3REF: Zero Scan-Status Counters +** ** Zero all sqlite3_stmt_scanstatus() related event counters. ** ** This API is only available if the library is built with pre-processor diff --git a/src/vdbe.c b/src/vdbe.c index a92c0509b3..1abccfeeab 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -3889,6 +3889,7 @@ case OP_NotExists: { /* jump, in3 */ res = 0; iKey = pIn3->u.i; rc = sqlite3BtreeMovetoUnpacked(pCrsr, 0, iKey, 0, &res); + IncrementExplainCounter(pC, nLoop); pC->movetoTarget = iKey; /* Used by OP_Delete */ pC->nullRow = 0; pC->cacheStatus = CACHE_STALE; @@ -3896,6 +3897,8 @@ case OP_NotExists: { /* jump, in3 */ VdbeBranchTaken(res!=0,2); if( res!=0 ){ pc = pOp->p2 - 1; + }else{ + IncrementExplainCounter(pC, nVisit); } pC->seekResult = res; break; diff --git a/src/where.c b/src/where.c index 3565327b52..ae3c12725e 100644 --- a/src/where.c +++ b/src/where.c @@ -2739,7 +2739,7 @@ static int codeAllEqualityTerms( return regBase; } -#ifndef SQLITE_OMIT_EXPLAIN +#if !defined(SQLITE_OMIT_EXPLAIN) || defined(SQLITE_ENABLE_STMT_SCANSTATUS) /* ** This routine is a helper for explainIndexRange() below ** @@ -2815,19 +2815,23 @@ static void explainIndexRange(StrAccum *pStr, WhereLoop *pLoop, Table *pTab){ static void explainOneScan( Parse *pParse, /* Parse context */ SrcList *pTabList, /* Table list this loop refers to */ - WhereLevel *pLevel, /* Scan to write OP_Explain opcode for */ + WhereInfo *pWInfo, /* WHERE clause this loop belongs to */ int iLevel, /* Value for "level" column of output */ - int iFrom, /* Value for "from" column of output */ u16 wctrlFlags /* Flags passed to sqlite3WhereBegin() */ ){ #if !defined(SQLITE_DEBUG) && !defined(SQLITE_ENABLE_STMT_SCANSTATUS) if( pParse->explain==2 ) #endif { + WhereLevel *pLevel = &pWInfo->a[iLevel]; struct SrcList_item *pItem = &pTabList->a[pLevel->iFrom]; Vdbe *v = pParse->pVdbe; /* VM being constructed */ sqlite3 *db = pParse->db; /* Database handle */ - int iId = pParse->iSelectId; /* Select id (left-most output column) */ +#ifdef SQLITE_OMIT_EXPLAIN + int iId = 0; /* Select id (left-most output column) */ +#else + int iId = pParse->iSelectId; /* Select id (left-most output column) */ +#endif int isSearch; /* True for a SEARCH. False for SCAN. */ WhereLoop *pLoop; /* The controlling WhereLoop object */ u32 flags; /* Flags that describe this loop */ @@ -2839,7 +2843,7 @@ static void explainOneScan( pLoop = pLevel->pWLoop; flags = pLoop->wsFlags; - if( (flags&WHERE_MULTI_OR) || (wctrlFlags&WHERE_ONETABLE_ONLY) ) return; + if( (flags&WHERE_MULTI_OR) ) return; sqlite3StrAccumInit(&str, zBuf, sizeof(zBuf), SQLITE_MAX_LENGTH); str.db = db; @@ -2865,7 +2869,11 @@ static void explainOneScan( || (wctrlFlags&(WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX)); sqlite3StrAccumAppendAll(&str, isSearch ? "SEARCH" : "SCAN"); if( pItem->pSelect ){ +#ifdef SQLITE_OMIT_EXPLAIN + sqlite3XPrintf(&str, 0, " SUBQUERY 0"); +#else sqlite3XPrintf(&str, 0, " SUBQUERY %d", pItem->iSelectId); +#endif }else{ sqlite3XPrintf(&str, 0, " TABLE %s", pItem->zName); } @@ -2937,15 +2945,15 @@ static void explainOneScan( pExplain->nEst = nEstRow; pExplain->zName = (const char*)&pExplain[1]; pExplain->zExplain = &pExplain->zName[sqlite3Strlen30(pExplain->zName)+1]; - sqlite3VdbeAddOp4(v, OP_Explain, - iId, iLevel, iFrom, (char*)pExplain,P4_EXPLAIN + pWInfo->iExplain = sqlite3VdbeAddOp4(v, OP_Explain, + iId, iLevel, pLevel->iFrom, (char*)pExplain,P4_EXPLAIN ); } } } #else -# define explainOneScan(u,v,w,x,y,z) -#endif /* SQLITE_OMIT_EXPLAIN */ +# define explainOneScan(v,w,x,y,z) +#endif /* !SQLITE_OMIT_EXPLAIN || SQLITE_ENABLE_STMT_SCANSTATUS */ /* @@ -3613,9 +3621,15 @@ static Bitmask codeOneLoopStart( assert( pSubWInfo || pParse->nErr || db->mallocFailed ); if( pSubWInfo ){ WhereLoop *pSubLoop; - explainOneScan( - pParse, pOrTab, &pSubWInfo->a[0], iLevel, pLevel->iFrom, 0 - ); + + /* If an OP_Explain was added for this sub-loop, fix the P2 and + ** P3 parameters to it so that they are relative to the current + ** context. */ + if( pSubWInfo->iExplain!=0 ){ + sqlite3VdbeChangeP2(v, pSubWInfo->iExplain, iLevel); + sqlite3VdbeChangeP3(v, pSubWInfo->iExplain, pLevel->iFrom); + } + /* This is the sub-WHERE clause body. First skip over ** duplicate rows from prior sub-WHERE clauses, and record the ** rowid (or PRIMARY KEY) for the current row so that the same @@ -6455,7 +6469,7 @@ WhereInfo *sqlite3WhereBegin( if( db->mallocFailed ) goto whereBeginError; } #endif - explainOneScan(pParse, pTabList, pLevel, ii, pLevel->iFrom, wctrlFlags); + explainOneScan(pParse, pTabList, pWInfo, ii, wctrlFlags); pLevel->addrBody = sqlite3VdbeCurrentAddr(v); notReady = codeOneLoopStart(pWInfo, ii, notReady); pWInfo->iContinue = pLevel->addrCont; diff --git a/src/whereInt.h b/src/whereInt.h index fd4cfdf88a..dcf0e00604 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -407,6 +407,7 @@ struct WhereInfo { int iTop; /* The very beginning of the WHERE loop */ int iContinue; /* Jump here to continue with next record */ int iBreak; /* Jump here to break out of the loop */ + int iExplain; /* Address of OP_Explain (if WHERE_ONETABLE_ONLY) */ int savedNQueryLoop; /* pParse->nQueryLoop outside the WHERE loop */ int aiCurOnePass[2]; /* OP_OpenWrite cursors for the ONEPASS opt */ WhereMaskSet sMaskSet; /* Map cursor numbers to bitmasks */ diff --git a/test/scanstatus.test b/test/scanstatus.test index 69a92dcb6a..9df80f9ba8 100644 --- a/test/scanstatus.test +++ b/test/scanstatus.test @@ -53,16 +53,148 @@ do_scanstatus_test 1.4 { nLoop 2 nVisit 6 nEst 3 zName t2 zExplain {SCAN TABLE t2} } -do_execsql_test 1.5 { - ANALYZE; +do_execsql_test 1.5 { ANALYZE } +do_execsql_test 1.6 { SELECT count(*) FROM t1, t2 WHERE t2.rowid>1; } 4 -do_scanstatus_test 1.6 { +do_scanstatus_test 1.7 { nLoop 1 nVisit 2 nEst 2 zName t2 zExplain {SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid>?)} nLoop 2 nVisit 4 nEst 2 zName t1 zExplain {SCAN TABLE t1} } +do_execsql_test 1.8 { + SELECT count(*) FROM t1, t2 WHERE t2.rowid>1; +} 4 + +do_scanstatus_test 1.9 { + nLoop 2 nVisit 4 nEst 2 zName t2 zExplain + {SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid>?)} + nLoop 4 nVisit 8 nEst 2 zName t1 zExplain {SCAN TABLE t1} +} + +do_test 1.9 { + sqlite3_stmt_scanstatus_reset [db_last_stmt_ptr db] +} {} + +do_scanstatus_test 1.10 { + nLoop 0 nVisit 0 nEst 2 zName t2 zExplain + {SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid>?)} + nLoop 0 nVisit 0 nEst 2 zName t1 zExplain {SCAN TABLE t1} +} + +#------------------------------------------------------------------------- +# Try a few different types of scans. +# +reset_db +do_execsql_test 2.1 { + CREATE TABLE x1(i INTEGER PRIMARY KEY, j); + INSERT INTO x1 VALUES(1, 'one'); + INSERT INTO x1 VALUES(2, 'two'); + INSERT INTO x1 VALUES(3, 'three'); + INSERT INTO x1 VALUES(4, 'four'); + CREATE INDEX x1j ON x1(j); + + SELECT * FROM x1 WHERE i=2; +} {2 two} + +do_scanstatus_test 2.2 { + nLoop 1 nVisit 1 nEst 1 zName x1 + zExplain {SEARCH TABLE x1 USING INTEGER PRIMARY KEY (rowid=?)} +} + +do_execsql_test 2.3.1 { + SELECT * FROM x1 WHERE j='two' +} {2 two} +do_scanstatus_test 2.3.2 { + nLoop 1 nVisit 1 nEst 10 zName x1j + zExplain {SEARCH TABLE x1 USING COVERING INDEX x1j (j=?)} +} + +do_execsql_test 2.4.1 { + SELECT * FROM x1 WHERE j<'two' +} {4 four 1 one 3 three} +do_scanstatus_test 2.4.2 { + nLoop 1 nVisit 4 nEst 262144 zName x1j + zExplain {SEARCH TABLE x1 USING COVERING INDEX x1j (j='two' +} {2 two} +do_scanstatus_test 2.5.2 { + nLoop 1 nVisit 1 nEst 262144 zName x1j + zExplain {SEARCH TABLE x1 USING COVERING INDEX x1j (j>?)} +} + +do_execsql_test 2.6.1 { + SELECT * FROM x1 WHERE j BETWEEN 'three' AND 'two' +} {3 three 2 two} +do_scanstatus_test 2.6.2 { + nLoop 1 nVisit 2 nEst 16384 zName x1j + zExplain {SEARCH TABLE x1 USING COVERING INDEX x1j (j>? AND j? AND j Date: Sat, 1 Nov 2014 18:32:18 +0000 Subject: [PATCH 059/133] Add requirements marks and make minor tweaks to documentation. FossilOrigin-Name: 49188b2bb53a92b0b0b6aaf8247edeb0c1bcd1f5 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/main.c | 15 +++++++++++---- src/sqlite.h.in | 4 ++-- src/sqliteInt.h | 7 +++---- 5 files changed, 25 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index ce0614fbee..8ab904b8d2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\scommand-line\sshell\sman-page\sto\suse\sthe\s".tr"\stroff\sdirective\ninstead\sof\s".cc"\sfor\sescaping\sthe\sinitial\s"."\scharacters\sin\sthe\s".help"\noutput. -D 2014-10-31T14:46:51.896 +C Add\srequirements\smarks\sand\smake\sminor\stweaks\sto\sdocumentation. +D 2014-11-01T18:32:18.050 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -194,7 +194,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 F src/loadext.c de741e66e5ddc1598d904d7289239696e40ed994 -F src/main.c 2b882f64580ea72e2d972a5296f9eaa75a353161 +F src/main.c 5f659bdb14cdba1c3b587b6ed09167c147e308a1 F src/malloc.c 3c3ac67969612493d435e14b6832793209afd2ec F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c faf615aafd8be74a71494dfa027c113ea5c6615f @@ -229,10 +229,10 @@ F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c 282f8f5278e0c78eb442217531172ec9e1538796 -F src/sqlite.h.in 737b7dd0f3f81fe183646d22828b39f85ef3e68c +F src/sqlite.h.in c3c6180be49d91d3b0001a8d3c604684f361adc2 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d -F src/sqliteInt.h 4f86ac648ea398c1bb3db036062934cde257ea23 +F src/sqliteInt.h 8f67ca79e957b8ece7453b8e320b6a996e1b4761 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 81712116e826b0089bb221b018929536b2b5406f F src/table.c f142bba7903e93ca8d113a5b8877a108ad1a27dc @@ -1210,7 +1210,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P a07078b60007e88adea67bec5f0caf91f707ad78 -R f904ce78baced6aeb6bcb91c631714db +P 67f0d469da28c023200239a1f3d0c6cef9ef0e45 +R dbf6a8dc4a878d1b1a7b9b3874ca58fb U drh -Z c463256d5d45e3bce4a212ca4dd9f3e4 +Z 455c35adeade418fab0456b1b175d27f diff --git a/manifest.uuid b/manifest.uuid index 4735237141..02f906627f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -67f0d469da28c023200239a1f3d0c6cef9ef0e45 \ No newline at end of file +49188b2bb53a92b0b0b6aaf8247edeb0c1bcd1f5 \ No newline at end of file diff --git a/src/main.c b/src/main.c index 5308698797..65a662ee5a 100644 --- a/src/main.c +++ b/src/main.c @@ -329,15 +329,17 @@ int sqlite3_config(int op, ...){ switch( op ){ /* Mutex configuration options are only available in a threadsafe - ** compile. + ** compile. */ -#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-54466-46756 */ case SQLITE_CONFIG_SINGLETHREAD: { /* Disable all mutexing */ sqlite3GlobalConfig.bCoreMutex = 0; sqlite3GlobalConfig.bFullMutex = 0; break; } +#endif +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-20520-54086 */ case SQLITE_CONFIG_MULTITHREAD: { /* Disable mutexing of database connections */ /* Enable mutexing of core data structures */ @@ -345,17 +347,23 @@ int sqlite3_config(int op, ...){ sqlite3GlobalConfig.bFullMutex = 0; break; } +#endif +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-59593-21810 */ case SQLITE_CONFIG_SERIALIZED: { /* Enable all mutexing */ sqlite3GlobalConfig.bCoreMutex = 1; sqlite3GlobalConfig.bFullMutex = 1; break; } +#endif +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-63666-48755 */ case SQLITE_CONFIG_MUTEX: { /* Specify an alternative mutex implementation */ sqlite3GlobalConfig.mutex = *va_arg(ap, sqlite3_mutex_methods*); break; } +#endif +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-14450-37597 */ case SQLITE_CONFIG_GETMUTEX: { /* Retrieve the current mutex implementation */ *va_arg(ap, sqlite3_mutex_methods*) = sqlite3GlobalConfig.mutex; @@ -363,7 +371,6 @@ int sqlite3_config(int op, ...){ } #endif - case SQLITE_CONFIG_MALLOC: { /* Specify an alternative malloc implementation */ sqlite3GlobalConfig.m = *va_arg(ap, sqlite3_mem_methods*); @@ -377,7 +384,7 @@ int sqlite3_config(int op, ...){ } case SQLITE_CONFIG_MEMSTATUS: { /* Enable or disable the malloc status collection */ - sqlite3GlobalConfig.bMemstat = va_arg(ap, int); + sqlite3GlobalConfig.bMemstat = va_arg(ap, int); /* IMP: R-27464-47829 */ break; } case SQLITE_CONFIG_SCRATCH: { diff --git a/src/sqlite.h.in b/src/sqlite.h.in index b4081f2a02..d0f213f156 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -1532,10 +1532,10 @@ struct sqlite3_mem_methods { ** ** [[SQLITE_CONFIG_SCRATCH]]
SQLITE_CONFIG_SCRATCH
**
^This option specifies a static memory buffer that SQLite can use for -** scratch memory. There are three arguments: A pointer an 8-byte +** scratch memory. ^(There are three arguments: A pointer an 8-byte ** aligned memory buffer from which the scratch allocations will be ** drawn, the size of each scratch allocation (sz), -** and the maximum number of scratch allocations (N). The sz +** and the maximum number of scratch allocations (N).)^ The sz ** argument must be a multiple of 16. ** The first argument must be a pointer to an 8-byte aligned buffer ** of at least sz*N bytes of memory. diff --git a/src/sqliteInt.h b/src/sqliteInt.h index f4785411d5..e9715efcf7 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -193,10 +193,9 @@ #endif /* -** The SQLITE_DEFAULT_MEMSTATUS macro must be defined as either 0 or 1. -** It determines whether or not the features related to -** SQLITE_CONFIG_MEMSTATUS are available by default or not. This value can -** be overridden at runtime using the sqlite3_config() API. +** EVIDENCE-OF: R-25715-37072 Memory allocation statistics are enabled by +** default unless SQLite is compiled with SQLITE_DEFAULT_MEMSTATUS=0 in +** which case memory allocation statistics are disabled by default. */ #if !defined(SQLITE_DEFAULT_MEMSTATUS) # define SQLITE_DEFAULT_MEMSTATUS 1 From 6f9702ed4dd4a6312e8a4e4ec9ae174d931298c8 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 1 Nov 2014 20:38:06 +0000 Subject: [PATCH 060/133] If SQLITE_ENABLE_STMT_SCANSTATUS is defined, record the number of times each VDBE opcode is executed. Derive the values returned by sqlite3_stmt_scanstatus() from these records on demand. FossilOrigin-Name: 9ea37422a8cc2fce51bb10508e5e90f40fd4b511 --- manifest | 26 ++++----- manifest.uuid | 2 +- src/vdbe.c | 44 +------------- src/vdbe.h | 22 ++----- src/vdbeInt.h | 18 ++++-- src/vdbeapi.c | 30 +++++----- src/vdbeaux.c | 52 ++++++++++++----- src/where.c | 134 +++++++++++++++++++++---------------------- src/whereInt.h | 3 + test/scanstatus.test | 49 +++++++++++++--- 10 files changed, 203 insertions(+), 177 deletions(-) diff --git a/manifest b/manifest index 8582d6b5e8..c81e911395 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\sfixes\sand\sdocumentation\simprovements\sfor\ssqlite3_stmt_scanstatus(). -D 2014-11-01T18:08:04.130 +C If\sSQLITE_ENABLE_STMT_SCANSTATUS\sis\sdefined,\srecord\sthe\snumber\sof\stimes\seach\sVDBE\sopcode\sis\sexecuted.\sDerive\sthe\svalues\sreturned\sby\ssqlite3_stmt_scanstatus()\sfrom\sthese\srecords\son\sdemand. +D 2014-11-01T20:38:06.833 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,11 +289,11 @@ F src/update.c 3c4ecc282accf12d39edb8d524cf089645e55a13 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 3b627daa45c7308c1e36e3dbaa3f9ce7e5c7fa73 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c 5240bad20ce0b707afee211f0c21a862b54b1e0b -F src/vdbe.h f8e5388173dbf8408da4ae1dcbf75911caf2253d -F src/vdbeInt.h 284b2294c188474daa9b5ce2bd0200560cb0940d -F src/vdbeapi.c 2afa2e162f290879ca9c51e8795a377035f1662a -F src/vdbeaux.c 1e2561eeb94749f6b8c0a1eb02b5fecb645ed48c +F src/vdbe.c 69d025732d242d7c97282e6570a4e5eb768ebaff +F src/vdbe.h 7d603b93d128e614ba2600f12a6c541435405522 +F src/vdbeInt.h ee8d44cba5998279039bcd91ebddb6d9a2d463b8 +F src/vdbeapi.c 19e433e69fe2b27bfc9337a207b6ebf499f41d03 +F src/vdbeaux.c bad342af7cadb8d3bf2990700a78787d602dcd69 F src/vdbeblob.c 8b5442ff0954c44b45cbabbe2e94091a2e16fdef F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f F src/vdbesort.c 975aeffa99acb0991b2f288d30294756bff41438 @@ -302,8 +302,8 @@ F src/vtab.c 2a30791bbd7926b589401bd09c3abb33de563793 F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 636ca646c8f2f53fd66f9ed5c773e84cb0c4b762 -F src/whereInt.h 99d324a8f921d7a40c605a8b197350c3cb18977d +F src/where.c fb404e3db40db61c212ffb39022e885a91252498 +F src/whereInt.h a2bc22f4e3e70eeaa57272f354c288bc3b71b80b F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test b35b4cd69fc913f90d39a575e171e1116c3a4bb7 @@ -801,7 +801,7 @@ F test/savepoint4.test c8f8159ade6d2acd9128be61e1230f1c1edc6cc0 F test/savepoint5.test 0735db177e0ebbaedc39812c8d065075d563c4fd F test/savepoint6.test f41279c5e137139fa5c21485773332c7adb98cd7 F test/savepoint7.test fbf319a7b2dda089ec5be30a424a0e95f121d423 -F test/scanstatus.test b4b1780bad243e1576329d05597b99f06886e4d2 +F test/scanstatus.test 40c7712c8bc0adc3fb88e0419356880679a3a5fb F test/schema.test 8f7999be894260f151adf15c2c7540f1c6d6a481 F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5 F test/schema3.test 1bc1008e1f8cb5654b248c55f27249366eb7ed38 @@ -1211,7 +1211,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 6a9bab34aeb6a01b612211a28c140de60a3e883c -R d964f269915d57cf95d9198c6ba22665 +P 8d8cc9608d30bb65fffcfe488e904411cbbc7f41 +R e1e809461d373caeeb0e83f126874426 U dan -Z 082decdd6298ff1a2330407f8d406dfb +Z 5d5debb04affeaeb7378f6b538c8f641 diff --git a/manifest.uuid b/manifest.uuid index 7325586095..f70d7e8aef 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8d8cc9608d30bb65fffcfe488e904411cbbc7f41 \ No newline at end of file +9ea37422a8cc2fce51bb10508e5e90f40fd4b511 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 1abccfeeab..bc49505dd0 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -167,18 +167,6 @@ int sqlite3_found_count = 0; /* Return true if the cursor was opened using the OP_OpenSorter opcode. */ #define isSorter(x) ((x)->pSorter!=0) -/* -** The first argument passed to the IncrementExplainCounter() macro must -** be a non-NULL pointer to an object of type VdbeCursor. The second -** argument must be either "nLoop" or "nVisit" (without the double-quotes). -*/ -#ifdef SQLITE_ENABLE_STMT_SCANSTATUS -# define IncrementExplainCounter(pCsr, counter) \ - if( (pCsr)->pExplain ) (pCsr)->pExplain->counter++ -#else -# define IncrementExplainCounter(pCsr, counter) -#endif - /* ** Allocate VdbeCursor number iCur. Return a pointer to it. Return NULL ** if we run out of memory. @@ -620,6 +608,9 @@ int sqlite3VdbeExec( #endif nVmStep++; pOp = &aOp[pc]; +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + if( p->pFrame==0 ) p->anExec[pc]++; +#endif /* Only allow tracing if SQLITE_DEBUG is defined. */ @@ -3568,7 +3559,6 @@ case OP_SeekGT: { /* jump, in3 */ #ifdef SQLITE_DEBUG pC->seekOp = pOp->opcode; #endif - IncrementExplainCounter(pC, nLoop); if( pC->isTable ){ /* The input value in P3 might be of any type: integer, real, string, ** blob, or NULL. But it needs to be an integer before we can do @@ -3677,8 +3667,6 @@ case OP_SeekGT: { /* jump, in3 */ VdbeBranchTaken(res!=0,2); if( res ){ pc = pOp->p2 - 1; - }else{ - IncrementExplainCounter(pC, nVisit); } break; } @@ -3889,7 +3877,6 @@ case OP_NotExists: { /* jump, in3 */ res = 0; iKey = pIn3->u.i; rc = sqlite3BtreeMovetoUnpacked(pCrsr, 0, iKey, 0, &res); - IncrementExplainCounter(pC, nLoop); pC->movetoTarget = iKey; /* Used by OP_Delete */ pC->nullRow = 0; pC->cacheStatus = CACHE_STALE; @@ -3897,8 +3884,6 @@ case OP_NotExists: { /* jump, in3 */ VdbeBranchTaken(res!=0,2); if( res!=0 ){ pc = pOp->p2 - 1; - }else{ - IncrementExplainCounter(pC, nVisit); } pC->seekResult = res; break; @@ -4467,8 +4452,6 @@ case OP_Last: { /* jump */ res = 0; assert( pCrsr!=0 ); rc = sqlite3BtreeLast(pCrsr, &res); - IncrementExplainCounter(pC, nLoop); - if( res==0 ) IncrementExplainCounter(pC, nVisit); pC->nullRow = (u8)res; pC->deferredMoveto = 0; pC->cacheStatus = CACHE_STALE; @@ -4538,14 +4521,11 @@ case OP_Rewind: { /* jump */ pC->deferredMoveto = 0; pC->cacheStatus = CACHE_STALE; } - IncrementExplainCounter(pC, nLoop); pC->nullRow = (u8)res; assert( pOp->p2>0 && pOp->p2nOp ); VdbeBranchTaken(res!=0,2); if( res ){ pc = pOp->p2 - 1; - }else{ - IncrementExplainCounter(pC, nVisit); } break; } @@ -4656,7 +4636,6 @@ next_tail: pC->cacheStatus = CACHE_STALE; VdbeBranchTaken(res==0,2); if( res==0 ){ - IncrementExplainCounter(pC, nVisit); pC->nullRow = 0; pc = pOp->p2 - 1; p->aCounter[pOp->p5]++; @@ -6079,10 +6058,7 @@ case OP_VFilter: { /* jump */ VdbeBranchTaken(res!=0,2); if( res ){ pc = pOp->p2 - 1; - }else{ - IncrementExplainCounter(pCur, nVisit); } - IncrementExplainCounter(pCur, nLoop); } pCur->nullRow = 0; @@ -6175,7 +6151,6 @@ case OP_VNext: { /* jump */ if( !res ){ /* If there is data, jump to P2 */ pc = pOp->p2 - 1; - IncrementExplainCounter(pCur, nVisit); } goto check_for_interrupt; } @@ -6375,19 +6350,6 @@ case OP_Init: { /* jump */ break; } -#ifdef SQLITE_ENABLE_STMT_SCANSTATUS -case OP_Explain: { - ExplainArg *pArg; - VdbeCursor *pCur; - if( pOp->p4type==P4_EXPLAIN && (pArg = pOp->p4.pExplain)->iCsr>=0 ){ - pArg = pOp->p4.pExplain; - pCur = p->apCsr[pArg->iCsr]; - pCur->pExplain = pArg; - } - break; -} -#endif - /* Opcode: Noop * * * * * ** diff --git a/src/vdbe.h b/src/vdbe.h index 1b470441be..966fdb622e 100644 --- a/src/vdbe.h +++ b/src/vdbe.h @@ -25,7 +25,6 @@ ** of this structure. */ typedef struct Vdbe Vdbe; -typedef struct ExplainArg ExplainArg; /* ** The names of the following types declared in vdbeInt.h are required @@ -60,7 +59,6 @@ struct VdbeOp { KeyInfo *pKeyInfo; /* Used when p4type is P4_KEYINFO */ int *ai; /* Used when p4type is P4_INTARRAY */ SubProgram *pProgram; /* Used when p4type is P4_SUBPROGRAM */ - ExplainArg *pExplain; /* Used when p4type is P4_EXPLAIN */ int (*xAdvance)(BtCursor *, int *); } p4; #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS @@ -102,19 +100,6 @@ struct VdbeOpList { }; typedef struct VdbeOpList VdbeOpList; -/* -** Structure used as the P4 parameter for OP_Explain opcodes. -*/ -struct ExplainArg { - int iCsr; /* Cursor number this applies to */ - i64 nLoop; /* Number of times loop has run */ - i64 nVisit; /* Total number of rows visited */ - i64 nEst; /* Estimated number of rows per scan */ - const char *zName; /* Name of table/index being scanned */ - const char *zExplain; /* EQP text for this loop */ -}; - - /* ** Allowed values of VdbeOp.p4type */ @@ -134,7 +119,6 @@ struct ExplainArg { #define P4_INTARRAY (-15) /* P4 is a vector of 32-bit integers */ #define P4_SUBPROGRAM (-18) /* P4 is a pointer to a SubProgram structure */ #define P4_ADVANCE (-19) /* P4 is a pointer to BtreeNext() or BtreePrev() */ -#define P4_EXPLAIN (-20) /* P4 is a pointer to an instance of ExplainArg */ /* Error message codes for OP_Halt */ #define P5_ConstraintNotNull 1 @@ -298,4 +282,10 @@ void sqlite3VdbeLinkSubProgram(Vdbe *, SubProgram *); # define VDBE_OFFSET_LINENO(x) 0 #endif +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS +void sqlite3VdbeScanCounter(Vdbe*, int, int, int, i64, const char*); +#else +# define sqlite3VdbeScanCounter(a,b,c,d,e) +#endif + #endif diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 839108f6bc..9f21ac7ebf 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -83,7 +83,6 @@ struct VdbeCursor { i64 seqCount; /* Sequence counter */ i64 movetoTarget; /* Argument to the deferred sqlite3BtreeMoveto() */ VdbeSorter *pSorter; /* Sorter object for OP_SorterOpen cursors */ - ExplainArg *pExplain; /* Object to store seek/visit counts (may be NULL) */ /* Cached information about the header for the data record that the ** cursor is currently pointing to. Only valid if cacheStatus matches @@ -292,12 +291,20 @@ struct Explain { char zBase[100]; /* Initial space */ }; - /* A bitfield type for use inside of structures. Always follow with :N where ** N is the number of bits. */ typedef unsigned bft; /* Bit Field Type */ +typedef struct ScanCounter ScanCounter; +struct ScanCounter { + int addrExplain; /* OP_Explain for loop */ + int addrLoop; /* Address of "loops" counter */ + int addrVisit; /* Address of "rows visited" counter */ + i64 nEst; /* Estimated rows per loop */ + char *zName; /* Name of table or index */ +}; + /* ** An instance of the virtual machine. This structure contains the complete ** state of the virtual machine. @@ -370,8 +377,11 @@ struct Vdbe { int nOnceFlag; /* Size of array aOnceFlag[] */ u8 *aOnceFlag; /* Flags for OP_Once */ AuxData *pAuxData; /* Linked list of auxdata allocations */ - ExplainArg **apExplain; /* Array of pointers to P4_EXPLAIN p4 values */ - int nExplain; /* Number of entries in array apExplain */ +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + i64 *anExec; /* Number of times each op has been executed */ + int nScan; /* Entries in aScan[] */ + ScanCounter *aScan; /* Scan definitions for sqlite3_stmt_scanstatus() */ +#endif }; /* diff --git a/src/vdbeapi.c b/src/vdbeapi.c index b09faa1fa5..f2e124dc51 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -1476,6 +1476,7 @@ int sqlite3_stmt_status(sqlite3_stmt *pStmt, int op, int resetFlag){ return (int)v; } +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS /* ** Return status data for a single loop within query pStmt. */ @@ -1489,14 +1490,20 @@ int sqlite3_stmt_scanstatus( const char **pzExplain /* OUT: EQP string */ ){ Vdbe *p = (Vdbe*)pStmt; - ExplainArg *pExplain; - if( idx<0 || idx>=p->nExplain ) return 1; - pExplain = p->apExplain[idx]; - if( pnLoop ) *pnLoop = pExplain->nLoop; - if( pnVisit ) *pnVisit = pExplain->nVisit; - if( pnEst ) *pnEst = pExplain->nEst; - if( *pzName ) *pzName = pExplain->zName; - if( *pzExplain ) *pzExplain = pExplain->zExplain; + ScanCounter *pScan; + if( idx<0 || idx>=p->nScan ) return 1; + pScan = &p->aScan[idx]; + if( pnLoop ) *pnLoop = p->anExec[pScan->addrLoop]; + if( pnVisit ) *pnVisit = p->anExec[pScan->addrVisit]; + if( pnEst ) *pnEst = pScan->nEst; + if( *pzName ) *pzName = pScan->zName; + if( *pzExplain ){ + if( pScan->addrExplain ){ + *pzExplain = p->aOp[ pScan->addrExplain ].p4.z; + }else{ + *pzExplain = 0; + } + } return 0; } @@ -1505,10 +1512,7 @@ int sqlite3_stmt_scanstatus( */ void sqlite3_stmt_scanstatus_reset(sqlite3_stmt *pStmt){ Vdbe *p = (Vdbe*)pStmt; - int i; - for(i=0; inExplain; i++){ - p->apExplain[i]->nLoop = 0; - p->apExplain[i]->nVisit = 0; - } + memset(p->anExec, 0, p->nOp * sizeof(i64)); } +#endif /* SQLITE_ENABLE_STMT_SCANSTATUS */ diff --git a/src/vdbeaux.c b/src/vdbeaux.c index a8223890bd..3f75cfee7a 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -597,6 +597,34 @@ int sqlite3VdbeAddOpList(Vdbe *p, int nOp, VdbeOpList const *aOp, int iLineno){ return addr; } +#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) +/* +** Add an entry to the array of counters managed by sqlite3_stmt_scanstatus(). +*/ +void sqlite3VdbeScanCounter( + Vdbe *p, /* VM to add scanstatus() to */ + int addrExplain, /* Address of OP_Explain (or 0) */ + int addrLoop, /* Address of loop counter */ + int addrVisit, /* Address of rows visited counter */ + i64 nEst, /* Estimated number of rows */ + const char *zName /* Name of table or index being scanned */ +){ + int nByte = (p->nScan+1) * sizeof(ScanCounter); + ScanCounter *aNew; + aNew = (ScanCounter*)sqlite3DbRealloc(p->db, p->aScan, nByte); + if( aNew ){ + ScanCounter *pNew = &aNew[p->nScan++]; + pNew->addrExplain = addrExplain; + pNew->addrLoop = addrLoop; + pNew->addrVisit = addrVisit; + pNew->nEst = nEst; + pNew->zName = sqlite3DbStrDup(p->db, zName); + p->aScan = aNew; + } +} +#endif + + /* ** Change the value of the P1 operand for a specific instruction. ** This routine is useful when a large program is loaded from a @@ -672,7 +700,6 @@ static void freeP4(sqlite3 *db, int p4type, void *p4){ if( p4 ){ assert( db ); switch( p4type ){ - case P4_EXPLAIN: case P4_REAL: case P4_INT64: case P4_DYNAMIC: @@ -821,13 +848,6 @@ void sqlite3VdbeChangeP4(Vdbe *p, int addr, const char *zP4, int n){ pOp->p4type = P4_VTAB; sqlite3VtabLock((VTable *)zP4); assert( ((VTable *)zP4)->db==p->db ); - }else if( n==P4_EXPLAIN ){ - pOp->p4.p = (void*)zP4; - pOp->p4type = P4_EXPLAIN; - p->apExplain = (ExplainArg**)sqlite3DbReallocOrFree( - p->db, p->apExplain, sizeof(ExplainArg*) * p->nExplain + 1 - ); - if( p->apExplain ) p->apExplain[p->nExplain++] = (ExplainArg*)zP4; }else if( n<0 ){ pOp->p4.p = (void*)zP4; pOp->p4type = (signed char)n; @@ -1111,10 +1131,6 @@ static char *displayP4(Op *pOp, char *zTemp, int nTemp){ zTemp[0] = 0; break; } - case P4_EXPLAIN: { - zP4 = (char*)pOp->p4.pExplain->zExplain; - break; - } default: { zP4 = pOp->p4.z; if( zP4==0 ){ @@ -1714,6 +1730,10 @@ void sqlite3VdbeMakeReady( zEnd = &zCsr[nByte]; }while( nByte && !db->mallocFailed ); +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + p->anExec = (i64*)sqlite3DbMallocZero(db, p->nOp*sizeof(i64)); +#endif + p->nCursor = nCursor; p->nOnceFlag = nOnce; if( p->aVar ){ @@ -2697,7 +2717,13 @@ void sqlite3VdbeClearObject(sqlite3 *db, Vdbe *p){ sqlite3DbFree(db, p->aColName); sqlite3DbFree(db, p->zSql); sqlite3DbFree(db, p->pFree); - sqlite3DbFree(db, p->apExplain); +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + sqlite3DbFree(db, p->anExec); + for(i=0; inScan; i++){ + sqlite3DbFree(db, p->aScan[i].zName); + } + sqlite3DbFree(db, p->aScan); +#endif } /* diff --git a/src/where.c b/src/where.c index ae3c12725e..8151a7057b 100644 --- a/src/where.c +++ b/src/where.c @@ -2739,7 +2739,7 @@ static int codeAllEqualityTerms( return regBase; } -#if !defined(SQLITE_OMIT_EXPLAIN) || defined(SQLITE_ENABLE_STMT_SCANSTATUS) +#ifndef SQLITE_OMIT_EXPLAIN /* ** This routine is a helper for explainIndexRange() below ** @@ -2812,68 +2812,43 @@ static void explainIndexRange(StrAccum *pStr, WhereLoop *pLoop, Table *pTab){ ** record is added to the output to describe the table scan strategy in ** pLevel. */ -static void explainOneScan( +static int explainOneScan( Parse *pParse, /* Parse context */ SrcList *pTabList, /* Table list this loop refers to */ - WhereInfo *pWInfo, /* WHERE clause this loop belongs to */ + WhereLevel *pLevel, /* Scan to write OP_Explain opcode for */ int iLevel, /* Value for "level" column of output */ + int iFrom, /* Value for "from" column of output */ u16 wctrlFlags /* Flags passed to sqlite3WhereBegin() */ ){ -#if !defined(SQLITE_DEBUG) && !defined(SQLITE_ENABLE_STMT_SCANSTATUS) + int ret = 0; +#ifndef SQLITE_DEBUG if( pParse->explain==2 ) #endif { - WhereLevel *pLevel = &pWInfo->a[iLevel]; struct SrcList_item *pItem = &pTabList->a[pLevel->iFrom]; Vdbe *v = pParse->pVdbe; /* VM being constructed */ sqlite3 *db = pParse->db; /* Database handle */ -#ifdef SQLITE_OMIT_EXPLAIN - int iId = 0; /* Select id (left-most output column) */ -#else - int iId = pParse->iSelectId; /* Select id (left-most output column) */ -#endif + int iId = pParse->iSelectId; /* Select id (left-most output column) */ int isSearch; /* True for a SEARCH. False for SCAN. */ WhereLoop *pLoop; /* The controlling WhereLoop object */ u32 flags; /* Flags that describe this loop */ + char *zMsg; /* Text to add to EQP output */ StrAccum str; /* EQP output string */ char zBuf[100]; /* Initial space for EQP output string */ - const char *zObj; - ExplainArg *pExplain; - i64 nEstRow; /* Estimated rows per scan of pLevel */ pLoop = pLevel->pWLoop; flags = pLoop->wsFlags; - if( (flags&WHERE_MULTI_OR) ) return; + if( (flags&WHERE_MULTI_OR) || (wctrlFlags&WHERE_ONETABLE_ONLY) ) return 0; - sqlite3StrAccumInit(&str, zBuf, sizeof(zBuf), SQLITE_MAX_LENGTH); - str.db = db; - - /* Reserve space at the start of the buffer managed by the StrAccum - ** object for *pExplain. */ - assert( sizeof(*pExplain)<=sizeof(zBuf) ); - str.nChar = sizeof(*pExplain); - - /* Append the object (table or index) name to the buffer. */ - if( pItem->pSelect ){ - zObj = ""; - }else if( (flags & (WHERE_IPK|WHERE_VIRTUALTABLE))==0 ){ - zObj = pLoop->u.btree.pIndex->zName; - }else{ - zObj = pItem->zName; - } - sqlite3StrAccumAppend(&str, zObj, sqlite3Strlen30(zObj)+1); - - /* Append the EQP text to the buffer */ isSearch = (flags&(WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))!=0 || ((flags&WHERE_VIRTUALTABLE)==0 && (pLoop->u.btree.nEq>0)) || (wctrlFlags&(WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX)); + + sqlite3StrAccumInit(&str, zBuf, sizeof(zBuf), SQLITE_MAX_LENGTH); + str.db = db; sqlite3StrAccumAppendAll(&str, isSearch ? "SEARCH" : "SCAN"); if( pItem->pSelect ){ -#ifdef SQLITE_OMIT_EXPLAIN - sqlite3XPrintf(&str, 0, " SUBQUERY 0"); -#else sqlite3XPrintf(&str, 0, " SUBQUERY %d", pItem->iSelectId); -#endif }else{ sqlite3XPrintf(&str, 0, " TABLE %s", pItem->zName); } @@ -2927,33 +2902,48 @@ static void explainOneScan( pLoop->u.vtab.idxNum, pLoop->u.vtab.idxStr); } #endif - nEstRow = pLoop->nOut>=10 ? sqlite3LogEstToInt(pLoop->nOut) : 1; #ifdef SQLITE_EXPLAIN_ESTIMATED_ROWS - sqlite3XPrintf(&str, 0, " (~%llu rows)", nEstRow); -#endif - pExplain = (ExplainArg*)sqlite3StrAccumFinish(&str); - assert( pExplain || db->mallocFailed ); - if( pExplain ){ - memset(pExplain, 0, sizeof(*pExplain)); - if( pLoop->wsFlags & WHERE_INDEXED ){ - pExplain->iCsr = pLevel->iIdxCur; - }else if( pItem->pSelect==0 ){ - pExplain->iCsr = pLevel->iTabCur; - }else{ - pExplain->iCsr = -1; - } - pExplain->nEst = nEstRow; - pExplain->zName = (const char*)&pExplain[1]; - pExplain->zExplain = &pExplain->zName[sqlite3Strlen30(pExplain->zName)+1]; - pWInfo->iExplain = sqlite3VdbeAddOp4(v, OP_Explain, - iId, iLevel, pLevel->iFrom, (char*)pExplain,P4_EXPLAIN - ); + if( pLoop->nOut>=10 ){ + sqlite3XPrintf(&str, 0, " (~%llu rows)", sqlite3LogEstToInt(pLoop->nOut)); + }else{ + sqlite3StrAccumAppend(&str, " (~1 row)", 9); } +#endif + zMsg = sqlite3StrAccumFinish(&str); + ret = sqlite3VdbeAddOp4(v, OP_Explain, iId, iLevel, iFrom, zMsg,P4_DYNAMIC); } + return ret; } #else -# define explainOneScan(v,w,x,y,z) -#endif /* !SQLITE_OMIT_EXPLAIN || SQLITE_ENABLE_STMT_SCANSTATUS */ +# define explainOneScan(u,v,w,x,y,z) 0 +#endif /* SQLITE_OMIT_EXPLAIN */ + +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS +static void addScanStatus( + Vdbe *v, + SrcList *pSrclist, + WhereLevel *pLvl, + int addrExplain +){ + const char *zObj = 0; + i64 nEst = 1; + WhereLoop *pLoop = pLvl->pWLoop; + if( (pLoop->wsFlags & (WHERE_IPK|WHERE_VIRTUALTABLE))==0 ){ + zObj = pLoop->u.btree.pIndex->zName; + }else{ + zObj = pSrclist->a[pLvl->iFrom].zName; + } + if( pLoop->nOut>=10 ){ + nEst = sqlite3LogEstToInt(pLoop->nOut); + } + sqlite3VdbeScanCounter( + v, addrExplain, pLvl->addrBody, pLvl->addrVisit, nEst, zObj + ); +} +#else +# define addScanStatus(a, b, c, d) +#endif + /* @@ -3621,14 +3611,10 @@ static Bitmask codeOneLoopStart( assert( pSubWInfo || pParse->nErr || db->mallocFailed ); if( pSubWInfo ){ WhereLoop *pSubLoop; - - /* If an OP_Explain was added for this sub-loop, fix the P2 and - ** P3 parameters to it so that they are relative to the current - ** context. */ - if( pSubWInfo->iExplain!=0 ){ - sqlite3VdbeChangeP2(v, pSubWInfo->iExplain, iLevel); - sqlite3VdbeChangeP3(v, pSubWInfo->iExplain, pLevel->iFrom); - } + int addrExplain = explainOneScan( + pParse, pOrTab, &pSubWInfo->a[0], iLevel, pLevel->iFrom, 0 + ); + addScanStatus(v, pOrTab, &pSubWInfo->a[0], addrExplain); /* This is the sub-WHERE clause body. First skip over ** duplicate rows from prior sub-WHERE clauses, and record the @@ -3760,6 +3746,10 @@ static Bitmask codeOneLoopStart( } } +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + pLevel->addrVisit = sqlite3VdbeCurrentAddr(v); +#endif + /* Insert code to test every subexpression that can be completely ** computed using the current set of tables. */ @@ -6461,7 +6451,10 @@ WhereInfo *sqlite3WhereBegin( */ notReady = ~(Bitmask)0; for(ii=0; iia[ii]; + wsFlags = pLevel->pWLoop->wsFlags; #ifndef SQLITE_OMIT_AUTOMATIC_INDEX if( (pLevel->pWLoop->wsFlags & WHERE_AUTO_INDEX)!=0 ){ constructAutomaticIndex(pParse, &pWInfo->sWC, @@ -6469,10 +6462,15 @@ WhereInfo *sqlite3WhereBegin( if( db->mallocFailed ) goto whereBeginError; } #endif - explainOneScan(pParse, pTabList, pWInfo, ii, wctrlFlags); + addrExplain = explainOneScan( + pParse, pTabList, pLevel, ii, pLevel->iFrom, wctrlFlags + ); pLevel->addrBody = sqlite3VdbeCurrentAddr(v); notReady = codeOneLoopStart(pWInfo, ii, notReady); pWInfo->iContinue = pLevel->addrCont; + if( (wsFlags&WHERE_MULTI_OR)==0 && (wctrlFlags&WHERE_ONETABLE_ONLY)==0 ){ + addScanStatus(v, pTabList, pLevel, addrExplain); + } } /* Done. */ diff --git a/src/whereInt.h b/src/whereInt.h index dcf0e00604..168e11f4c7 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -85,6 +85,9 @@ struct WhereLevel { } u; struct WhereLoop *pWLoop; /* The selected WhereLoop object */ Bitmask notReady; /* FROM entries not usable at this level */ +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + int addrVisit; /* Address at which row is visited */ +#endif }; /* diff --git a/test/scanstatus.test b/test/scanstatus.test index 9df80f9ba8..e4e0bbd4a5 100644 --- a/test/scanstatus.test +++ b/test/scanstatus.test @@ -115,7 +115,7 @@ do_execsql_test 2.4.1 { SELECT * FROM x1 WHERE j<'two' } {4 four 1 one 3 three} do_scanstatus_test 2.4.2 { - nLoop 1 nVisit 4 nEst 262144 zName x1j + nLoop 1 nVisit 3 nEst 262144 zName x1j zExplain {SEARCH TABLE x1 USING COVERING INDEX x1j (j? AND a? AND b? AND b? AND a Date: Sat, 1 Nov 2014 21:00:04 +0000 Subject: [PATCH 061/133] Minor performance enhancements to SQLITE_ENABLE_STMT_SCANSTATUS code. FossilOrigin-Name: f13d6ba8a72d75838c4aaf85326c1129da027f8b --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/vdbe.c | 4 +++- src/vdbeInt.h | 1 + src/vdbeaux.c | 7 ++----- src/where.c | 2 +- 6 files changed, 17 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index c81e911395..974494b3b9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C If\sSQLITE_ENABLE_STMT_SCANSTATUS\sis\sdefined,\srecord\sthe\snumber\sof\stimes\seach\sVDBE\sopcode\sis\sexecuted.\sDerive\sthe\svalues\sreturned\sby\ssqlite3_stmt_scanstatus()\sfrom\sthese\srecords\son\sdemand. -D 2014-11-01T20:38:06.833 +C Minor\sperformance\senhancements\sto\sSQLITE_ENABLE_STMT_SCANSTATUS\scode. +D 2014-11-01T21:00:04.841 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,11 +289,11 @@ F src/update.c 3c4ecc282accf12d39edb8d524cf089645e55a13 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 3b627daa45c7308c1e36e3dbaa3f9ce7e5c7fa73 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c 69d025732d242d7c97282e6570a4e5eb768ebaff +F src/vdbe.c ccc626d70659160596d28b4a910b6086da788695 F src/vdbe.h 7d603b93d128e614ba2600f12a6c541435405522 -F src/vdbeInt.h ee8d44cba5998279039bcd91ebddb6d9a2d463b8 +F src/vdbeInt.h 21570e5ec8b3a385d003e6e20f3a91712b7050e5 F src/vdbeapi.c 19e433e69fe2b27bfc9337a207b6ebf499f41d03 -F src/vdbeaux.c bad342af7cadb8d3bf2990700a78787d602dcd69 +F src/vdbeaux.c 2887d02721c540b25969d3260d0d3d5668d11583 F src/vdbeblob.c 8b5442ff0954c44b45cbabbe2e94091a2e16fdef F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f F src/vdbesort.c 975aeffa99acb0991b2f288d30294756bff41438 @@ -302,7 +302,7 @@ F src/vtab.c 2a30791bbd7926b589401bd09c3abb33de563793 F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c fb404e3db40db61c212ffb39022e885a91252498 +F src/where.c ad4a3bca9070e013ff41e8e63835788bcc43dd1c F src/whereInt.h a2bc22f4e3e70eeaa57272f354c288bc3b71b80b F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1211,7 +1211,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 8d8cc9608d30bb65fffcfe488e904411cbbc7f41 -R e1e809461d373caeeb0e83f126874426 +P 9ea37422a8cc2fce51bb10508e5e90f40fd4b511 +R 4191876729196c1564c4c28cce5855c1 U dan -Z 5d5debb04affeaeb7378f6b538c8f641 +Z 150c277a759b1b656399ef38dfe29c92 diff --git a/manifest.uuid b/manifest.uuid index f70d7e8aef..72c2809a53 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9ea37422a8cc2fce51bb10508e5e90f40fd4b511 \ No newline at end of file +f13d6ba8a72d75838c4aaf85326c1129da027f8b \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index bc49505dd0..890162dddc 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -609,7 +609,7 @@ int sqlite3VdbeExec( nVmStep++; pOp = &aOp[pc]; #ifdef SQLITE_ENABLE_STMT_SCANSTATUS - if( p->pFrame==0 ) p->anExec[pc]++; + if( p->anExec ) p->anExec[pc]++; #endif /* Only allow tracing if SQLITE_DEBUG is defined. @@ -5409,6 +5409,7 @@ case OP_Program: { /* jump */ pFrame->token = pProgram->token; pFrame->aOnceFlag = p->aOnceFlag; pFrame->nOnceFlag = p->nOnceFlag; + pFrame->anExec = p->anExec; pEnd = &VdbeFrameMem(pFrame)[pFrame->nChildMem]; for(pMem=VdbeFrameMem(pFrame); pMem!=pEnd; pMem++){ @@ -5437,6 +5438,7 @@ case OP_Program: { /* jump */ p->nOp = pProgram->nOp; p->aOnceFlag = (u8 *)&p->apCsr[p->nCursor]; p->nOnceFlag = pProgram->nOnce; + p->anExec = 0; pc = -1; memset(p->aOnceFlag, 0, p->nOnceFlag); diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 9f21ac7ebf..eb2438a87c 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -132,6 +132,7 @@ struct VdbeFrame { Vdbe *v; /* VM this frame belongs to */ VdbeFrame *pParent; /* Parent of this frame, or NULL if parent is main */ Op *aOp; /* Program instructions for parent frame */ + i64 *anExec; /* Event counters from parent frame */ Mem *aMem; /* Array of memory cells for parent frame */ u8 *aOnceFlag; /* Array of OP_Once flags for parent frame */ VdbeCursor **apCsr; /* Array of Vdbe cursors for parent frame */ diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 3f75cfee7a..548ef0ac5c 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -1723,6 +1723,7 @@ void sqlite3VdbeMakeReady( p->apCsr = allocSpace(p->apCsr, nCursor*sizeof(VdbeCursor*), &zCsr, zEnd, &nByte); p->aOnceFlag = allocSpace(p->aOnceFlag, nOnce, &zCsr, zEnd, &nByte); + p->anExec = allocSpace(p->anExec, p->nOp*sizeof(i64), &zCsr, zEnd, &nByte); if( nByte ){ p->pFree = sqlite3DbMallocZero(db, nByte); } @@ -1730,10 +1731,6 @@ void sqlite3VdbeMakeReady( zEnd = &zCsr[nByte]; }while( nByte && !db->mallocFailed ); -#ifdef SQLITE_ENABLE_STMT_SCANSTATUS - p->anExec = (i64*)sqlite3DbMallocZero(db, p->nOp*sizeof(i64)); -#endif - p->nCursor = nCursor; p->nOnceFlag = nOnce; if( p->aVar ){ @@ -1794,6 +1791,7 @@ void sqlite3VdbeFreeCursor(Vdbe *p, VdbeCursor *pCx){ */ int sqlite3VdbeFrameRestore(VdbeFrame *pFrame){ Vdbe *v = pFrame->v; + v->anExec = pFrame->anExec; v->aOnceFlag = pFrame->aOnceFlag; v->nOnceFlag = pFrame->nOnceFlag; v->aOp = pFrame->aOp; @@ -2718,7 +2716,6 @@ void sqlite3VdbeClearObject(sqlite3 *db, Vdbe *p){ sqlite3DbFree(db, p->zSql); sqlite3DbFree(db, p->pFree); #ifdef SQLITE_ENABLE_STMT_SCANSTATUS - sqlite3DbFree(db, p->anExec); for(i=0; inScan; i++){ sqlite3DbFree(db, p->aScan[i].zName); } diff --git a/src/where.c b/src/where.c index 8151a7057b..61c0ab7d3d 100644 --- a/src/where.c +++ b/src/where.c @@ -2821,7 +2821,7 @@ static int explainOneScan( u16 wctrlFlags /* Flags passed to sqlite3WhereBegin() */ ){ int ret = 0; -#ifndef SQLITE_DEBUG +#if !defined(SQLITE_DEBUG) && !defined(SQLITE_ENABLE_STMT_SCANSTATUS) if( pParse->explain==2 ) #endif { From 037b5324bd634e6288d7502b560bf53f273f0c94 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 3 Nov 2014 11:25:32 +0000 Subject: [PATCH 062/133] Remove unused variable from struct WhereInfo. Add some explanatory comments to new code. FossilOrigin-Name: f5313e0c680d9baebefb1cf50ddadedd4418a334 --- manifest | 24 ++++++++++++------------ manifest.uuid | 2 +- src/vdbe.h | 4 ++-- src/vdbeInt.h | 6 +++--- src/vdbeapi.c | 2 +- src/vdbeaux.c | 10 +++++----- src/where.c | 28 ++++++++++++++++++++-------- src/whereInt.h | 1 - test/scanstatus.test | 35 +++++++++++++++++++++++++++++++++++ 9 files changed, 79 insertions(+), 33 deletions(-) diff --git a/manifest b/manifest index 974494b3b9..9b7cf5c34a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\sperformance\senhancements\sto\sSQLITE_ENABLE_STMT_SCANSTATUS\scode. -D 2014-11-01T21:00:04.841 +C Remove\sunused\svariable\sfrom\sstruct\sWhereInfo.\sAdd\ssome\sexplanatory\scomments\sto\snew\scode. +D 2014-11-03T11:25:32.305 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -290,10 +290,10 @@ F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 3b627daa45c7308c1e36e3dbaa3f9ce7e5c7fa73 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a F src/vdbe.c ccc626d70659160596d28b4a910b6086da788695 -F src/vdbe.h 7d603b93d128e614ba2600f12a6c541435405522 -F src/vdbeInt.h 21570e5ec8b3a385d003e6e20f3a91712b7050e5 -F src/vdbeapi.c 19e433e69fe2b27bfc9337a207b6ebf499f41d03 -F src/vdbeaux.c 2887d02721c540b25969d3260d0d3d5668d11583 +F src/vdbe.h d412bd01e89f0d69991b8f46601f96bc169d28f4 +F src/vdbeInt.h 539ba284790e871f98be74a78cbdfcedfae22639 +F src/vdbeapi.c addf446ecade237bebd7e9fe769bdfb9db8d9fb1 +F src/vdbeaux.c 0aeb90cb62d7c07572798b41882838b3d4e55b44 F src/vdbeblob.c 8b5442ff0954c44b45cbabbe2e94091a2e16fdef F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f F src/vdbesort.c 975aeffa99acb0991b2f288d30294756bff41438 @@ -302,8 +302,8 @@ F src/vtab.c 2a30791bbd7926b589401bd09c3abb33de563793 F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c ad4a3bca9070e013ff41e8e63835788bcc43dd1c -F src/whereInt.h a2bc22f4e3e70eeaa57272f354c288bc3b71b80b +F src/where.c 53dae5ed6133438a9342c17bf3e95e00edbb0556 +F src/whereInt.h d3633e9b592103241b74b0ec76185f3e5b8b62e0 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test b35b4cd69fc913f90d39a575e171e1116c3a4bb7 @@ -801,7 +801,7 @@ F test/savepoint4.test c8f8159ade6d2acd9128be61e1230f1c1edc6cc0 F test/savepoint5.test 0735db177e0ebbaedc39812c8d065075d563c4fd F test/savepoint6.test f41279c5e137139fa5c21485773332c7adb98cd7 F test/savepoint7.test fbf319a7b2dda089ec5be30a424a0e95f121d423 -F test/scanstatus.test 40c7712c8bc0adc3fb88e0419356880679a3a5fb +F test/scanstatus.test 0c0baa647e98940d753d40691bf6475345c05cc5 F test/schema.test 8f7999be894260f151adf15c2c7540f1c6d6a481 F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5 F test/schema3.test 1bc1008e1f8cb5654b248c55f27249366eb7ed38 @@ -1211,7 +1211,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 9ea37422a8cc2fce51bb10508e5e90f40fd4b511 -R 4191876729196c1564c4c28cce5855c1 +P f13d6ba8a72d75838c4aaf85326c1129da027f8b +R 79a96158570090f7f8b0799c9e59db67 U dan -Z 150c277a759b1b656399ef38dfe29c92 +Z b85dee7fa48b3d6596e077d675db2dc9 diff --git a/manifest.uuid b/manifest.uuid index 72c2809a53..5972c23854 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f13d6ba8a72d75838c4aaf85326c1129da027f8b \ No newline at end of file +f5313e0c680d9baebefb1cf50ddadedd4418a334 \ No newline at end of file diff --git a/src/vdbe.h b/src/vdbe.h index 966fdb622e..1b9ad8b6bf 100644 --- a/src/vdbe.h +++ b/src/vdbe.h @@ -283,9 +283,9 @@ void sqlite3VdbeLinkSubProgram(Vdbe *, SubProgram *); #endif #ifdef SQLITE_ENABLE_STMT_SCANSTATUS -void sqlite3VdbeScanCounter(Vdbe*, int, int, int, i64, const char*); +void sqlite3VdbeScanStatus(Vdbe*, int, int, int, i64, const char*); #else -# define sqlite3VdbeScanCounter(a,b,c,d,e) +# define sqlite3VdbeScanStatus(a,b,c,d,e) #endif #endif diff --git a/src/vdbeInt.h b/src/vdbeInt.h index eb2438a87c..29117dd064 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -297,8 +297,8 @@ struct Explain { */ typedef unsigned bft; /* Bit Field Type */ -typedef struct ScanCounter ScanCounter; -struct ScanCounter { +typedef struct ScanStatus ScanStatus; +struct ScanStatus { int addrExplain; /* OP_Explain for loop */ int addrLoop; /* Address of "loops" counter */ int addrVisit; /* Address of "rows visited" counter */ @@ -381,7 +381,7 @@ struct Vdbe { #ifdef SQLITE_ENABLE_STMT_SCANSTATUS i64 *anExec; /* Number of times each op has been executed */ int nScan; /* Entries in aScan[] */ - ScanCounter *aScan; /* Scan definitions for sqlite3_stmt_scanstatus() */ + ScanStatus *aScan; /* Scan definitions for sqlite3_stmt_scanstatus() */ #endif }; diff --git a/src/vdbeapi.c b/src/vdbeapi.c index f2e124dc51..e6eb034ae9 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -1490,7 +1490,7 @@ int sqlite3_stmt_scanstatus( const char **pzExplain /* OUT: EQP string */ ){ Vdbe *p = (Vdbe*)pStmt; - ScanCounter *pScan; + ScanStatus *pScan; if( idx<0 || idx>=p->nScan ) return 1; pScan = &p->aScan[idx]; if( pnLoop ) *pnLoop = p->anExec[pScan->addrLoop]; diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 548ef0ac5c..da24291b4e 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -601,7 +601,7 @@ int sqlite3VdbeAddOpList(Vdbe *p, int nOp, VdbeOpList const *aOp, int iLineno){ /* ** Add an entry to the array of counters managed by sqlite3_stmt_scanstatus(). */ -void sqlite3VdbeScanCounter( +void sqlite3VdbeScanStatus( Vdbe *p, /* VM to add scanstatus() to */ int addrExplain, /* Address of OP_Explain (or 0) */ int addrLoop, /* Address of loop counter */ @@ -609,11 +609,11 @@ void sqlite3VdbeScanCounter( i64 nEst, /* Estimated number of rows */ const char *zName /* Name of table or index being scanned */ ){ - int nByte = (p->nScan+1) * sizeof(ScanCounter); - ScanCounter *aNew; - aNew = (ScanCounter*)sqlite3DbRealloc(p->db, p->aScan, nByte); + int nByte = (p->nScan+1) * sizeof(ScanStatus); + ScanStatus *aNew; + aNew = (ScanStatus*)sqlite3DbRealloc(p->db, p->aScan, nByte); if( aNew ){ - ScanCounter *pNew = &aNew[p->nScan++]; + ScanStatus *pNew = &aNew[p->nScan++]; pNew->addrExplain = addrExplain; pNew->addrLoop = addrLoop; pNew->addrVisit = addrVisit; diff --git a/src/where.c b/src/where.c index 61c0ab7d3d..81298b7821 100644 --- a/src/where.c +++ b/src/where.c @@ -2808,9 +2808,12 @@ static void explainIndexRange(StrAccum *pStr, WhereLoop *pLoop, Table *pTab){ /* ** This function is a no-op unless currently processing an EXPLAIN QUERY PLAN -** command. If the query being compiled is an EXPLAIN QUERY PLAN, a single -** record is added to the output to describe the table scan strategy in -** pLevel. +** command, or if either SQLITE_DEBUG or SQLITE_ENABLE_STMT_SCANSTATUS was +** defined at compile-time. If it is not a no-op, a single OP_Explain opcode +** is added to the output to describe the table scan strategy in pLevel. +** +** If an OP_Explain opcode is added to the VM, its address is returned. +** Otherwise, if no OP_Explain is coded, zero is returned. */ static int explainOneScan( Parse *pParse, /* Parse context */ @@ -2919,11 +2922,20 @@ static int explainOneScan( #endif /* SQLITE_OMIT_EXPLAIN */ #ifdef SQLITE_ENABLE_STMT_SCANSTATUS +/* +** Configure the VM passed as the first argument with an +** sqlite3_stmt_scanstatus() entry corresponding to the scan used to +** implement level pLvl. Argument pSrclist is a pointer to the FROM +** clause that the scan reads data from. +** +** If argument addrExplain is not 0, it must be the address of an +** OP_Explain instruction that describes the same loop. +*/ static void addScanStatus( - Vdbe *v, - SrcList *pSrclist, - WhereLevel *pLvl, - int addrExplain + Vdbe *v, /* Vdbe to add scanstatus entry to */ + SrcList *pSrclist, /* FROM clause pLvl reads data from */ + WhereLevel *pLvl, /* Level to add scanstatus() entry for */ + int addrExplain /* Address of OP_Explain (or 0) */ ){ const char *zObj = 0; i64 nEst = 1; @@ -2936,7 +2948,7 @@ static void addScanStatus( if( pLoop->nOut>=10 ){ nEst = sqlite3LogEstToInt(pLoop->nOut); } - sqlite3VdbeScanCounter( + sqlite3VdbeScanStatus( v, addrExplain, pLvl->addrBody, pLvl->addrVisit, nEst, zObj ); } diff --git a/src/whereInt.h b/src/whereInt.h index 168e11f4c7..2ccc6ec064 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -410,7 +410,6 @@ struct WhereInfo { int iTop; /* The very beginning of the WHERE loop */ int iContinue; /* Jump here to continue with next record */ int iBreak; /* Jump here to break out of the loop */ - int iExplain; /* Address of OP_Explain (if WHERE_ONETABLE_ONLY) */ int savedNQueryLoop; /* pParse->nQueryLoop outside the WHERE loop */ int aiCurOnePass[2]; /* OP_OpenWrite cursors for the ONEPASS opt */ WhereMaskSet sMaskSet; /* Map cursor numbers to bitmasks */ diff --git a/test/scanstatus.test b/test/scanstatus.test index e4e0bbd4a5..d4c70fbef4 100644 --- a/test/scanstatus.test +++ b/test/scanstatus.test @@ -231,4 +231,39 @@ do_scanstatus_test 3.4.2 { zExplain {SEARCH TABLE a1 USING INTEGER PRIMARY KEY (rowid=?)} } +#------------------------------------------------------------------------- +# Test that scanstatus() data is not available for searches performed +# by triggers. +# +# It is available for searches performed as part of FK processing, but +# not FK action processing. +# +do_execsql_test 4.0 { + CREATE TABLE t1(a, b, c); + CREATE TABLE t2(x PRIMARY KEY, y, z); + CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN + SELECT * FROM t2 WHERE x BETWEEN 20 AND 40; + END; + WITH d(x) AS (SELECT 1 UNION ALL SELECT x+1 AS n FROM d WHERE n<=100) + INSERT INTO t2 SELECT x, x*2, x*3 FROM d; +} + +do_execsql_test 4.1.1 { INSERT INTO t1 VALUES(1, 2, 3); } +do_scanstatus_test 4.1.2 { } + +do_execsql_test 4.2 { + CREATE TABLE p1(x PRIMARY KEY); + INSERT INTO p1 VALUES(1), (2), (3), (4); + CREATE TABLE c1(y REFERENCES p1); + INSERT INTO c1 VALUES(1), (2), (3); + PRAGMA foreign_keys=on; +} +do_execsql_test 4.2.1 { DELETE FROM p1 WHERE x=4 } +do_scanstatus_test 4.2.2 { + nLoop 1 nVisit 1 nEst 1 zName sqlite_autoindex_p1_1 + zExplain {SEARCH TABLE p1 USING INDEX sqlite_autoindex_p1_1 (x=?)} + + nLoop 1 nVisit 3 nEst 524288 zName c1 zExplain {SCAN TABLE c1} +} + finish_test From 7f5a7ecd216747e738588ad32064a182d2ce2928 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 3 Nov 2014 13:24:12 +0000 Subject: [PATCH 063/133] When enlarging the size of a StrAccum object, use sqlite3DbMallocSize() to record the entire size of the allocation, not just the requested size. FossilOrigin-Name: 3dda3c937469ce661d1cd0e8d8963a03e698ee39 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/printf.c | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 8ab904b8d2..3034b1c180 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\srequirements\smarks\sand\smake\sminor\stweaks\sto\sdocumentation. -D 2014-11-01T18:32:18.050 +C When\senlarging\sthe\ssize\sof\sa\sStrAccum\sobject,\suse\ssqlite3DbMallocSize()\sto\nrecord\sthe\sentire\ssize\sof\sthe\sallocation,\snot\sjust\sthe\srequested\ssize. +D 2014-11-03T13:24:12.668 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -223,7 +223,7 @@ F src/pcache.h 9b559127b83f84ff76d735c8262f04853be0c59a F src/pcache1.c e412cb585f777c840ddce0500eddc5c6043c2bb5 F src/pragma.c 3f3e959390a10c0131676f0e307acce372777e0f F src/prepare.c b7b7bf020bd4c962f7c8aed5a3c542c7dfe9f9c7 -F src/printf.c 10a2493593c8e4a538915cd3674bd7a67f70c488 +F src/printf.c ee13daccd49ddf767d057bc98395c9e83d4e2067 F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e @@ -1210,7 +1210,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 67f0d469da28c023200239a1f3d0c6cef9ef0e45 -R dbf6a8dc4a878d1b1a7b9b3874ca58fb +P 49188b2bb53a92b0b0b6aaf8247edeb0c1bcd1f5 +R 041ffec3947ff9e90700478ea54978f5 U drh -Z 455c35adeade418fab0456b1b175d27f +Z 42c888343c3ca87538d56b9b17ae76ab diff --git a/manifest.uuid b/manifest.uuid index 02f906627f..8be6ab9856 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -49188b2bb53a92b0b0b6aaf8247edeb0c1bcd1f5 \ No newline at end of file +3dda3c937469ce661d1cd0e8d8963a03e698ee39 \ No newline at end of file diff --git a/src/printf.c b/src/printf.c index 85d237f0e7..e20619f9a7 100644 --- a/src/printf.c +++ b/src/printf.c @@ -786,6 +786,7 @@ static int sqlite3StrAccumEnlarge(StrAccum *p, int N){ assert( p->zText!=0 || p->nChar==0 ); if( zOld==0 && p->nChar>0 ) memcpy(zNew, p->zText, p->nChar); p->zText = zNew; + p->nAlloc = sqlite3DbMallocSize(p->db, zNew); }else{ sqlite3StrAccumReset(p); setStrAccumError(p, STRACCUM_NOMEM); From 7b4d780b541ac9af0f5b183c72afa12916fd5d26 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 3 Nov 2014 14:46:29 +0000 Subject: [PATCH 064/133] Use exponential buffer size growth in StrAccum, as long as the size does not grow to large, to avoid excess memory allocation resize operations. Also, document the fact that setting scratch memory causes SQLite to try to avoid large memory allocations. FossilOrigin-Name: a518bc3318232d652349eb29303ff250aee40459 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/printf.c | 5 +++++ src/sqlite.h.in | 11 ++++++++--- src/vdbesort.c | 8 +++----- 5 files changed, 25 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index 3034b1c180..de67a3e45a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\senlarging\sthe\ssize\sof\sa\sStrAccum\sobject,\suse\ssqlite3DbMallocSize()\sto\nrecord\sthe\sentire\ssize\sof\sthe\sallocation,\snot\sjust\sthe\srequested\ssize. -D 2014-11-03T13:24:12.668 +C Use\sexponential\sbuffer\ssize\sgrowth\sin\sStrAccum,\sas\slong\sas\sthe\ssize\sdoes\snot\ngrow\sto\slarge,\sto\savoid\sexcess\smemory\sallocation\sresize\soperations.\s\sAlso,\ndocument\sthe\sfact\sthat\ssetting\sscratch\smemory\scauses\sSQLite\sto\stry\sto\savoid\nlarge\smemory\sallocations. +D 2014-11-03T14:46:29.027 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -223,13 +223,13 @@ F src/pcache.h 9b559127b83f84ff76d735c8262f04853be0c59a F src/pcache1.c e412cb585f777c840ddce0500eddc5c6043c2bb5 F src/pragma.c 3f3e959390a10c0131676f0e307acce372777e0f F src/prepare.c b7b7bf020bd4c962f7c8aed5a3c542c7dfe9f9c7 -F src/printf.c ee13daccd49ddf767d057bc98395c9e83d4e2067 +F src/printf.c 9e75a6a0b55bf61cfff7d7e19d89834a1b938236 F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c 282f8f5278e0c78eb442217531172ec9e1538796 -F src/sqlite.h.in c3c6180be49d91d3b0001a8d3c604684f361adc2 +F src/sqlite.h.in dd0cb63fb7cb558892afe8253c3279c508c50329 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 8f67ca79e957b8ece7453b8e320b6a996e1b4761 @@ -296,7 +296,7 @@ F src/vdbeapi.c 02d8afcff710eb35e3d9e49cb677308296b00009 F src/vdbeaux.c 3d6b2b412ef2193aa4729922dfc5df1efadbf6df F src/vdbeblob.c 8b5442ff0954c44b45cbabbe2e94091a2e16fdef F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f -F src/vdbesort.c 975aeffa99acb0991b2f288d30294756bff41438 +F src/vdbesort.c daf87ea542df088ac4607660d09976d36b6bd95d F src/vdbetrace.c 7e4222955e07dd707a2f360c0eb73452be1cb010 F src/vtab.c 2a30791bbd7926b589401bd09c3abb33de563793 F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 @@ -1210,7 +1210,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 49188b2bb53a92b0b0b6aaf8247edeb0c1bcd1f5 -R 041ffec3947ff9e90700478ea54978f5 +P 3dda3c937469ce661d1cd0e8d8963a03e698ee39 +R e2c699e9cb6d832117ea76de0a03f26b U drh -Z 42c888343c3ca87538d56b9b17ae76ab +Z 91c9a2887aef0ce4f97740bae4c1a9ba diff --git a/manifest.uuid b/manifest.uuid index 8be6ab9856..11815fb0c2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3dda3c937469ce661d1cd0e8d8963a03e698ee39 \ No newline at end of file +a518bc3318232d652349eb29303ff250aee40459 \ No newline at end of file diff --git a/src/printf.c b/src/printf.c index e20619f9a7..ac7b051631 100644 --- a/src/printf.c +++ b/src/printf.c @@ -770,6 +770,11 @@ static int sqlite3StrAccumEnlarge(StrAccum *p, int N){ char *zOld = (p->zText==p->zBase ? 0 : p->zText); i64 szNew = p->nChar; szNew += N + 1; + if( szNew+p->nChar<=p->mxAlloc ){ + /* Force exponential buffer size growth as long as it does not overflow, + ** to avoid having to call this routine too often */ + szNew += p->nChar; + } if( szNew > p->mxAlloc ){ sqlite3StrAccumReset(p); setStrAccumError(p, STRACCUM_TOOBIG); diff --git a/src/sqlite.h.in b/src/sqlite.h.in index d0f213f156..ffd0ade039 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -1535,8 +1535,7 @@ struct sqlite3_mem_methods { ** scratch memory. ^(There are three arguments: A pointer an 8-byte ** aligned memory buffer from which the scratch allocations will be ** drawn, the size of each scratch allocation (sz), -** and the maximum number of scratch allocations (N).)^ The sz -** argument must be a multiple of 16. +** and the maximum number of scratch allocations (N).)^ ** The first argument must be a pointer to an 8-byte aligned buffer ** of at least sz*N bytes of memory. ** ^SQLite will not use more than two scratch buffers per thread and not @@ -1548,7 +1547,13 @@ struct sqlite3_mem_methods { ** of the size of the WAL file. ** ^If SQLite needs needs additional ** scratch memory beyond what is provided by this configuration option, then -** [sqlite3_malloc()] will be used to obtain the memory needed.
+** [sqlite3_malloc()] will be used to obtain the memory needed.

+** ^When the application provides any amount of scratch memory using +** SQLITE_CONFIG_SCRATCH, SQLite avoids unnecessary large +** [sqlite3_malloc|heap allocations]. +** This can help [Robson proof|prevent memory allocation failures] due to heap +** fragmentation in low-memory embedded systems. +** ** ** [[SQLITE_CONFIG_PAGECACHE]]

SQLITE_CONFIG_PAGECACHE
**
^This option specifies a static memory buffer that SQLite can use for diff --git a/src/vdbesort.c b/src/vdbesort.c index 46c9f3789d..918d840716 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -847,11 +847,9 @@ int sqlite3VdbeSorterInit( if( mxCachemxPmaSize = mxCache * pgsz; - /* If the application has not configure scratch memory using - ** SQLITE_CONFIG_SCRATCH then we assume it is OK to do large memory - ** allocations. If scratch memory has been configured, then assume - ** large memory allocations should be avoided to prevent heap - ** fragmentation. + /* EVIDENCE-OF: R-26747-61719 When the application provides any amount of + ** scratch memory using SQLITE_CONFIG_SCRATCH, SQLite avoids unnecessary + ** large heap allocations. */ if( sqlite3GlobalConfig.pScratch==0 ){ assert( pSorter->iMemory==0 ); From e2f771b04706d0c055d5c3b7731e4cda14467600 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 3 Nov 2014 15:33:17 +0000 Subject: [PATCH 065/133] Add further tests. Fixes so that compilation without ENABLE_STMT_SCANSTATUS works. FossilOrigin-Name: a2303c719222f1effb51acc6b37930561148c00c --- manifest | 20 +++---- manifest.uuid | 2 +- src/test_config.c | 6 ++ src/vdbe.c | 4 ++ src/vdbeaux.c | 4 ++ src/where.c | 2 +- test/scanstatus.test | 129 +++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 155 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 9b7cf5c34a..5c51df5fcc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sunused\svariable\sfrom\sstruct\sWhereInfo.\sAdd\ssome\sexplanatory\scomments\sto\snew\scode. -D 2014-11-03T11:25:32.305 +C Add\sfurther\stests.\sFixes\sso\sthat\scompilation\swithout\sENABLE_STMT_SCANSTATUS\sworks. +D 2014-11-03T15:33:17.869 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -250,7 +250,7 @@ F src/test_async.c 21e11293a2f72080eda70e1124e9102044531cd8 F src/test_autoext.c dea8a01a7153b9adc97bd26161e4226329546e12 F src/test_backup.c 3875e899222b651e18b662f86e0e50daa946344e F src/test_btree.c 2e9978eca99a9a4bfa8cae949efb00886860a64f -F src/test_config.c a4cdebe093474c02eecc5e4008b1a22198edf975 +F src/test_config.c c8b8b50bf2fe5102de10e4c7100b746d7f6bf62f F src/test_demovfs.c 69b2085076654ebc18014cbc6386f04409c959a9 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc F src/test_fs.c ced436e3d4b8e4681328409b8081051ce614e28f @@ -289,11 +289,11 @@ F src/update.c 3c4ecc282accf12d39edb8d524cf089645e55a13 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 3b627daa45c7308c1e36e3dbaa3f9ce7e5c7fa73 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c ccc626d70659160596d28b4a910b6086da788695 +F src/vdbe.c 175a360c56e75ce4eb2b60704fd7c011b93926f5 F src/vdbe.h d412bd01e89f0d69991b8f46601f96bc169d28f4 F src/vdbeInt.h 539ba284790e871f98be74a78cbdfcedfae22639 F src/vdbeapi.c addf446ecade237bebd7e9fe769bdfb9db8d9fb1 -F src/vdbeaux.c 0aeb90cb62d7c07572798b41882838b3d4e55b44 +F src/vdbeaux.c cf6b8152dd22155201d57c216e6266866b61da59 F src/vdbeblob.c 8b5442ff0954c44b45cbabbe2e94091a2e16fdef F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f F src/vdbesort.c 975aeffa99acb0991b2f288d30294756bff41438 @@ -302,7 +302,7 @@ F src/vtab.c 2a30791bbd7926b589401bd09c3abb33de563793 F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 53dae5ed6133438a9342c17bf3e95e00edbb0556 +F src/where.c d5fa1081bf7cb70478ed06489e063695c92ee1e1 F src/whereInt.h d3633e9b592103241b74b0ec76185f3e5b8b62e0 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -801,7 +801,7 @@ F test/savepoint4.test c8f8159ade6d2acd9128be61e1230f1c1edc6cc0 F test/savepoint5.test 0735db177e0ebbaedc39812c8d065075d563c4fd F test/savepoint6.test f41279c5e137139fa5c21485773332c7adb98cd7 F test/savepoint7.test fbf319a7b2dda089ec5be30a424a0e95f121d423 -F test/scanstatus.test 0c0baa647e98940d753d40691bf6475345c05cc5 +F test/scanstatus.test 01afb2220f18ce85f9e338c20684f428d56e5c01 F test/schema.test 8f7999be894260f151adf15c2c7540f1c6d6a481 F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5 F test/schema3.test 1bc1008e1f8cb5654b248c55f27249366eb7ed38 @@ -1211,7 +1211,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f13d6ba8a72d75838c4aaf85326c1129da027f8b -R 79a96158570090f7f8b0799c9e59db67 +P f5313e0c680d9baebefb1cf50ddadedd4418a334 +R b9cde6c71acdc4fef0c5c1ac03d62018 U dan -Z b85dee7fa48b3d6596e077d675db2dc9 +Z f0600457ea98c0c998694790015a9674 diff --git a/manifest.uuid b/manifest.uuid index 5972c23854..fe7315da12 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f5313e0c680d9baebefb1cf50ddadedd4418a334 \ No newline at end of file +a2303c719222f1effb51acc6b37930561148c00c \ No newline at end of file diff --git a/src/test_config.c b/src/test_config.c index 2c11f713fb..03f4f76f6d 100644 --- a/src/test_config.c +++ b/src/test_config.c @@ -484,6 +484,12 @@ Tcl_SetVar2(interp, "sqlite_options", "mergesort", "1", TCL_GLOBAL_ONLY); Tcl_SetVar2(interp, "sqlite_options", "stat3", "0", TCL_GLOBAL_ONLY); #endif +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + Tcl_SetVar2(interp, "sqlite_options", "scanstatus", "1", TCL_GLOBAL_ONLY); +#else + Tcl_SetVar2(interp, "sqlite_options", "scanstatus", "0", TCL_GLOBAL_ONLY); +#endif + #if !defined(SQLITE_ENABLE_LOCKING_STYLE) # if defined(__APPLE__) # define SQLITE_ENABLE_LOCKING_STYLE 1 diff --git a/src/vdbe.c b/src/vdbe.c index 890162dddc..62470a743b 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -5409,7 +5409,9 @@ case OP_Program: { /* jump */ pFrame->token = pProgram->token; pFrame->aOnceFlag = p->aOnceFlag; pFrame->nOnceFlag = p->nOnceFlag; +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS pFrame->anExec = p->anExec; +#endif pEnd = &VdbeFrameMem(pFrame)[pFrame->nChildMem]; for(pMem=VdbeFrameMem(pFrame); pMem!=pEnd; pMem++){ @@ -5438,7 +5440,9 @@ case OP_Program: { /* jump */ p->nOp = pProgram->nOp; p->aOnceFlag = (u8 *)&p->apCsr[p->nCursor]; p->nOnceFlag = pProgram->nOnce; +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS p->anExec = 0; +#endif pc = -1; memset(p->aOnceFlag, 0, p->nOnceFlag); diff --git a/src/vdbeaux.c b/src/vdbeaux.c index da24291b4e..7cf996ce5c 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -1723,7 +1723,9 @@ void sqlite3VdbeMakeReady( p->apCsr = allocSpace(p->apCsr, nCursor*sizeof(VdbeCursor*), &zCsr, zEnd, &nByte); p->aOnceFlag = allocSpace(p->aOnceFlag, nOnce, &zCsr, zEnd, &nByte); +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS p->anExec = allocSpace(p->anExec, p->nOp*sizeof(i64), &zCsr, zEnd, &nByte); +#endif if( nByte ){ p->pFree = sqlite3DbMallocZero(db, nByte); } @@ -1791,7 +1793,9 @@ void sqlite3VdbeFreeCursor(Vdbe *p, VdbeCursor *pCx){ */ int sqlite3VdbeFrameRestore(VdbeFrame *pFrame){ Vdbe *v = pFrame->v; +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS v->anExec = pFrame->anExec; +#endif v->aOnceFlag = pFrame->aOnceFlag; v->nOnceFlag = pFrame->nOnceFlag; v->aOp = pFrame->aOp; diff --git a/src/where.c b/src/where.c index 81298b7821..d7966fdfbc 100644 --- a/src/where.c +++ b/src/where.c @@ -2953,7 +2953,7 @@ static void addScanStatus( ); } #else -# define addScanStatus(a, b, c, d) +# define addScanStatus(a, b, c, d) ((void)d) #endif diff --git a/test/scanstatus.test b/test/scanstatus.test index d4c70fbef4..9b34506e91 100644 --- a/test/scanstatus.test +++ b/test/scanstatus.test @@ -14,6 +14,11 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl set testprefix scanstatus +ifcapable !scanstatus { + finish_test + return +} + do_execsql_test 1.0 { CREATE TABLE t1(a, b); CREATE TABLE t2(x, y); @@ -266,4 +271,128 @@ do_scanstatus_test 4.2.2 { nLoop 1 nVisit 3 nEst 524288 zName c1 zExplain {SCAN TABLE c1} } +#------------------------------------------------------------------------- +# Further tests of different scan types. +# +reset_db +proc tochar {i} { + set alphabet {a b c d e f g h i j k l m n o p q r s t u v w x y z} + return [lindex $alphabet [expr $i % [llength $alphabet]]] +} +db func tochar tochar +do_execsql_test 5.0 { + CREATE TABLE t1(a PRIMARY KEY, b, c); + INSERT INTO t1 VALUES(0, 1, 'a'); + INSERT INTO t1 VALUES(1, 0, 'b'); + INSERT INTO t1 VALUES(2, 1, 'c'); + INSERT INTO t1 VALUES(3, 0, 'd'); + INSERT INTO t1 VALUES(4, 1, 'e'); + INSERT INTO t1 VALUES(5, 0, 'a'); + INSERT INTO t1 VALUES(6, 1, 'b'); + INSERT INTO t1 VALUES(7, 0, 'c'); + INSERT INTO t1 VALUES(8, 1, 'd'); + INSERT INTO t1 VALUES(9, 0, 'e'); + CREATE INDEX t1bc ON t1(b, c); + + CREATE TABLE t2(x, y); + CREATE INDEX t2xy ON t2(x, y); + WITH data(i, x, y) AS ( + SELECT 0, 0, tochar(0) + UNION ALL + SELECT i+1, (i+1)%2, tochar(i+1) FROM data WHERE i<500 + ) INSERT INTO t2 SELECT x, y FROM data; + + CREATE TABLE t3(x, y); + INSERT INTO t3 SELECT * FROM t2; + + ANALYZE; +} + +do_execsql_test 5.1.1 { + SELECT count(*) FROM t1 WHERE a IN (SELECT b FROM t1 AS ii) +} {2} +do_scanstatus_test 5.1.2 { + nLoop 1 nVisit 10 nEst 10 zName t1bc + zExplain {SCAN TABLE t1 AS ii USING COVERING INDEX t1bc} + nLoop 1 nVisit 2 nEst 8 zName sqlite_autoindex_t1_1 + zExplain {SEARCH TABLE t1 USING COVERING INDEX sqlite_autoindex_t1_1 (a=?)} +} + +do_execsql_test 5.2.1 { + SELECT count(*) FROM t1 WHERE a IN (0, 1) +} {2} +do_scanstatus_test 5.2.2 { + nLoop 1 nVisit 2 nEst 2 zName sqlite_autoindex_t1_1 + zExplain {SEARCH TABLE t1 USING COVERING INDEX sqlite_autoindex_t1_1 (a=?)} +} + +do_eqp_test 5.3.1 { + SELECT count(*) FROM t2 WHERE y = 'j'; +} {0 0 0 {SEARCH TABLE t2 USING COVERING INDEX t2xy (ANY(x) AND y=?)}} +do_execsql_test 5.3.2 { + SELECT count(*) FROM t2 WHERE y = 'j'; +} {19} +do_scanstatus_test 5.3.3 { + nLoop 1 nVisit 19 nEst 56 zName t2xy zExplain + {SEARCH TABLE t2 USING COVERING INDEX t2xy (ANY(x) AND y=?)} +} + +do_eqp_test 5.4.1 { + SELECT count(*) FROM t1, t2 WHERE y = c; +} { + 0 0 0 {SCAN TABLE t1 USING COVERING INDEX t1bc} + 0 1 1 {SEARCH TABLE t2 USING COVERING INDEX t2xy (ANY(x) AND y=?)} +} +do_execsql_test 5.4.2 { + SELECT count(*) FROM t1, t2 WHERE y = c; +} {200} +do_scanstatus_test 5.4.3 { + nLoop 1 nVisit 10 nEst 10 zName t1bc + zExplain {SCAN TABLE t1 USING COVERING INDEX t1bc} + nLoop 10 nVisit 200 nEst 56 zName t2xy + zExplain {SEARCH TABLE t2 USING COVERING INDEX t2xy (ANY(x) AND y=?)} +} + +do_eqp_test 5.5.1 { + SELECT count(*) FROM t1, t3 WHERE y = c; +} { + 0 0 1 {SCAN TABLE t3} + 0 1 0 {SEARCH TABLE t1 USING AUTOMATIC COVERING INDEX (c=?)} +} +do_execsql_test 5.5.2 { + SELECT count(*) FROM t1, t3 WHERE y = c; +} {200} +do_scanstatus_test 5.5.3 { + nLoop 1 nVisit 501 nEst 480 zName t3 zExplain {SCAN TABLE t3} + nLoop 501 nVisit 200 nEst 20 zName auto-index zExplain + {SEARCH TABLE t1 USING AUTOMATIC COVERING INDEX (c=?)} +} + +#------------------------------------------------------------------------- +# Virtual table scans +# +ifcapable fts3 { + do_execsql_test 6.0 { + CREATE VIRTUAL TABLE ft1 USING fts4; + INSERT INTO ft1 VALUES('a d c f g h e i f c'); + INSERT INTO ft1 VALUES('g c h b g b f f f g'); + INSERT INTO ft1 VALUES('h h c c h f a e d d'); + INSERT INTO ft1 VALUES('e j i j i e b c f g'); + INSERT INTO ft1 VALUES('g f b g j c h a d f'); + INSERT INTO ft1 VALUES('j i a e g f a i a c'); + INSERT INTO ft1 VALUES('f d g g j j c a h g'); + INSERT INTO ft1 VALUES('b d h a d j j j b i'); + INSERT INTO ft1 VALUES('j e a b j e c b c i'); + INSERT INTO ft1 VALUES('a d e f b j j c g d'); + } + do_execsql_test 6.1.1 { + SELECT count(*) FROM ft1 WHERE ft1 MATCH 'd' + } {6} + do_scanstatus_test 6.1.2 { + nLoop 1 nVisit 6 nEst 24 zName ft1 zExplain + {SCAN TABLE ft1 VIRTUAL TABLE INDEX 3:} + } +} + + finish_test From d1a1c23423560f4cac2cab305d92bb2d67a04d57 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 3 Nov 2014 16:35:55 +0000 Subject: [PATCH 066/133] Refactor the interface to make it more easily extensible. FossilOrigin-Name: 7955342da4a35b57e4ae26690b8d40f7bba20e8f --- manifest | 18 +++++----- manifest.uuid | 2 +- src/sqlite.h.in | 93 ++++++++++++++++++++++++++++++------------------- src/test1.c | 8 +++-- src/vdbeapi.c | 46 +++++++++++++++--------- 5 files changed, 102 insertions(+), 65 deletions(-) diff --git a/manifest b/manifest index 5c51df5fcc..134366e0aa 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sfurther\stests.\sFixes\sso\sthat\scompilation\swithout\sENABLE_STMT_SCANSTATUS\sworks. -D 2014-11-03T15:33:17.869 +C Refactor\sthe\sinterface\sto\smake\sit\smore\seasily\sextensible. +D 2014-11-03T16:35:55.977 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -229,7 +229,7 @@ F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c 282f8f5278e0c78eb442217531172ec9e1538796 -F src/sqlite.h.in aeba29025ba5fc721a11c1b81ed8745f93029590 +F src/sqlite.h.in a110c6320b3af8d7c0660fa1adbc530b6caa2d95 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 4f86ac648ea398c1bb3db036062934cde257ea23 @@ -237,7 +237,7 @@ F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 81712116e826b0089bb221b018929536b2b5406f F src/table.c f142bba7903e93ca8d113a5b8877a108ad1a27dc F src/tclsqlite.c 7cdd4dd3c2a4183483feca260070d73d6e22cd47 -F src/test1.c b53f4da2f386efa5c248716cd4bdc8344a87e952 +F src/test1.c 5890094c09691fe9564cf0f0d5b22d35b3218c47 F src/test2.c 98049e51a17dc62606a99a9eb95ee477f9996712 F src/test3.c 1c0e5d6f080b8e33c1ce8b3078e7013fdbcd560c F src/test4.c 9b32d22f5f150abe23c1830e2057c4037c45b3df @@ -292,7 +292,7 @@ F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a F src/vdbe.c 175a360c56e75ce4eb2b60704fd7c011b93926f5 F src/vdbe.h d412bd01e89f0d69991b8f46601f96bc169d28f4 F src/vdbeInt.h 539ba284790e871f98be74a78cbdfcedfae22639 -F src/vdbeapi.c addf446ecade237bebd7e9fe769bdfb9db8d9fb1 +F src/vdbeapi.c 76d62888455e3d3ffeaf70911cefa94e76a4678a F src/vdbeaux.c cf6b8152dd22155201d57c216e6266866b61da59 F src/vdbeblob.c 8b5442ff0954c44b45cbabbe2e94091a2e16fdef F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f @@ -1211,7 +1211,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f5313e0c680d9baebefb1cf50ddadedd4418a334 -R b9cde6c71acdc4fef0c5c1ac03d62018 -U dan -Z f0600457ea98c0c998694790015a9674 +P a2303c719222f1effb51acc6b37930561148c00c +R cbd06b886fdee03f70c2bf18e1342ca1 +U drh +Z 0328a60e2b5415c07361c47ade3f21c5 diff --git a/manifest.uuid b/manifest.uuid index fe7315da12..5fd1f91e84 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a2303c719222f1effb51acc6b37930561148c00c \ No newline at end of file +7955342da4a35b57e4ae26690b8d40f7bba20e8f \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 51383426d9..30ef4afb60 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -7406,54 +7406,75 @@ int sqlite3_vtab_on_conflict(sqlite3 *); /* #define SQLITE_ABORT 4 // Also an error code */ #define SQLITE_REPLACE 5 +/* CAPI3REF: Prepared Statement Scan Status Opcodes +** KEYWORDS: {scanstatus option} +** +** The following constants can be used for the T parameter to the +** [sqlite3_stmt_scanstatus(S,X,T,V)] interface. Each constant designates a +** different metric for sqlite3_stmt_scanstatus() to return. +** +**
+** [[SQLITE_SCANSTAT_NLOOP]]
SQLITE_SCANSTAT_NLOOP +**
The [sqlite3_int64] variable pointed to by the T parameter will be set to the +** total number of times that the X-th loop has run.
+** +** [[SQLITE_SCANSTAT_NVISIT]] 		SQLITE_SCANSTAT_NVISIT +**
The [sqlite3_int64] variable pointed to by the T parameter will be set to the +** total number of rows visited by the X-th loop.
+** +** [[SQLITE_SCANSTAT_EST]] 		SQLITE_SCANSTAT_EST +**
The [sqlite3_int64] variable pointed to by the T parameter will be set to the +** query planner's estimate for the number of rows visited for each +** iteration of the X-th loop. If the query planner's estimate was accurate, +** then this value should be approximately NVISIT/NLOOP. +** +** [[SQLITE_SCANSTAT_NAME]]
SQLITE_SCANSTAT_NAME +**
The "const char *" variable pointed to by the T parameter will be set to +** a zero-terminated UTF-8 string containing the name of the index or table used +** for the X-th loop. +** +** [[SQLITE_SCANSTAT_EXPLAIN]]
SQLITE_SCANSTAT_EXPLAIN +**
The "const char *" variable pointed to by the T parameter will be set to +** a zero-terminated UTF-8 string containing the [EXPLAIN QUERY PLAN] description +** for the X-th loop. +** +*/ +#define SQLITE_SCANSTAT_NLOOP 0 +#define SQLITE_SCANSTAT_NVISIT 1 +#define SQLITE_SCANSTAT_NEST 2 +#define SQLITE_SCANSTAT_NAME 3 +#define SQLITE_SCANSTAT_EXPLAIN 4 /* -** CAPI3REF: Prepared Statement Scan Statuses +** CAPI3REF: Prepared Statement Scan Status ** ** Return status data for a single loop within query pStmt. ** +** The "iScanStatusOp" parameter determines which status information to return. +** The "iScanStatusOp" must be one of the [scanstatus options] or the behavior of +** this interface is undefined. +** The requested measurement is written into a variable pointed to by +** the "pOut" parameter. ** Parameter "idx" identifies the specific loop to retrieve statistics for. ** Loops are numbered starting from zero. If idx is out of range - less than ** zero or greater than or equal to the total number of loops used to implement -** the statement - a non-zero value is returned. In this case the final value -** of all five output parameters is undefined. Otherwise, if idx is in range, -** zero is returned and the output parameters set as follows: +** the statement - a non-zero value is returned and the variable that pOut +** points to is unchanged. ** -**
    -**
  • (*pnLoop) is set to the total number of times the loop has been run. -**
  • (*pnVisit) is set to the total number of rows visited by the loop. -**
  • (*pnEst) is set to the estimate of the number of rows visited -** by each run of the loop used by the SQL optimizer. Ideally, this -** value should be close to (*pnVisit)/(*pnLoop). -**
  • (*pzName) is set to point to a nul-terminated string containing the -** name of the index of table used by this loop. -**
  • (*pzExplain) is set to point to a nul-terminated string containing -** same text that would be returned for this loop by an EXPLAIN -** QUERY PLAN command. -**
-** -** Output parameters *pzName and *pzExplain are set to point to buffers -** managed by the statement object. Both of these pointers may be invalidated -** by any API call on the same statement object, including an sqlite3_step() -** sqlite3_bind_*() call. -** -** Statistics may not be available for all loops in all statements. In cases -** where there exist loops with no available statistics, this function ignores -** them completely. +** Statistics might not be available for all loops in all statements. In cases +** where there exist loops with no available statistics, this function behaves +** as if the loop did not exist - it returns non-zero and leave the variable +** that pOut points to unchanged. ** ** This API is only available if the library is built with pre-processor -** symbol SQLITE_ENABLE_STMT_SCANSTATUS defined. +** symbol [SQLITE_ENABLE_STMT_SCANSTATUS] defined. */ SQLITE_EXPERIMENTAL int sqlite3_stmt_scanstatus( - sqlite3_stmt *pStmt, - int idx, /* Index of loop to report on */ - sqlite3_int64 *pnLoop, /* OUT: Number of times loop was run */ - sqlite3_int64 *pnVisit, /* OUT: Number of rows visited (all loops) */ - sqlite3_int64 *pnEst, /* OUT: Number of rows estimated (per loop) */ - const char **pzName, /* OUT: Object name (table or index) */ - const char **pzExplain /* OUT: EQP string */ -); - + sqlite3_stmt *pStmt, /* Prepared statement for which info desired */ + int idx, /* Index of loop to report on */ + int iScanStatusOp, /* Information desired. SQLITE_SCANSTAT_* */ + void *pOut /* Result written here */ +); /* ** CAPI3REF: Zero Scan-Status Counters @@ -7461,7 +7482,7 @@ SQLITE_EXPERIMENTAL int sqlite3_stmt_scanstatus( ** Zero all sqlite3_stmt_scanstatus() related event counters. ** ** This API is only available if the library is built with pre-processor -** symbol SQLITE_ENABLE_STMT_SCANSTATUS defined. +** symbol [SQLITE_ENABLE_STMT_SCANSTATUS] defined. */ SQLITE_EXPERIMENTAL void sqlite3_stmt_scanstatus_reset(sqlite3_stmt*); diff --git a/src/test1.c b/src/test1.c index 72e70513bc..5e526b3013 100644 --- a/src/test1.c +++ b/src/test1.c @@ -2328,19 +2328,21 @@ static int test_stmt_scanstatus( if( getStmtPointer(interp, Tcl_GetString(objv[1]), &pStmt) ) return TCL_ERROR; if( Tcl_GetIntFromObj(interp, objv[2], &idx) ) return TCL_ERROR; - res = sqlite3_stmt_scanstatus( - pStmt, idx, &nLoop, &nVisit, &nEst, &zName, &zExplain - ); + res = sqlite3_stmt_scanstatus(pStmt, idx, SQLITE_SCANSTAT_NLOOP, (void*)&nLoop); if( res==0 ){ Tcl_Obj *pRet = Tcl_NewObj(); Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj("nLoop", -1)); Tcl_ListObjAppendElement(0, pRet, Tcl_NewWideIntObj(nLoop)); + sqlite3_stmt_scanstatus(pStmt, idx, SQLITE_SCANSTAT_NVISIT, (void*)&nVisit); Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj("nVisit", -1)); Tcl_ListObjAppendElement(0, pRet, Tcl_NewWideIntObj(nVisit)); + sqlite3_stmt_scanstatus(pStmt, idx, SQLITE_SCANSTAT_EST, (void*)&nEst); Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj("nEst", -1)); Tcl_ListObjAppendElement(0, pRet, Tcl_NewWideIntObj(nEst)); + sqlite3_stmt_scanstatus(pStmt, idx, SQLITE_SCANSTAT_NAME, (void*)&zName); Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj("zName", -1)); Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zName, -1)); + sqlite3_stmt_scanstatus(pStmt, idx, SQLITE_SCANSTAT_EXPLAIN, (void*)&zExplain); Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj("zExplain", -1)); Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zExplain, -1)); Tcl_SetObjResult(interp, pRet); diff --git a/src/vdbeapi.c b/src/vdbeapi.c index e6eb034ae9..d3ba4e95d2 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -1481,27 +1481,42 @@ int sqlite3_stmt_status(sqlite3_stmt *pStmt, int op, int resetFlag){ ** Return status data for a single loop within query pStmt. */ int sqlite3_stmt_scanstatus( - sqlite3_stmt *pStmt, + sqlite3_stmt *pStmt, /* Prepared statement being queried */ int idx, /* Index of loop to report on */ - sqlite3_int64 *pnLoop, /* OUT: Number of times loop was run */ - sqlite3_int64 *pnVisit, /* OUT: Number of rows visited (all loops) */ - sqlite3_int64 *pnEst, /* OUT: Number of rows estimated (per loop) */ - const char **pzName, /* OUT: Object name (table or index) */ - const char **pzExplain /* OUT: EQP string */ + int iScanStatusOp, /* Which metric to return */ + void *pOut /* OUT: Write the answer here */ ){ Vdbe *p = (Vdbe*)pStmt; ScanStatus *pScan; if( idx<0 || idx>=p->nScan ) return 1; pScan = &p->aScan[idx]; - if( pnLoop ) *pnLoop = p->anExec[pScan->addrLoop]; - if( pnVisit ) *pnVisit = p->anExec[pScan->addrVisit]; - if( pnEst ) *pnEst = pScan->nEst; - if( *pzName ) *pzName = pScan->zName; - if( *pzExplain ){ - if( pScan->addrExplain ){ - *pzExplain = p->aOp[ pScan->addrExplain ].p4.z; - }else{ - *pzExplain = 0; + switch( iScanStatusOp ){ + case SQLITE_SCANSTAT_NLOOP: { + *(sqlite3_int64*)pOut = p->anExec[pScan->addrLoop]; + break; + } + case SQLITE_SCANSTAT_NVISIT: { + *(sqlite3_int64*)pOut = p->anExec[pScan->addrVisit]; + break; + } + case SQLITE_SCANSTAT_EST: { + *(sqlite3_int64*)pOut = pScan->nEst; + break; + } + case SQLITE_SCANSTAT_NAME: { + *(const char**)pOut = pScan->zName + break; + } + case SQLITE_SCANSTAT_EXPLAIN: { + if( pScan->addrExplain ){ + *(const char**)pOut = p->aOp[ pScan->addrExplain ].p4.z; + }else{ + *(const char**)pOut = 0; + } + break; + } + default: { + return 1; } } return 0; @@ -1515,4 +1530,3 @@ void sqlite3_stmt_scanstatus_reset(sqlite3_stmt *pStmt){ memset(p->anExec, 0, p->nOp * sizeof(i64)); } #endif /* SQLITE_ENABLE_STMT_SCANSTATUS */ - From d72219da438dcc06d78fc10b9c7958cf4f12ffab Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 3 Nov 2014 16:39:37 +0000 Subject: [PATCH 067/133] Fix a typo preventing this from building with SQLITE_ENABLE_STMT_SCANSTATUS defined. FossilOrigin-Name: 4c5714ab3dba19513374c7b1478221a0b90b450c --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/sqlite.h.in | 2 +- src/vdbeapi.c | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 134366e0aa..75cfa9ca43 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Refactor\sthe\sinterface\sto\smake\sit\smore\seasily\sextensible. -D 2014-11-03T16:35:55.977 +C Fix\sa\stypo\spreventing\sthis\sfrom\sbuilding\swith\sSQLITE_ENABLE_STMT_SCANSTATUS\sdefined. +D 2014-11-03T16:39:37.742 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -229,7 +229,7 @@ F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c 282f8f5278e0c78eb442217531172ec9e1538796 -F src/sqlite.h.in a110c6320b3af8d7c0660fa1adbc530b6caa2d95 +F src/sqlite.h.in 26b6aa6a1825b97c0fa16ce14fc657be00c299f6 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 4f86ac648ea398c1bb3db036062934cde257ea23 @@ -292,7 +292,7 @@ F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a F src/vdbe.c 175a360c56e75ce4eb2b60704fd7c011b93926f5 F src/vdbe.h d412bd01e89f0d69991b8f46601f96bc169d28f4 F src/vdbeInt.h 539ba284790e871f98be74a78cbdfcedfae22639 -F src/vdbeapi.c 76d62888455e3d3ffeaf70911cefa94e76a4678a +F src/vdbeapi.c 900259bdd85cd66a9f210d8ec08147a9034593bd F src/vdbeaux.c cf6b8152dd22155201d57c216e6266866b61da59 F src/vdbeblob.c 8b5442ff0954c44b45cbabbe2e94091a2e16fdef F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f @@ -1211,7 +1211,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P a2303c719222f1effb51acc6b37930561148c00c -R cbd06b886fdee03f70c2bf18e1342ca1 -U drh -Z 0328a60e2b5415c07361c47ade3f21c5 +P 7955342da4a35b57e4ae26690b8d40f7bba20e8f +R c9becfda3a2880c1d6a40a42c50145fa +U dan +Z a994e9671fe739ddd2243271f124e718 diff --git a/manifest.uuid b/manifest.uuid index 5fd1f91e84..2f63e2e7f1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7955342da4a35b57e4ae26690b8d40f7bba20e8f \ No newline at end of file +4c5714ab3dba19513374c7b1478221a0b90b450c \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 30ef4afb60..8f51f0e180 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -7441,7 +7441,7 @@ int sqlite3_vtab_on_conflict(sqlite3 *); */ #define SQLITE_SCANSTAT_NLOOP 0 #define SQLITE_SCANSTAT_NVISIT 1 -#define SQLITE_SCANSTAT_NEST 2 +#define SQLITE_SCANSTAT_EST 2 #define SQLITE_SCANSTAT_NAME 3 #define SQLITE_SCANSTAT_EXPLAIN 4 diff --git a/src/vdbeapi.c b/src/vdbeapi.c index d3ba4e95d2..d472004590 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -1504,7 +1504,7 @@ int sqlite3_stmt_scanstatus( break; } case SQLITE_SCANSTAT_NAME: { - *(const char**)pOut = pScan->zName + *(const char**)pOut = pScan->zName; break; } case SQLITE_SCANSTAT_EXPLAIN: { From d84bf205c38c9f46cee73c1e24f1dee67d32a441 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 3 Nov 2014 18:03:00 +0000 Subject: [PATCH 068/133] Updates to the sqlite3_stmt_scanstatus() documentation. No changes to code. FossilOrigin-Name: d97c324eb1d870c994911c53fbf84205f4e3e7a1 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/sqlite.h.in | 35 +++++++++++++++++++---------------- 3 files changed, 27 insertions(+), 24 deletions(-) diff --git a/manifest b/manifest index 07d97518f1..8d217ecfe0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sexperimental\ssqlite3_stmt_scanstatus()\sAPI.\sFor\scomparing\sthe\snumber\sof\srows\sactually\svisited\sby\sa\sloop\swith\sthe\sestimate\sused\sby\sthe\squery\splanner. -D 2014-11-03T16:56:43.943 +C Updates\sto\sthe\ssqlite3_stmt_scanstatus()\sdocumentation.\s\sNo\schanges\sto\scode. +D 2014-11-03T18:03:00.467 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -229,7 +229,7 @@ F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c 282f8f5278e0c78eb442217531172ec9e1538796 -F src/sqlite.h.in 3b69f7d9b74e2e5d0448dc2f0d2d160517afa253 +F src/sqlite.h.in 11f33a3e968a9637d6fa4ae879b83edd171ac88f F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 8f67ca79e957b8ece7453b8e320b6a996e1b4761 @@ -1211,7 +1211,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P a518bc3318232d652349eb29303ff250aee40459 4c5714ab3dba19513374c7b1478221a0b90b450c -R ce3f2c2aee62a076b6ed59326c9cde3e -U dan -Z cd275722bcaf367ed1a07165e9a16c03 +P ab3b0fc5760c6d428dbe1f974726a7d3526640bc +R e3e372d3594a5227dc7ce2380d71aae3 +U drh +Z 7bcc3180ea33e48aa6d0e7855da7bfd4 diff --git a/manifest.uuid b/manifest.uuid index c71e9c3f37..0611c8a12e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ab3b0fc5760c6d428dbe1f974726a7d3526640bc \ No newline at end of file +d97c324eb1d870c994911c53fbf84205f4e3e7a1 \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index f51dca0d84..41cc439846 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -7411,35 +7411,36 @@ int sqlite3_vtab_on_conflict(sqlite3 *); /* #define SQLITE_ABORT 4 // Also an error code */ #define SQLITE_REPLACE 5 -/* CAPI3REF: Prepared Statement Scan Status Opcodes -** KEYWORDS: {scanstatus option} +/* +** CAPI3REF: Prepared Statement Scan Status Opcodes +** KEYWORDS: {scanstatus options} ** ** The following constants can be used for the T parameter to the ** [sqlite3_stmt_scanstatus(S,X,T,V)] interface. Each constant designates a ** different metric for sqlite3_stmt_scanstatus() to return. ** **
-** [[SQLITE_SCANSTAT_NLOOP]]
SQLITE_SCANSTAT_NLOOP -**
The [sqlite3_int64] variable pointed to by the T parameter will be set to the +** [[SQLITE_SCANSTAT_NLOOP]]
SQLITE_SCANSTAT_NLOOP
+**
^The [sqlite3_int64] variable pointed to by the T parameter will be set to the ** total number of times that the X-th loop has run.
** -** [[SQLITE_SCANSTAT_NVISIT]] 		SQLITE_SCANSTAT_NVISIT -**
The [sqlite3_int64] variable pointed to by the T parameter will be set to the +** [[SQLITE_SCANSTAT_NVISIT]]
SQLITE_SCANSTAT_NVISIT
+**
^The [sqlite3_int64] variable pointed to by the T parameter will be set to the ** total number of rows visited by the X-th loop.
** -** [[SQLITE_SCANSTAT_EST]] 		SQLITE_SCANSTAT_EST -**
The [sqlite3_int64] variable pointed to by the T parameter will be set to the +** [[SQLITE_SCANSTAT_EST]]
SQLITE_SCANSTAT_EST
+**
^The [sqlite3_int64] variable pointed to by the T parameter will be set to the ** query planner's estimate for the number of rows visited for each ** iteration of the X-th loop. If the query planner's estimate was accurate, ** then this value should be approximately NVISIT/NLOOP. ** -** [[SQLITE_SCANSTAT_NAME]]
SQLITE_SCANSTAT_NAME -**
The "const char *" variable pointed to by the T parameter will be set to +** [[SQLITE_SCANSTAT_NAME]]
SQLITE_SCANSTAT_NAME
+**
^The "const char *" variable pointed to by the T parameter will be set to ** a zero-terminated UTF-8 string containing the name of the index or table used ** for the X-th loop. ** -** [[SQLITE_SCANSTAT_EXPLAIN]]
SQLITE_SCANSTAT_EXPLAIN -**
The "const char *" variable pointed to by the T parameter will be set to +** [[SQLITE_SCANSTAT_EXPLAIN]]
SQLITE_SCANSTAT_EXPLAIN
+**
^The "const char *" variable pointed to by the T parameter will be set to ** a zero-terminated UTF-8 string containing the [EXPLAIN QUERY PLAN] description ** for the X-th loop. ** @@ -7458,21 +7459,23 @@ int sqlite3_vtab_on_conflict(sqlite3 *); ** The "iScanStatusOp" parameter determines which status information to return. ** The "iScanStatusOp" must be one of the [scanstatus options] or the behavior of ** this interface is undefined. -** The requested measurement is written into a variable pointed to by +** ^The requested measurement is written into a variable pointed to by ** the "pOut" parameter. ** Parameter "idx" identifies the specific loop to retrieve statistics for. -** Loops are numbered starting from zero. If idx is out of range - less than +** Loops are numbered starting from zero. ^If idx is out of range - less than ** zero or greater than or equal to the total number of loops used to implement ** the statement - a non-zero value is returned and the variable that pOut ** points to is unchanged. ** -** Statistics might not be available for all loops in all statements. In cases +** ^Statistics might not be available for all loops in all statements. ^In cases ** where there exist loops with no available statistics, this function behaves ** as if the loop did not exist - it returns non-zero and leave the variable ** that pOut points to unchanged. ** ** This API is only available if the library is built with pre-processor ** symbol [SQLITE_ENABLE_STMT_SCANSTATUS] defined. +** +** See also: [sqlite3_stmt_scanstatus_reset()] */ SQLITE_EXPERIMENTAL int sqlite3_stmt_scanstatus( sqlite3_stmt *pStmt, /* Prepared statement for which info desired */ @@ -7484,7 +7487,7 @@ SQLITE_EXPERIMENTAL int sqlite3_stmt_scanstatus( /* ** CAPI3REF: Zero Scan-Status Counters ** -** Zero all sqlite3_stmt_scanstatus() related event counters. +** ^Zero all [sqlite3_stmt_scanstatus()] related event counters. ** ** This API is only available if the library is built with pre-processor ** symbol [SQLITE_ENABLE_STMT_SCANSTATUS] defined. From def6889d214a04780390df0aa77efcd5011620cf Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 4 Nov 2014 12:11:23 +0000 Subject: [PATCH 069/133] Add the SQLITE_CONFIG_PCACHE_HDRSZ option for sqlite3_config(). FossilOrigin-Name: 6eb03e62a34e8e0964175283587247b0212db604 --- manifest | 24 ++++++++++++------------ manifest.uuid | 2 +- src/btree.c | 5 +++++ src/btree.h | 1 + src/main.c | 9 +++++++++ src/pcache.c | 7 +++++++ src/pcache.h | 4 ++++ src/pcache1.c | 5 +++++ src/sqlite.h.in | 36 ++++++++++++++++++++++++------------ 9 files changed, 68 insertions(+), 25 deletions(-) diff --git a/manifest b/manifest index 8d217ecfe0..2d1da1580d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Updates\sto\sthe\ssqlite3_stmt_scanstatus()\sdocumentation.\s\sNo\schanges\sto\scode. -D 2014-11-03T18:03:00.467 +C Add\sthe\sSQLITE_CONFIG_PCACHE_HDRSZ\soption\sfor\ssqlite3_config(). +D 2014-11-04T12:11:23.751 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,8 +172,8 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 7f841396adfd47507ff670a471162d2bfcda3136 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 61d96c2edacc5267fae6e477c24c774cd540d7f0 -F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 +F src/btree.c 3cc918768d100f0efea62404b8f4e779b8c94731 +F src/btree.h 49b408be9c1cd41249076898e0673711071205d8 F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 67bb05b1077e0cdaccb2e36bfcbe7a5df9ed31e8 F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 @@ -194,7 +194,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 F src/loadext.c de741e66e5ddc1598d904d7289239696e40ed994 -F src/main.c 5f659bdb14cdba1c3b587b6ed09167c147e308a1 +F src/main.c 8903165064534858a79ae06615ea7f4b482ad482 F src/malloc.c 3c3ac67969612493d435e14b6832793209afd2ec F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c faf615aafd8be74a71494dfa027c113ea5c6615f @@ -218,9 +218,9 @@ F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 F src/pager.c 8d97b3633f098fef817656dcbf167ca904511d78 F src/pager.h d1eee3c3f741be247ce6d82752a178515fc8578b F src/parse.y 5dfead8aed90cb0c7c1115898ee2266804daff45 -F src/pcache.c 4121a0571c18581ee9f82f086d5e2030051ebd6a -F src/pcache.h 9b559127b83f84ff76d735c8262f04853be0c59a -F src/pcache1.c e412cb585f777c840ddce0500eddc5c6043c2bb5 +F src/pcache.c ace1b67632deeaa84859b4c16c27711dfb7db3d4 +F src/pcache.h b44658c9c932d203510279439d891a2a83e12ba8 +F src/pcache1.c facbdd3ecc09c8f750089d941305694301328e98 F src/pragma.c 3f3e959390a10c0131676f0e307acce372777e0f F src/prepare.c b7b7bf020bd4c962f7c8aed5a3c542c7dfe9f9c7 F src/printf.c 9e75a6a0b55bf61cfff7d7e19d89834a1b938236 @@ -229,7 +229,7 @@ F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c 282f8f5278e0c78eb442217531172ec9e1538796 -F src/sqlite.h.in 11f33a3e968a9637d6fa4ae879b83edd171ac88f +F src/sqlite.h.in 915b5a955dd2f1d3c7e848f721af48ccc47b972b F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 8f67ca79e957b8ece7453b8e320b6a996e1b4761 @@ -1211,7 +1211,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ab3b0fc5760c6d428dbe1f974726a7d3526640bc -R e3e372d3594a5227dc7ce2380d71aae3 +P d97c324eb1d870c994911c53fbf84205f4e3e7a1 +R 9efeed3832394f98added1ade80fb8e9 U drh -Z 7bcc3180ea33e48aa6d0e7855da7bfd4 +Z 388d5bd03930b5f393bf21b730c0228b diff --git a/manifest.uuid b/manifest.uuid index 0611c8a12e..d3cdc0284b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d97c324eb1d870c994911c53fbf84205f4e3e7a1 \ No newline at end of file +6eb03e62a34e8e0964175283587247b0212db604 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 9300a6a54f..f815e85a6d 100644 --- a/src/btree.c +++ b/src/btree.c @@ -8965,3 +8965,8 @@ void sqlite3BtreeCursorHints(BtCursor *pCsr, unsigned int mask){ int sqlite3BtreeIsReadonly(Btree *p){ return (p->pBt->btsFlags & BTS_READ_ONLY)!=0; } + +/* +** Return the size of the header added to each page by this module. +*/ +int sqlite3HeaderSizeBtree(void){ return sizeof(MemPage); } diff --git a/src/btree.h b/src/btree.h index 38abdca1a2..63337b1946 100644 --- a/src/btree.h +++ b/src/btree.h @@ -196,6 +196,7 @@ void sqlite3BtreeClearCursor(BtCursor *); int sqlite3BtreeSetVersion(Btree *pBt, int iVersion); void sqlite3BtreeCursorHints(BtCursor *, unsigned int mask); int sqlite3BtreeIsReadonly(Btree *pBt); +int sqlite3HeaderSizeBtree(void); #ifndef NDEBUG int sqlite3BtreeCursorIsValid(BtCursor*); diff --git a/src/main.c b/src/main.c index 65a662ee5a..6a896b2a3d 100644 --- a/src/main.c +++ b/src/main.c @@ -401,6 +401,15 @@ int sqlite3_config(int op, ...){ sqlite3GlobalConfig.nPage = va_arg(ap, int); break; } + case SQLITE_CONFIG_PCACHE_HDRSZ: { + /* Return the total size of all headers added to each page + ** of the page cache */ + *va_arg(ap, int*) = + sqlite3HeaderSizeBtree() + + sqlite3HeaderSizePcache() + + sqlite3HeaderSizePcache1(); + break; + } case SQLITE_CONFIG_PCACHE: { /* no-op */ diff --git a/src/pcache.c b/src/pcache.c index 191a9d00f4..13551872d1 100644 --- a/src/pcache.c +++ b/src/pcache.c @@ -651,6 +651,13 @@ void sqlite3PcacheShrink(PCache *pCache){ sqlite3GlobalConfig.pcache2.xShrink(pCache->pCache); } +/* +** Return the size of the header added by this middleware layer +** in the page-cache hierarchy. +*/ +int sqlite3HeaderSizePcache(void){ return sizeof(PgHdr); } + + #if defined(SQLITE_CHECK_PAGES) || defined(SQLITE_DEBUG) /* ** For all dirty pages currently in the cache, invoke the specified diff --git a/src/pcache.h b/src/pcache.h index dd9bfc7451..9ed62a88ff 100644 --- a/src/pcache.h +++ b/src/pcache.h @@ -160,4 +160,8 @@ void sqlite3PcacheStats(int*,int*,int*,int*); void sqlite3PCacheSetDefault(void); +/* Return the header size */ +int sqlite3HeaderSizePcache(void); +int sqlite3HeaderSizePcache1(void); + #endif /* _PCACHE_H_ */ diff --git a/src/pcache1.c b/src/pcache1.c index a8c3217382..cad41b1b73 100644 --- a/src/pcache1.c +++ b/src/pcache1.c @@ -981,6 +981,11 @@ void sqlite3PCacheSetDefault(void){ sqlite3_config(SQLITE_CONFIG_PCACHE2, &defaultMethods); } +/* +** Return the size of the header on each page of this PCACHE implementation. +*/ +int sqlite3HeaderSizePcache1(void){ return sizeof(PgHdr1); } + #ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT /* ** This function is called to free superfluous dynamically allocated memory diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 41cc439846..13373afe4a 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -1559,28 +1559,30 @@ struct sqlite3_mem_methods { **
^This option specifies a static memory buffer that SQLite can use for ** the database page cache with the default page cache implementation. ** This configuration should not be used if an application-define page -** cache implementation is loaded using the SQLITE_CONFIG_PCACHE2 option. -** There are three arguments to this option: A pointer to 8-byte aligned +** cache implementation is loaded using the [SQLITE_CONFIG_PCACHE2] +** configuration option. +** ^There are three arguments to this option: A pointer to 8-byte aligned ** memory, the size of each page buffer (sz), and the number of pages (N). ** The sz argument should be the size of the largest database page -** (a power of two between 512 and 32768) plus a little extra for each -** page header. ^The page header size is 20 to 40 bytes depending on -** the host architecture. ^It is harmless, apart from the wasted memory, -** to make sz a little too large. The first -** argument should point to an allocation of at least sz*N bytes of memory. +** (a power of two between 512 and 32768) plus some extra bytes for each +** page header. ^The number of extra bytes needed by the page header +** can be determined using the [SQLITE_CONFIG_PCACHE_HDRSZ] option +** to [sqlite3_config()]. +** ^It is harmless, apart from the wasted memory, +** for the sz parameter to be larger than necessary. The first +** argument should pointer to an 8-byte aligned block of memory that +** is at least sz*N bytes of memory, otherwise subsequent behavior is +** undefined. ** ^SQLite will use the memory provided by the first argument to satisfy its ** memory needs for the first N pages that it adds to cache. ^If additional ** page cache memory is needed beyond what is provided by this option, then -** SQLite goes to [sqlite3_malloc()] for the additional storage space. -** The pointer in the first argument must -** be aligned to an 8-byte boundary or subsequent behavior of SQLite -** will be undefined.
+** SQLite goes to [sqlite3_malloc()] for the additional storage space. ** ** [[SQLITE_CONFIG_HEAP]]
SQLITE_CONFIG_HEAP
**
^This option specifies a static memory buffer that SQLite will use ** for all of its dynamic memory allocation needs beyond those provided ** for by [SQLITE_CONFIG_SCRATCH] and [SQLITE_CONFIG_PAGECACHE]. -** There are three arguments: An 8-byte aligned pointer to the memory, +** ^There are three arguments: An 8-byte aligned pointer to the memory, ** the number of bytes in the memory buffer, and the minimum allocation size. ** ^If the first pointer (the memory pointer) is NULL, then SQLite reverts ** to using its default memory allocator (the system malloc() implementation), @@ -1728,6 +1730,15 @@ struct sqlite3_mem_methods { ** SQLITE_CONFIG_WIN32_HEAPSIZE takes a 32-bit unsigned integer value ** that specifies the maximum size of the created heap. ** +** +** [[SQLITE_CONFIG_PCACHE_HDRSZ]] +**
SQLITE_CONFIG_PCACHE_HDRSZ +**
^This option takes a single parameter which is a pointer to an integer +** and writes into that integer the number of extra bytes per page required +** for each page in [SQLITE_CONFIG_PAGECACHE]. The amount of +** extra space required can change depending on the compiler, +** target platform, and SQLite version. +** */ #define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ #define SQLITE_CONFIG_MULTITHREAD 2 /* nil */ @@ -1752,6 +1763,7 @@ struct sqlite3_mem_methods { #define SQLITE_CONFIG_SQLLOG 21 /* xSqllog, void* */ #define SQLITE_CONFIG_MMAP_SIZE 22 /* sqlite3_int64, sqlite3_int64 */ #define SQLITE_CONFIG_WIN32_HEAPSIZE 23 /* int nByte */ +#define SQLITE_CONFIG_PCACHE_HDRSZ 24 /* int *psz */ /* ** CAPI3REF: Database Connection Configuration Options From 5279d3433c2ef6089559eb65d3ed1a3e298cc3b3 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 4 Nov 2014 13:41:32 +0000 Subject: [PATCH 070/133] Improved documentation and addition of source-code evidence marks for the sqlite3_config() interface. FossilOrigin-Name: 681031a436fdd4cce426d6cd43cbae6b83167d26 --- manifest | 16 ++++---- manifest.uuid | 2 +- src/btree.c | 8 +++- src/main.c | 45 ++++++++++++++++++----- src/sqlite.h.in | 98 ++++++++++++++++++++++++++----------------------- 5 files changed, 103 insertions(+), 66 deletions(-) diff --git a/manifest b/manifest index 2d1da1580d..c02ddb43b9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sSQLITE_CONFIG_PCACHE_HDRSZ\soption\sfor\ssqlite3_config(). -D 2014-11-04T12:11:23.751 +C Improved\sdocumentation\sand\saddition\sof\ssource-code\sevidence\smarks\sfor\sthe\nsqlite3_config()\sinterface. +D 2014-11-04T13:41:32.444 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 7f841396adfd47507ff670a471162d2bfcda3136 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 3cc918768d100f0efea62404b8f4e779b8c94731 +F src/btree.c 5b6e02a2cb69bfba5d44d0a093b8cc7468fdcf61 F src/btree.h 49b408be9c1cd41249076898e0673711071205d8 F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 67bb05b1077e0cdaccb2e36bfcbe7a5df9ed31e8 @@ -194,7 +194,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 F src/loadext.c de741e66e5ddc1598d904d7289239696e40ed994 -F src/main.c 8903165064534858a79ae06615ea7f4b482ad482 +F src/main.c c673d0972da4b6826a3e907029faf5be0840cd57 F src/malloc.c 3c3ac67969612493d435e14b6832793209afd2ec F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c faf615aafd8be74a71494dfa027c113ea5c6615f @@ -229,7 +229,7 @@ F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c 282f8f5278e0c78eb442217531172ec9e1538796 -F src/sqlite.h.in 915b5a955dd2f1d3c7e848f721af48ccc47b972b +F src/sqlite.h.in 81c5105a19bd1c16a06d59e729ddc3bb0c26d003 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 8f67ca79e957b8ece7453b8e320b6a996e1b4761 @@ -1211,7 +1211,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P d97c324eb1d870c994911c53fbf84205f4e3e7a1 -R 9efeed3832394f98added1ade80fb8e9 +P 6eb03e62a34e8e0964175283587247b0212db604 +R a92bedb66491253840f18dd40e991665 U drh -Z 388d5bd03930b5f393bf21b730c0228b +Z 6591907c91e192439bb8b0655576863e diff --git a/manifest.uuid b/manifest.uuid index d3cdc0284b..e57823b45e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6eb03e62a34e8e0964175283587247b0212db604 \ No newline at end of file +681031a436fdd4cce426d6cd43cbae6b83167d26 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index f815e85a6d..6fe8e10f3d 100644 --- a/src/btree.c +++ b/src/btree.c @@ -6649,7 +6649,13 @@ static int balance_nonroot( nMaxCells*sizeof(u8*) /* apCell */ + nMaxCells*sizeof(u16) /* szCell */ + pBt->pageSize; /* aSpace1 */ - assert( szScratch<=16896 || szScratch<=6*pBt->pageSize ); + + /* EVIDENCE-OF: R-37926-08392 SQLite will never request a scratch buffer + ** that is more than 6 times the database page size, except when + ** performing a checkpoint in WAL mode when the scratch buffer request + ** size is a small fraction of the size of the WAL file. */ + assert( szScratch<=6*pBt->pageSize ); + apCell = sqlite3ScratchMalloc( szScratch ); if( apCell==0 ){ rc = SQLITE_NOMEM; diff --git a/src/main.c b/src/main.c index 6a896b2a3d..80800d7cf6 100644 --- a/src/main.c +++ b/src/main.c @@ -372,38 +372,54 @@ int sqlite3_config(int op, ...){ #endif case SQLITE_CONFIG_MALLOC: { - /* Specify an alternative malloc implementation */ + /* EVIDENCE-OF: R-55594-21030 The SQLITE_CONFIG_MALLOC option takes a + ** single argument which is a pointer to an instance of the + ** sqlite3_mem_methods structure. The argument specifies alternative + ** low-level memory allocation routines to be used in place of the memory + ** allocation routines built into SQLite. */ sqlite3GlobalConfig.m = *va_arg(ap, sqlite3_mem_methods*); break; } case SQLITE_CONFIG_GETMALLOC: { - /* Retrieve the current malloc() implementation */ + /* EVIDENCE-OF: R-51213-46414 The SQLITE_CONFIG_GETMALLOC option takes a + ** single argument which is a pointer to an instance of the + ** sqlite3_mem_methods structure. The sqlite3_mem_methods structure is + ** filled with the currently defined memory allocation routines. */ if( sqlite3GlobalConfig.m.xMalloc==0 ) sqlite3MemSetDefault(); *va_arg(ap, sqlite3_mem_methods*) = sqlite3GlobalConfig.m; break; } case SQLITE_CONFIG_MEMSTATUS: { - /* Enable or disable the malloc status collection */ - sqlite3GlobalConfig.bMemstat = va_arg(ap, int); /* IMP: R-27464-47829 */ + /* EVIDENCE-OF: R-61275-35157 The SQLITE_CONFIG_MEMSTATUS option takes + ** single argument of type int, interpreted as a boolean, which enables + ** or disables the collection of memory allocation statistics. */ + sqlite3GlobalConfig.bMemstat = va_arg(ap, int); break; } case SQLITE_CONFIG_SCRATCH: { - /* Designate a buffer for scratch memory space */ + /* EVIDENCE-OF: R-08404-60887 There are three arguments to + ** SQLITE_CONFIG_SCRATCH: A pointer an 8-byte aligned memory buffer from + ** which the scratch allocations will be drawn, the size of each scratch + ** allocation (sz), and the maximum number of scratch allocations (N). */ sqlite3GlobalConfig.pScratch = va_arg(ap, void*); sqlite3GlobalConfig.szScratch = va_arg(ap, int); sqlite3GlobalConfig.nScratch = va_arg(ap, int); break; } case SQLITE_CONFIG_PAGECACHE: { - /* Designate a buffer for page cache memory space */ + /* EVIDENCE-OF: R-31408-40510 There are three arguments to + ** SQLITE_CONFIG_PAGECACHE: A pointer to 8-byte aligned memory, the size + ** of each page buffer (sz), and the number of pages (N). */ sqlite3GlobalConfig.pPage = va_arg(ap, void*); sqlite3GlobalConfig.szPage = va_arg(ap, int); sqlite3GlobalConfig.nPage = va_arg(ap, int); break; } case SQLITE_CONFIG_PCACHE_HDRSZ: { - /* Return the total size of all headers added to each page - ** of the page cache */ + /* EVIDENCE-OF: R-39100-27317 The SQLITE_CONFIG_PCACHE_HDRSZ option takes + ** a single parameter which is a pointer to an integer and writes into + ** that integer the number of extra bytes per page required for each page + ** in SQLITE_CONFIG_PAGECACHE. */ *va_arg(ap, int*) = sqlite3HeaderSizeBtree() + sqlite3HeaderSizePcache() + @@ -422,11 +438,18 @@ int sqlite3_config(int op, ...){ } case SQLITE_CONFIG_PCACHE2: { - /* Specify an alternative page cache implementation */ + /* EVIDENCE-OF: R-63325-48378 The SQLITE_CONFIG_PCACHE2 option takes a + ** single argument which is a pointer to an sqlite3_pcache_methods2 + ** object. This object specifies the interface to a custom page cache + ** implementation. */ sqlite3GlobalConfig.pcache2 = *va_arg(ap, sqlite3_pcache_methods2*); break; } case SQLITE_CONFIG_GETPCACHE2: { + /* EVIDENCE-OF: R-22035-46182 The SQLITE_CONFIG_GETPCACHE2 option takes a + ** single argument which is a pointer to an sqlite3_pcache_methods2 + ** object. SQLite copies of the current page cache implementation into + ** that object. */ if( sqlite3GlobalConfig.pcache2.xInit==0 ){ sqlite3PCacheSetDefault(); } @@ -436,7 +459,9 @@ int sqlite3_config(int op, ...){ #if defined(SQLITE_ENABLE_MEMSYS3) || defined(SQLITE_ENABLE_MEMSYS5) case SQLITE_CONFIG_HEAP: { - /* Designate a buffer for heap memory space */ + /* EVIDENCE-OF: R-19854-42126 There are three arguments to + ** SQLITE_CONFIG_HEAP: An 8-byte aligned pointer to the memory, the + ** number of bytes in the memory buffer, and the minimum allocation size. */ sqlite3GlobalConfig.pHeap = va_arg(ap, void*); sqlite3GlobalConfig.nHeap = va_arg(ap, int); sqlite3GlobalConfig.mnReq = va_arg(ap, int); diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 13373afe4a..271b7a0ad1 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -1499,25 +1499,27 @@ struct sqlite3_mem_methods { ** SQLITE_CONFIG_SERIALIZED configuration option.
** ** [[SQLITE_CONFIG_MALLOC]]
SQLITE_CONFIG_MALLOC
-**
^(This option takes a single argument which is a pointer to an -** instance of the [sqlite3_mem_methods] structure. The argument specifies +**
^(The SQLITE_CONFIG_MALLOC option takes a single argument which is +** a pointer to an instance of the [sqlite3_mem_methods] structure. +** The argument specifies ** alternative low-level memory allocation routines to be used in place of ** the memory allocation routines built into SQLite.)^ ^SQLite makes ** its own private copy of the content of the [sqlite3_mem_methods] structure ** before the [sqlite3_config()] call returns.
** ** [[SQLITE_CONFIG_GETMALLOC]]
SQLITE_CONFIG_GETMALLOC
-**
^(This option takes a single argument which is a pointer to an -** instance of the [sqlite3_mem_methods] structure. The [sqlite3_mem_methods] +**
^(The SQLITE_CONFIG_GETMALLOC option takes a single argument which +** is a pointer to an instance of the [sqlite3_mem_methods] structure. +** The [sqlite3_mem_methods] ** structure is filled with the currently defined memory allocation routines.)^ ** This option can be used to overload the default memory allocation ** routines with a wrapper that simulations memory allocation failure or ** tracks memory usage, for example.
** ** [[SQLITE_CONFIG_MEMSTATUS]]
SQLITE_CONFIG_MEMSTATUS
-**
^This option takes single argument of type int, interpreted as a -** boolean, which enables or disables the collection of memory allocation -** statistics. ^(When memory allocation statistics are disabled, the +**
^The SQLITE_CONFIG_MEMSTATUS option takes single argument of type int, +** interpreted as a boolean, which enables or disables the collection of +** memory allocation statistics. ^(When memory allocation statistics are disabled, the ** following SQLite interfaces become non-operational: **
    **
  • [sqlite3_memory_used()] @@ -1531,8 +1533,9 @@ struct sqlite3_mem_methods { **
** ** [[SQLITE_CONFIG_SCRATCH]]
SQLITE_CONFIG_SCRATCH
-**
^This option specifies a static memory buffer that SQLite can use for -** scratch memory. ^(There are three arguments: A pointer an 8-byte +**
^The SQLITE_CONFIG_SCRATCH option specifies a static memory buffer +** that SQLite can use for scratch memory. ^(There are three arguments +** to SQLITE_CONFIG_SCRATCH: A pointer an 8-byte ** aligned memory buffer from which the scratch allocations will be ** drawn, the size of each scratch allocation (sz), ** and the maximum number of scratch allocations (N).)^ @@ -1556,12 +1559,13 @@ struct sqlite3_mem_methods { **
** ** [[SQLITE_CONFIG_PAGECACHE]]
SQLITE_CONFIG_PAGECACHE
-**
^This option specifies a static memory buffer that SQLite can use for -** the database page cache with the default page cache implementation. +**
^The SQLITE_CONFIG_PAGECACHE option specifies a static memory buffer +** that SQLite can use for the database page cache with the default page +** cache implementation. ** This configuration should not be used if an application-define page ** cache implementation is loaded using the [SQLITE_CONFIG_PCACHE2] ** configuration option. -** ^There are three arguments to this option: A pointer to 8-byte aligned +** ^There are three arguments to SQLITE_CONFIG_PAGECACHE: A pointer to 8-byte aligned ** memory, the size of each page buffer (sz), and the number of pages (N). ** The sz argument should be the size of the largest database page ** (a power of two between 512 and 32768) plus some extra bytes for each @@ -1579,10 +1583,11 @@ struct sqlite3_mem_methods { ** SQLite goes to [sqlite3_malloc()] for the additional storage space.
** ** [[SQLITE_CONFIG_HEAP]]
SQLITE_CONFIG_HEAP
-**
^This option specifies a static memory buffer that SQLite will use -** for all of its dynamic memory allocation needs beyond those provided -** for by [SQLITE_CONFIG_SCRATCH] and [SQLITE_CONFIG_PAGECACHE]. -** ^There are three arguments: An 8-byte aligned pointer to the memory, +**
^The SQLITE_CONFIG_HEAP option specifies a static memory buffer +** that SQLite will use for all of its dynamic memory allocation needs +** beyond those provided for by [SQLITE_CONFIG_SCRATCH] and [SQLITE_CONFIG_PAGECACHE]. +** ^There are three arguments to SQLITE_CONFIG_HEAP: +** An 8-byte aligned pointer to the memory, ** the number of bytes in the memory buffer, and the minimum allocation size. ** ^If the first pointer (the memory pointer) is NULL, then SQLite reverts ** to using its default memory allocator (the system malloc() implementation), @@ -1596,9 +1601,9 @@ struct sqlite3_mem_methods { ** for the minimum allocation size are 2**5 through 2**8.
** ** [[SQLITE_CONFIG_MUTEX]]
SQLITE_CONFIG_MUTEX
-**
^(This option takes a single argument which is a pointer to an -** instance of the [sqlite3_mutex_methods] structure. The argument specifies -** alternative low-level mutex routines to be used in place +**
^(The SQLITE_CONFIG_MUTEX option takes a single argument which is a +** pointer to an instance of the [sqlite3_mutex_methods] structure. +** The argument specifies alternative low-level mutex routines to be used in place ** the mutex routines built into SQLite.)^ ^SQLite makes a copy of the ** content of the [sqlite3_mutex_methods] structure before the call to ** [sqlite3_config()] returns. ^If SQLite is compiled with @@ -1608,8 +1613,8 @@ struct sqlite3_mem_methods { ** return [SQLITE_ERROR].
** ** [[SQLITE_CONFIG_GETMUTEX]]
SQLITE_CONFIG_GETMUTEX
-**
^(This option takes a single argument which is a pointer to an -** instance of the [sqlite3_mutex_methods] structure. The +**
^(The SQLITE_CONFIG_GETMUTEX option takes a single argument which +** is a pointer to an instance of the [sqlite3_mutex_methods] structure. The ** [sqlite3_mutex_methods] ** structure is filled with the currently defined mutex routines.)^ ** This option can be used to overload the default mutex allocation @@ -1621,24 +1626,24 @@ struct sqlite3_mem_methods { ** return [SQLITE_ERROR].
** ** [[SQLITE_CONFIG_LOOKASIDE]]
SQLITE_CONFIG_LOOKASIDE
-**
^(This option takes two arguments that determine the default -** memory allocation for the lookaside memory allocator on each -** [database connection]. The first argument is the +**
^(The SQLITE_CONFIG_LOOKASIDE option takes two arguments that determine +** the default size of lookaside memory on each [database connection]. +** The first argument is the ** size of each lookaside buffer slot and the second is the number of -** slots allocated to each database connection.)^ ^(This option sets the -** default lookaside size. The [SQLITE_DBCONFIG_LOOKASIDE] -** verb to [sqlite3_db_config()] can be used to change the lookaside +** slots allocated to each database connection.)^ ^(SQLITE_CONFIG_LOOKASIDE +** sets the default lookaside size. The [SQLITE_DBCONFIG_LOOKASIDE] +** option to [sqlite3_db_config()] can be used to change the lookaside ** configuration on individual connections.)^
** ** [[SQLITE_CONFIG_PCACHE2]]
SQLITE_CONFIG_PCACHE2
-**
^(This option takes a single argument which is a pointer to -** an [sqlite3_pcache_methods2] object. This object specifies the interface -** to a custom page cache implementation.)^ ^SQLite makes a copy of the -** object and uses it for page cache memory allocations.
+**
^(The SQLITE_CONFIG_PCACHE2 option takes a single argument which is +** a pointer to an [sqlite3_pcache_methods2] object. This object specifies +** the interface to a custom page cache implementation.)^ +** ^SQLite makes a copy of the [sqlite3_pcache_methods2] object.
** ** [[SQLITE_CONFIG_GETPCACHE2]]
SQLITE_CONFIG_GETPCACHE2
-**
^(This option takes a single argument which is a pointer to an -** [sqlite3_pcache_methods2] object. SQLite copies of the current +**
^(The SQLITE_CONFIG_GETPCACHE2 option takes a single argument which +** is a pointer to an [sqlite3_pcache_methods2] object. SQLite copies of the current ** page cache implementation into that object.)^
** ** [[SQLITE_CONFIG_LOG]]
SQLITE_CONFIG_LOG
@@ -1662,10 +1667,10 @@ struct sqlite3_mem_methods { ** function must be threadsafe. ** ** [[SQLITE_CONFIG_URI]]
SQLITE_CONFIG_URI -**
^(This option takes a single argument of type int. If non-zero, then -** URI handling is globally enabled. If the parameter is zero, then URI handling -** is globally disabled.)^ ^If URI handling is globally enabled, all filenames -** passed to [sqlite3_open()], [sqlite3_open_v2()], [sqlite3_open16()] or +**
^(The SQLITE_CONFIG_URI option takes a single argument of type int. +** If non-zero, then URI handling is globally enabled. If the parameter is zero, +** then URI handling is globally disabled.)^ ^If URI handling is globally enabled, +** all filenames passed to [sqlite3_open()], [sqlite3_open_v2()], [sqlite3_open16()] or ** specified as part of [ATTACH] commands are interpreted as URIs, regardless ** of whether or not the [SQLITE_OPEN_URI] flag is set when the database ** connection is opened. ^If it is globally disabled, filenames are @@ -1675,9 +1680,10 @@ struct sqlite3_mem_methods { ** [SQLITE_USE_URI] symbol defined.)^ ** ** [[SQLITE_CONFIG_COVERING_INDEX_SCAN]]
SQLITE_CONFIG_COVERING_INDEX_SCAN -**
^This option takes a single integer argument which is interpreted as -** a boolean in order to enable or disable the use of covering indices for -** full table scans in the query optimizer. ^The default setting is determined +**
^The SQLITE_CONFIG_COVERING_INDEX_SCAN option takes a single integer +** argument which is interpreted as a boolean in order to enable or disable +** the use of covering indices for full table scans in the query optimizer. +** ^The default setting is determined ** by the [SQLITE_ALLOW_COVERING_INDEX_SCAN] compile-time option, or is "on" ** if that compile-time option is omitted. ** The ability to disable the use of covering indices for full table scans @@ -1725,17 +1731,17 @@ struct sqlite3_mem_methods { ** ** [[SQLITE_CONFIG_WIN32_HEAPSIZE]] **
SQLITE_CONFIG_WIN32_HEAPSIZE -**
^This option is only available if SQLite is compiled for Windows -** with the [SQLITE_WIN32_MALLOC] pre-processor macro defined. -** SQLITE_CONFIG_WIN32_HEAPSIZE takes a 32-bit unsigned integer value +**
^The SQLITE_CONFIG_WIN32_HEAPSIZE option is only available if SQLite is +** compiled for Windows with the [SQLITE_WIN32_MALLOC] pre-processor macro defined. +** ^SQLITE_CONFIG_WIN32_HEAPSIZE takes a 32-bit unsigned integer value ** that specifies the maximum size of the created heap. ** ** ** [[SQLITE_CONFIG_PCACHE_HDRSZ]] **
SQLITE_CONFIG_PCACHE_HDRSZ -**
^This option takes a single parameter which is a pointer to an integer -** and writes into that integer the number of extra bytes per page required -** for each page in [SQLITE_CONFIG_PAGECACHE]. The amount of +**
^The SQLITE_CONFIG_PCACHE_HDRSZ option takes a single parameter which +** is a pointer to an integer and writes into that integer the number of extra +** bytes per page required for each page in [SQLITE_CONFIG_PAGECACHE]. The amount of ** extra space required can change depending on the compiler, ** target platform, and SQLite version. ** From cbd55b0362c8dd2a0c7c3bd0bc56bd16ccd2d962 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 4 Nov 2014 14:22:27 +0000 Subject: [PATCH 071/133] Change the definition of SQLITE_CONFIG_SCRATCH so that at most one scratch buffer is used per thread. Use the generic heap memory allocator for the WalIterator object when running a checkpoint. FossilOrigin-Name: 391c9b85abcb5ba300fb2e116384639310c69ed2 --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/btree.c | 7 ++----- src/malloc.c | 11 ++++++----- src/sqlite.h.in | 8 ++------ src/wal.c | 8 ++++---- 6 files changed, 24 insertions(+), 30 deletions(-) diff --git a/manifest b/manifest index c02ddb43b9..9decec2f00 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\sdocumentation\sand\saddition\sof\ssource-code\sevidence\smarks\sfor\sthe\nsqlite3_config()\sinterface. -D 2014-11-04T13:41:32.444 +C Change\sthe\sdefinition\sof\sSQLITE_CONFIG_SCRATCH\sso\sthat\sat\smost\sone\sscratch\nbuffer\sis\sused\sper\sthread.\s\sUse\sthe\sgeneric\sheap\smemory\sallocator\sfor\sthe\nWalIterator\sobject\swhen\srunning\sa\scheckpoint. +D 2014-11-04T14:22:27.705 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 7f841396adfd47507ff670a471162d2bfcda3136 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 5b6e02a2cb69bfba5d44d0a093b8cc7468fdcf61 +F src/btree.c 11d1262110c2d459b68121833fa3ec6625b1d022 F src/btree.h 49b408be9c1cd41249076898e0673711071205d8 F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 67bb05b1077e0cdaccb2e36bfcbe7a5df9ed31e8 @@ -195,7 +195,7 @@ F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 F src/loadext.c de741e66e5ddc1598d904d7289239696e40ed994 F src/main.c c673d0972da4b6826a3e907029faf5be0840cd57 -F src/malloc.c 3c3ac67969612493d435e14b6832793209afd2ec +F src/malloc.c 740db54387204c9a2eb67c6d98e68b08e9ef4eab F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c faf615aafd8be74a71494dfa027c113ea5c6615f F src/mem2.c f1940d9e91948dd6a908fbb9ce3835c36b5d83c3 @@ -229,7 +229,7 @@ F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c 282f8f5278e0c78eb442217531172ec9e1538796 -F src/sqlite.h.in 81c5105a19bd1c16a06d59e729ddc3bb0c26d003 +F src/sqlite.h.in 6e9af739d79f0bea2584b70fb1c54d3bb1a2eab6 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 8f67ca79e957b8ece7453b8e320b6a996e1b4761 @@ -299,7 +299,7 @@ F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f F src/vdbesort.c daf87ea542df088ac4607660d09976d36b6bd95d F src/vdbetrace.c 7e4222955e07dd707a2f360c0eb73452be1cb010 F src/vtab.c 2a30791bbd7926b589401bd09c3abb33de563793 -F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 +F src/wal.c 825c948066c7604a07d56e67958cdab210749016 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 F src/where.c d5fa1081bf7cb70478ed06489e063695c92ee1e1 @@ -1211,7 +1211,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 6eb03e62a34e8e0964175283587247b0212db604 -R a92bedb66491253840f18dd40e991665 +P 681031a436fdd4cce426d6cd43cbae6b83167d26 +R cb98a95faa41015902ec00cd7a294542 U drh -Z 6591907c91e192439bb8b0655576863e +Z 113a55cc7642fbc8d6bb5b88b140b929 diff --git a/manifest.uuid b/manifest.uuid index e57823b45e..cd70e30b51 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -681031a436fdd4cce426d6cd43cbae6b83167d26 \ No newline at end of file +391c9b85abcb5ba300fb2e116384639310c69ed2 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 6fe8e10f3d..fe15c922ca 100644 --- a/src/btree.c +++ b/src/btree.c @@ -6650,12 +6650,9 @@ static int balance_nonroot( + nMaxCells*sizeof(u16) /* szCell */ + pBt->pageSize; /* aSpace1 */ - /* EVIDENCE-OF: R-37926-08392 SQLite will never request a scratch buffer - ** that is more than 6 times the database page size, except when - ** performing a checkpoint in WAL mode when the scratch buffer request - ** size is a small fraction of the size of the WAL file. */ + /* EVIDENCE-OF: R-28375-38319 SQLite will never request a scratch buffer + ** that is more than 6 times the database page size. */ assert( szScratch<=6*pBt->pageSize ); - apCell = sqlite3ScratchMalloc( szScratch ); if( apCell==0 ){ rc = SQLITE_NOMEM; diff --git a/src/malloc.c b/src/malloc.c index 6fb9d53d1b..4960f91e02 100644 --- a/src/malloc.c +++ b/src/malloc.c @@ -377,11 +377,12 @@ void *sqlite3ScratchMalloc(int n){ #if SQLITE_THREADSAFE==0 && !defined(NDEBUG) - /* Verify that no more than two scratch allocations per thread - ** are outstanding at one time. (This is only checked in the - ** single-threaded case since checking in the multi-threaded case - ** would be much more complicated.) */ - assert( scratchAllocOut<=1 ); + /* EVIDENCE-OF: R-12970-05880 SQLite will not use more than one scratch + ** buffers per thread. + ** + ** This can only be checked in single-threaded mode. + */ + assert( scratchAllocOut==0 ); if( p ) scratchAllocOut++; #endif diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 271b7a0ad1..44f7800ca2 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -1541,13 +1541,9 @@ struct sqlite3_mem_methods { ** and the maximum number of scratch allocations (N).)^ ** The first argument must be a pointer to an 8-byte aligned buffer ** of at least sz*N bytes of memory. -** ^SQLite will not use more than two scratch buffers per thread and not -** more than one scratch buffer per thread when not performing -** a [checkpoint] in [WAL mode]. +** ^SQLite will not use more than one scratch buffers per thread. ** ^SQLite will never request a scratch buffer that is more than 6 -** times the database page size, except when performing a [checkpoint] -** in [WAL mode] when the scratch buffer request size is a small fraction -** of the size of the WAL file. +** times the database page size. ** ^If SQLite needs needs additional ** scratch memory beyond what is provided by this configuration option, then ** [sqlite3_malloc()] will be used to obtain the memory needed.

diff --git a/src/wal.c b/src/wal.c index c0861d5be7..d2ed293a4d 100644 --- a/src/wal.c +++ b/src/wal.c @@ -1504,7 +1504,7 @@ static void walMergesort( ** Free an iterator allocated by walIteratorInit(). */ static void walIteratorFree(WalIterator *p){ - sqlite3ScratchFree(p); + sqlite3_free(p); } /* @@ -1539,7 +1539,7 @@ static int walIteratorInit(Wal *pWal, WalIterator **pp){ nByte = sizeof(WalIterator) + (nSegment-1)*sizeof(struct WalSegment) + iLast*sizeof(ht_slot); - p = (WalIterator *)sqlite3ScratchMalloc(nByte); + p = (WalIterator *)sqlite3_malloc(nByte); if( !p ){ return SQLITE_NOMEM; } @@ -1549,7 +1549,7 @@ static int walIteratorInit(Wal *pWal, WalIterator **pp){ /* Allocate temporary space used by the merge-sort routine. This block ** of memory will be freed before this function returns. */ - aTmp = (ht_slot *)sqlite3ScratchMalloc( + aTmp = (ht_slot *)sqlite3_malloc( sizeof(ht_slot) * (iLast>HASHTABLE_NPAGE?HASHTABLE_NPAGE:iLast) ); if( !aTmp ){ @@ -1586,7 +1586,7 @@ static int walIteratorInit(Wal *pWal, WalIterator **pp){ p->aSegment[i].aPgno = (u32 *)aPgno; } } - sqlite3ScratchFree(aTmp); + sqlite3_free(aTmp); if( rc!=SQLITE_OK ){ walIteratorFree(p); From 4d9f188f005b204e5e7aa4f60762ad67948d2f9c Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 4 Nov 2014 17:23:24 +0000 Subject: [PATCH 072/133] Add various requirements evidence marks for sqlite3_config() options. FossilOrigin-Name: d423349d2cd8bc7e04f3d90ca7bab11e1ad86e25 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/global.c | 8 ++++++++ src/main.c | 25 ++++++++++++++++++++----- src/vdbesort.c | 2 +- 5 files changed, 38 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 9decec2f00..e9f3481cba 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\sdefinition\sof\sSQLITE_CONFIG_SCRATCH\sso\sthat\sat\smost\sone\sscratch\nbuffer\sis\sused\sper\sthread.\s\sUse\sthe\sgeneric\sheap\smemory\sallocator\sfor\sthe\nWalIterator\sobject\swhen\srunning\sa\scheckpoint. -D 2014-11-04T14:22:27.705 +C Add\svarious\srequirements\sevidence\smarks\sfor\ssqlite3_config()\soptions. +D 2014-11-04T17:23:24.610 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -185,7 +185,7 @@ F src/expr.c 0391a657df4959eaf2a2fd7d77de5ebe750686ee F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c da985ae673efef2c712caef825a5d2edb087ead7 F src/func.c ba47c1671ab3cfdafa6e9d6ee490939ea578adee -F src/global.c 01c1f36ecfcf10770db648422a8852c222308bb9 +F src/global.c a50ad0b9ee328107a65aa8f5f3cd34905e74745c F src/hash.c 4263fbc955f26c2e8cdc0cf214bc42435aa4e4f5 F src/hash.h c8f3c31722cf3277d03713909761e152a5b81094 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 @@ -194,7 +194,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 F src/loadext.c de741e66e5ddc1598d904d7289239696e40ed994 -F src/main.c c673d0972da4b6826a3e907029faf5be0840cd57 +F src/main.c 42f857be3cef3e1f9752d8e46d61345f80932396 F src/malloc.c 740db54387204c9a2eb67c6d98e68b08e9ef4eab F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c faf615aafd8be74a71494dfa027c113ea5c6615f @@ -296,7 +296,7 @@ F src/vdbeapi.c 900259bdd85cd66a9f210d8ec08147a9034593bd F src/vdbeaux.c cf6b8152dd22155201d57c216e6266866b61da59 F src/vdbeblob.c 8b5442ff0954c44b45cbabbe2e94091a2e16fdef F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f -F src/vdbesort.c daf87ea542df088ac4607660d09976d36b6bd95d +F src/vdbesort.c 87f3923483113d1c95d84640becb4e4946f27d9a F src/vdbetrace.c 7e4222955e07dd707a2f360c0eb73452be1cb010 F src/vtab.c 2a30791bbd7926b589401bd09c3abb33de563793 F src/wal.c 825c948066c7604a07d56e67958cdab210749016 @@ -1211,7 +1211,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 681031a436fdd4cce426d6cd43cbae6b83167d26 -R cb98a95faa41015902ec00cd7a294542 +P 391c9b85abcb5ba300fb2e116384639310c69ed2 +R 59d979fd0dabfa4a2d3ae24d9471db8f U drh -Z 113a55cc7642fbc8d6bb5b88b140b929 +Z d72c7362e49ef35f8e5912118c20e6ff diff --git a/manifest.uuid b/manifest.uuid index cd70e30b51..969ed55bc1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -391c9b85abcb5ba300fb2e116384639310c69ed2 \ No newline at end of file +d423349d2cd8bc7e04f3d90ca7bab11e1ad86e25 \ No newline at end of file diff --git a/src/global.c b/src/global.c index e769eb425f..b5323e8df6 100644 --- a/src/global.c +++ b/src/global.c @@ -135,11 +135,19 @@ const unsigned char sqlite3CtypeMap[256] = { ** ** EVIDENCE-OF: R-38799-08373 URI filenames can be enabled or disabled ** using the SQLITE_USE_URI=1 or SQLITE_USE_URI=0 compile-time options. +** +** EVIDENCE-OF: R-43642-56306 By default, URI handling is globally +** disabled. The default value may be changed by compiling with the +** SQLITE_USE_URI symbol defined. */ #ifndef SQLITE_USE_URI # define SQLITE_USE_URI 0 #endif +/* EVIDENCE-OF: R-38720-18127 The default setting is determined by the +** SQLITE_ALLOW_COVERING_INDEX_SCAN compile-time option, or is "on" if +** that compile-time option is omitted. +*/ #ifndef SQLITE_ALLOW_COVERING_INDEX_SCAN # define SQLITE_ALLOW_COVERING_INDEX_SCAN 1 #endif diff --git a/src/main.c b/src/main.c index 80800d7cf6..4c33bbde9d 100644 --- a/src/main.c +++ b/src/main.c @@ -523,11 +523,19 @@ int sqlite3_config(int op, ...){ ** sqlite3_config(SQLITE_CONFIG_URI,0) configuration calls. */ case SQLITE_CONFIG_URI: { + /* EVIDENCE-OF: R-25451-61125 The SQLITE_CONFIG_URI option takes a single + ** argument of type int. If non-zero, then URI handling is globally + ** enabled. If the parameter is zero, then URI handling is globally + ** disabled. */ sqlite3GlobalConfig.bOpenUri = va_arg(ap, int); break; } case SQLITE_CONFIG_COVERING_INDEX_SCAN: { + /* EVIDENCE-OF: R-36592-02772 The SQLITE_CONFIG_COVERING_INDEX_SCAN + ** option takes a single integer argument which is interpreted as a + ** boolean in order to enable or disable the use of covering indices for + ** full table scans in the query optimizer. */ sqlite3GlobalConfig.bUseCis = va_arg(ap, int); break; } @@ -542,20 +550,27 @@ int sqlite3_config(int op, ...){ #endif case SQLITE_CONFIG_MMAP_SIZE: { + /* EVIDENCE-OF: R-58063-38258 SQLITE_CONFIG_MMAP_SIZE takes two 64-bit + ** integer (sqlite3_int64) values that are the default mmap size limit + ** (the default setting for PRAGMA mmap_size) and the maximum allowed + ** mmap size limit. */ sqlite3_int64 szMmap = va_arg(ap, sqlite3_int64); sqlite3_int64 mxMmap = va_arg(ap, sqlite3_int64); - if( mxMmap<0 || mxMmap>SQLITE_MAX_MMAP_SIZE ){ - mxMmap = SQLITE_MAX_MMAP_SIZE; - } - sqlite3GlobalConfig.mxMmap = mxMmap; + /* EVIDENCE-OF: R-53367-43190 If either argument to this option is + ** negative, then that argument is changed to its compile-time default. */ + if( mxMmap<0 || mxMmap>SQLITE_MAX_MMAP_SIZE ) mxMmap = SQLITE_MAX_MMAP_SIZE; if( szMmap<0 ) szMmap = SQLITE_DEFAULT_MMAP_SIZE; if( szMmap>mxMmap) szMmap = mxMmap; + sqlite3GlobalConfig.mxMmap = mxMmap; sqlite3GlobalConfig.szMmap = szMmap; break; } -#if SQLITE_OS_WIN && defined(SQLITE_WIN32_MALLOC) +#if SQLITE_OS_WIN && defined(SQLITE_WIN32_MALLOC) /* IMP: R-04780-55815 */ case SQLITE_CONFIG_WIN32_HEAPSIZE: { + /* EVIDENCE-OF: R-34926-03360 SQLITE_CONFIG_WIN32_HEAPSIZE takes a 32-bit + ** unsigned integer value that specifies the maximum size of the created + ** heap. */ sqlite3GlobalConfig.nHeap = va_arg(ap, int); break; } diff --git a/src/vdbesort.c b/src/vdbesort.c index 918d840716..df8357a57e 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -100,7 +100,7 @@ ** The sorter is running in multi-threaded mode if (a) the library was built ** with pre-processor symbol SQLITE_MAX_WORKER_THREADS set to a value greater ** than zero, and (b) worker threads have been enabled at runtime by calling -** sqlite3_config(SQLITE_CONFIG_WORKER_THREADS, ...). +** "PRAGMA threads=N" with some value of N greater than 0. ** ** When Rewind() is called, any data remaining in memory is flushed to a ** final PMA. So at this point the data is stored in some number of sorted From 73767829b9a224197d8d046f653535770bd97174 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Tue, 4 Nov 2014 19:37:22 +0000 Subject: [PATCH 073/133] For the Win32 VFS, allow memory mapped files to work when compiled without WAL support. FossilOrigin-Name: 1fc7e2f3d34e25e7b59aa8b51d10c1e27ab4a527 --- manifest | 17 ++++++++++------- manifest.uuid | 2 +- src/os_win.c | 27 +++++++++++++++++---------- 3 files changed, 28 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index e9f3481cba..74e32143e0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\svarious\srequirements\sevidence\smarks\sfor\ssqlite3_config()\soptions. -D 2014-11-04T17:23:24.610 +C For\sthe\sWin32\sVFS,\sallow\smemory\smapped\sfiles\sto\swork\swhen\scompiled\swithout\sWAL\ssupport. +D 2014-11-04T19:37:22.576 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -213,7 +213,7 @@ F src/os.h 3e57a24e2794a94d3cf2342c6d9a884888cd96bf F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c fb587121840f690101336879adfa6d0b2cd0e8c7 -F src/os_win.c a019caaae2bcbbc0cc4c39af6e7d7e43d8426053 +F src/os_win.c a9e500dd963fb1f67d7860e58b5772abe6123862 F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 F src/pager.c 8d97b3633f098fef817656dcbf167ca904511d78 F src/pager.h d1eee3c3f741be247ce6d82752a178515fc8578b @@ -1211,7 +1211,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 391c9b85abcb5ba300fb2e116384639310c69ed2 -R 59d979fd0dabfa4a2d3ae24d9471db8f -U drh -Z d72c7362e49ef35f8e5912118c20e6ff +P d423349d2cd8bc7e04f3d90ca7bab11e1ad86e25 +R c0de55f8c674bc2f58a7f862c94a6470 +T *branch * winMmapNoWal +T *sym-winMmapNoWal * +T -sym-trunk * +U mistachkin +Z 8ecf95ce02a024463c10f127838cfcbc diff --git a/manifest.uuid b/manifest.uuid index 969ed55bc1..e9b06734b6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d423349d2cd8bc7e04f3d90ca7bab11e1ad86e25 \ No newline at end of file +1fc7e2f3d34e25e7b59aa8b51d10c1e27ab4a527 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 8ca2107d90..2a7681c73f 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -34,6 +34,11 @@ with SQLITE_OMIT_WAL." #endif +#if !SQLITE_OS_WINNT && SQLITE_MAX_MMAP_SIZE>0 +# error "Memory mapped files require support from the Windows NT kernel,\ + compile with SQLITE_MAX_MMAP_SIZE=0." +#endif + /* ** Are most of the Win32 ANSI APIs available (i.e. with certain exceptions ** based on the sub-platform)? @@ -163,10 +168,11 @@ /* ** Do we need to manually define the Win32 file mapping APIs for use with WAL -** mode (e.g. these APIs are available in the Windows CE SDK; however, they -** are not present in the header file)? +** mode or memory mapped files (e.g. these APIs are available in the Windows +** CE SDK; however, they are not present in the header file)? */ -#if SQLITE_WIN32_FILEMAPPING_API && !defined(SQLITE_OMIT_WAL) +#if SQLITE_WIN32_FILEMAPPING_API && \ + (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0) /* ** Two of the file mapping APIs are different under WinRT. Figure out which ** set we need. @@ -194,7 +200,7 @@ WINBASEAPI LPVOID WINAPI MapViewOfFile(HANDLE, DWORD, DWORD, DWORD, SIZE_T); ** This file mapping API is common to both Win32 and WinRT. */ WINBASEAPI BOOL WINAPI UnmapViewOfFile(LPCVOID); -#endif /* SQLITE_WIN32_FILEMAPPING_API && !defined(SQLITE_OMIT_WAL) */ +#endif /* SQLITE_WIN32_FILEMAPPING_API */ /* ** Some Microsoft compilers lack this definition. @@ -487,7 +493,7 @@ static struct win_syscall { LPSECURITY_ATTRIBUTES,DWORD,DWORD,HANDLE))aSyscall[5].pCurrent) #if (!SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_ANSI) && \ - !defined(SQLITE_OMIT_WAL)) + (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0)) { "CreateFileMappingA", (SYSCALL)CreateFileMappingA, 0 }, #else { "CreateFileMappingA", (SYSCALL)0, 0 }, @@ -497,7 +503,7 @@ static struct win_syscall { DWORD,DWORD,DWORD,LPCSTR))aSyscall[6].pCurrent) #if SQLITE_OS_WINCE || (!SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) && \ - !defined(SQLITE_OMIT_WAL)) + (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0)) { "CreateFileMappingW", (SYSCALL)CreateFileMappingW, 0 }, #else { "CreateFileMappingW", (SYSCALL)0, 0 }, @@ -837,7 +843,8 @@ static struct win_syscall { LPOVERLAPPED))aSyscall[48].pCurrent) #endif -#if SQLITE_OS_WINCE || (!SQLITE_OS_WINRT && !defined(SQLITE_OMIT_WAL)) +#if SQLITE_OS_WINCE || (!SQLITE_OS_WINRT && \ + (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0)) { "MapViewOfFile", (SYSCALL)MapViewOfFile, 0 }, #else { "MapViewOfFile", (SYSCALL)0, 0 }, @@ -907,7 +914,7 @@ static struct win_syscall { #define osUnlockFileEx ((BOOL(WINAPI*)(HANDLE,DWORD,DWORD,DWORD, \ LPOVERLAPPED))aSyscall[58].pCurrent) -#if SQLITE_OS_WINCE || !defined(SQLITE_OMIT_WAL) +#if SQLITE_OS_WINCE || !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 { "UnmapViewOfFile", (SYSCALL)UnmapViewOfFile, 0 }, #else { "UnmapViewOfFile", (SYSCALL)0, 0 }, @@ -970,7 +977,7 @@ static struct win_syscall { #define osGetFileInformationByHandleEx ((BOOL(WINAPI*)(HANDLE, \ FILE_INFO_BY_HANDLE_CLASS,LPVOID,DWORD))aSyscall[66].pCurrent) -#if SQLITE_OS_WINRT && !defined(SQLITE_OMIT_WAL) +#if SQLITE_OS_WINRT && (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0) { "MapViewOfFileFromApp", (SYSCALL)MapViewOfFileFromApp, 0 }, #else { "MapViewOfFileFromApp", (SYSCALL)0, 0 }, @@ -1034,7 +1041,7 @@ static struct win_syscall { #define osGetProcessHeap ((HANDLE(WINAPI*)(VOID))aSyscall[74].pCurrent) -#if SQLITE_OS_WINRT && !defined(SQLITE_OMIT_WAL) +#if SQLITE_OS_WINRT && (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0) { "CreateFileMappingFromApp", (SYSCALL)CreateFileMappingFromApp, 0 }, #else { "CreateFileMappingFromApp", (SYSCALL)0, 0 }, From fdece7bad171dbb43985dfc7f53cf4a6e368c350 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Tue, 4 Nov 2014 19:52:15 +0000 Subject: [PATCH 074/133] Skip tests that require WAL mode when it is not enabled. FossilOrigin-Name: 6fc4ead26d19b9348bbda34c3053ae1e066abc32 --- manifest | 15 ++++----- manifest.uuid | 2 +- test/mmap1.test | 86 +++++++++++++++++++++++++------------------------ 3 files changed, 51 insertions(+), 52 deletions(-) diff --git a/manifest b/manifest index 74e32143e0..cf7c330a5e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C For\sthe\sWin32\sVFS,\sallow\smemory\smapped\sfiles\sto\swork\swhen\scompiled\swithout\sWAL\ssupport. -D 2014-11-04T19:37:22.576 +C Skip\stests\sthat\srequire\sWAL\smode\swhen\sit\sis\snot\senabled. +D 2014-11-04T19:52:15.610 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -728,7 +728,7 @@ F test/misc5.test 528468b26d03303b1f047146e5eefc941b9069f5 F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test edd0b63e2ee29a256900b0514f6fff27e19e9bb2 F test/misuse.test 3c34719944ba045cc6c188a4852ba04680728912 -F test/mmap1.test 93d167b328255cbe6679fe1e1a23be1b1197d07b +F test/mmap1.test 1bfd611b9841eafb44f7d83c0788e146d84a33c9 F test/mmap2.test 9d6dd9ddb4ad2379f29cc78f38ce1e63ed418022 F test/mmap3.test c92273e16eb8d23c1d55c9815b446bb72ef0512e F test/mmapfault.test d4c9eff9cd8c2dc14bc43e71e042f175b0a26fe3 @@ -1211,10 +1211,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P d423349d2cd8bc7e04f3d90ca7bab11e1ad86e25 -R c0de55f8c674bc2f58a7f862c94a6470 -T *branch * winMmapNoWal -T *sym-winMmapNoWal * -T -sym-trunk * +P 1fc7e2f3d34e25e7b59aa8b51d10c1e27ab4a527 +R ab0636a3afda48f38dd438546ea79dde U mistachkin -Z 8ecf95ce02a024463c10f127838cfcbc +Z 673ee028789258ee8d4aa78195cdb56d diff --git a/manifest.uuid b/manifest.uuid index e9b06734b6..b10e2d0d51 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1fc7e2f3d34e25e7b59aa8b51d10c1e27ab4a527 \ No newline at end of file +6fc4ead26d19b9348bbda34c3053ae1e066abc32 \ No newline at end of file diff --git a/test/mmap1.test b/test/mmap1.test index ece3e0201e..18aec9f8fa 100644 --- a/test/mmap1.test +++ b/test/mmap1.test @@ -33,7 +33,7 @@ proc register_rblob_code {dbname seed} { set ::rcnt $seed proc rblob {n} { set ::rcnt [expr (([set ::rcnt] << 3) + [set ::rcnt] + 456) & 0xFFFFFFFF] - set str [format %.8x [expr [set ::rcnt] ^ 0xbdf20da3]] + set str [format %.8x [expr [set ::rcnt] ^ 0xbdf20da3]] string range [string repeat [set str] [expr [set n]/4]] 1 [set n] } $dbname func rblob rblob @@ -42,7 +42,7 @@ proc register_rblob_code {dbname seed} { # For cases 1.1 and 1.4, the number of pages read using xRead() is 4 on # unix and 9 on windows. The difference is that windows only ever maps -# an integer number of OS pages (i.e. creates mappings that are a multiple +# an integer number of OS pages (i.e. creates mappings that are a multiple # of 4KB in size). Whereas on unix any sized mapping may be created. # foreach {t mmap_size nRead c2init} { @@ -106,50 +106,52 @@ foreach {t mmap_size nRead c2init} { set ::rcnt 0 proc rblob {n} { set ::rcnt [expr (($::rcnt << 3) + $::rcnt + 456) & 0xFFFFFFFF] - set str [format %.8x [expr $::rcnt ^ 0xbdf20da3]] + set str [format %.8x [expr $::rcnt ^ 0xbdf20da3]] string range [string repeat $str [expr $n/4]] 1 $n } reset_db db func rblob rblob -do_execsql_test 2.1 { - PRAGMA auto_vacuum = 1; - PRAGMA mmap_size = 67108864; - PRAGMA journal_mode = wal; - CREATE TABLE t1(a, b, UNIQUE(a, b)); - INSERT INTO t1 VALUES(rblob(500), rblob(500)); - INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 2 - INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 4 - INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 8 - INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 16 - INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 32 - PRAGMA wal_checkpoint; -} {67108864 wal 0 103 103} - -do_execsql_test 2.2 { - PRAGMA auto_vacuum; - SELECT count(*) FROM t1; -} {1 32} - -if {[permutation] != "inmemory_journal"} { - do_test 2.3 { - sqlite3 db2 test.db - db2 func rblob rblob - db2 eval { - DELETE FROM t1 WHERE (rowid%4); - PRAGMA wal_checkpoint; - } - db2 eval { - INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 16 - SELECT count(*) FROM t1; - } - } {16} - - do_execsql_test 2.4 { +ifcapable wal { + do_execsql_test 2.1 { + PRAGMA auto_vacuum = 1; + PRAGMA mmap_size = 67108864; + PRAGMA journal_mode = wal; + CREATE TABLE t1(a, b, UNIQUE(a, b)); + INSERT INTO t1 VALUES(rblob(500), rblob(500)); + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 2 + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 4 + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 8 + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 16 + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 32 PRAGMA wal_checkpoint; - } {0 24 24} - db2 close + } {67108864 wal 0 103 103} + + do_execsql_test 2.2 { + PRAGMA auto_vacuum; + SELECT count(*) FROM t1; + } {1 32} + + if {[permutation] != "inmemory_journal"} { + do_test 2.3 { + sqlite3 db2 test.db + db2 func rblob rblob + db2 eval { + DELETE FROM t1 WHERE (rowid%4); + PRAGMA wal_checkpoint; + } + db2 eval { + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 16 + SELECT count(*) FROM t1; + } + } {16} + + do_execsql_test 2.4 { + PRAGMA wal_checkpoint; + } {0 24 24} + db2 close + } } reset_db @@ -227,7 +229,7 @@ do_test 4.4 { do_execsql_test 4.5 { COMMIT } #------------------------------------------------------------------------- -# Ensure that existing cursors holding xFetch() references are not +# Ensure that existing cursors holding xFetch() references are not # confused if those pages are moved to make way for the root page of a # new table or index. # @@ -296,7 +298,7 @@ foreach {tn1 mmap1 mmap2} { sql1 "PRAGMA mmap_size = $mmap1" sql2 "PRAGMA mmap_size = $mmap2" - do_test $tn1.$tn { + do_test $tn1.$tn { for {set i 1} {$i <= 100} {incr i} { if {$i % 2} { set c1 sql1 @@ -311,7 +313,7 @@ foreach {tn1 mmap1 mmap2} { UPDATE t2 SET x = (SELECT md5sum(a) FROM t1); } - set res [$c2 { + set res [$c2 { SELECT count(*) FROM t1; SELECT x == (SELECT md5sum(a) FROM t1) FROM t2; PRAGMA integrity_check; From 547fb61807b3ac19ee97d5acc99e0810e4baba5d Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 4 Nov 2014 21:38:45 +0000 Subject: [PATCH 075/133] Improved output formatting for the showstat4 tool. FossilOrigin-Name: 7df82c46da437bc743576358c25e758280067df8 --- manifest | 12 ++++++------ manifest.uuid | 2 +- tool/showstat4.c | 9 +++++---- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index e9f3481cba..9ad9755cc5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\svarious\srequirements\sevidence\smarks\sfor\ssqlite3_config()\soptions. -D 2014-11-04T17:23:24.610 +C Improved\soutput\sformatting\sfor\sthe\sshowstat4\stool. +D 2014-11-04T21:38:45.383 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1190,7 +1190,7 @@ F tool/restore_jrnl.tcl 6957a34f8f1f0f8285e07536225ec3b292a9024a F tool/rollback-test.c 9fc98427d1e23e84429d7e6d07d9094fbdec65a5 F tool/showdb.c bd073a78bce714a0e42d92ea474b3eb8cb53be5d F tool/showjournal.c 053eb1cc774710c6890b7dd6293300cc297b16a5 -F tool/showstat4.c c39279d6bd37cb999b634f0064f6f86ad7af008f +F tool/showstat4.c 9515faa8ec176599d4a8288293ba8ec61f7b728a F tool/showwal.c 85cb36d4fe3e93e2fbd63e786e0d1ce42d0c4fad F tool/soak1.tcl 8d407956e1a45b485a8e072470a3e629a27037fe F tool/space_used.tcl f714c41a59e326b8b9042f415b628b561bafa06b @@ -1211,7 +1211,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 391c9b85abcb5ba300fb2e116384639310c69ed2 -R 59d979fd0dabfa4a2d3ae24d9471db8f +P d423349d2cd8bc7e04f3d90ca7bab11e1ad86e25 +R 3d8e834a0bd7bfcb019dc53478b58f7b U drh -Z d72c7362e49ef35f8e5912118c20e6ff +Z 0a79c8f46f24bb92a10190eb30b27d6d diff --git a/manifest.uuid b/manifest.uuid index 969ed55bc1..11e49b8e8c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d423349d2cd8bc7e04f3d90ca7bab11e1ad86e25 \ No newline at end of file +7df82c46da437bc743576358c25e758280067df8 \ No newline at end of file diff --git a/tool/showstat4.c b/tool/showstat4.c index 668d2106af..215962919e 100644 --- a/tool/showstat4.c +++ b/tool/showstat4.c @@ -39,6 +39,7 @@ int main(int argc, char **argv){ int nSample; i64 iVal; const char *zSep; + int iRow = 0; if( argc!=2 ){ fprintf(stderr, "Usage: %s DATABASE-FILE\n", argv[0]); @@ -60,13 +61,13 @@ int main(int argc, char **argv){ } while( SQLITE_ROW==sqlite3_step(pStmt) ){ if( zIdx==0 || strcmp(zIdx, (const char*)sqlite3_column_text(pStmt,0))!=0 ){ - if( zIdx ) printf("\n"); + if( zIdx ) printf("\n**************************************" + "**************\n\n"); sqlite3_free(zIdx); zIdx = sqlite3_mprintf("%s", sqlite3_column_text(pStmt,0)); - printf("%s:\n", zIdx); - }else{ - printf(" -----------------------------------------------------------\n"); + iRow = 0; } + printf("%s sample %d ------------------------------------\n", zIdx, ++iRow); printf(" nEq = %s\n", sqlite3_column_text(pStmt,1)); printf(" nLt = %s\n", sqlite3_column_text(pStmt,2)); printf(" nDLt = %s\n", sqlite3_column_text(pStmt,3)); From 8d1edb92c4c5d29bb0746b79778692b66e2c1296 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 5 Nov 2014 09:07:28 +0000 Subject: [PATCH 076/133] Add the ".scanstats on" command to the shell tool. Executing this command causes the shell tool to print values from sqlite3_stmt_scanstatus() after each query is run. FossilOrigin-Name: 7974c0ed10ffdc960a43fed89845c2bed428958d --- manifest | 14 ++++++------- manifest.uuid | 2 +- src/shell.c | 54 +++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 62 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 9ad9755cc5..29e1d31cf2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\soutput\sformatting\sfor\sthe\sshowstat4\stool. -D 2014-11-04T21:38:45.383 +C Add\sthe\s".scanstats\son"\scommand\sto\sthe\sshell\stool.\sExecuting\sthis\scommand\scauses\sthe\sshell\stool\sto\sprint\svalues\sfrom\ssqlite3_stmt_scanstatus()\safter\seach\squery\sis\srun. +D 2014-11-05T09:07:28.365 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -228,7 +228,7 @@ F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b -F src/shell.c 282f8f5278e0c78eb442217531172ec9e1538796 +F src/shell.c 5ad1eb4dfcd7a57e15825207a9bd559415bf34b1 F src/sqlite.h.in 6e9af739d79f0bea2584b70fb1c54d3bb1a2eab6 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d @@ -1211,7 +1211,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P d423349d2cd8bc7e04f3d90ca7bab11e1ad86e25 -R 3d8e834a0bd7bfcb019dc53478b58f7b -U drh -Z 0a79c8f46f24bb92a10190eb30b27d6d +P 7df82c46da437bc743576358c25e758280067df8 +R f4931dcb1177b970d566df3dbaa382ec +U dan +Z 44baa22d0450fc9fcacf5946d93d71c5 diff --git a/manifest.uuid b/manifest.uuid index 11e49b8e8c..ccd696efe4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7df82c46da437bc743576358c25e758280067df8 \ No newline at end of file +7974c0ed10ffdc960a43fed89845c2bed428958d \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index 59cd2011e7..3423bb0143 100644 --- a/src/shell.c +++ b/src/shell.c @@ -457,6 +457,7 @@ struct ShellState { int echoOn; /* True to echo input commands */ int autoEQP; /* Run EXPLAIN QUERY PLAN prior to seach SQL stmt */ int statsOn; /* True to display memory stats before each finalize */ + int scanstatsOn; /* True to display scan stats before each finalize */ int outCount; /* Revert to stdout when reaching zero */ int cnt; /* Number of records displayed so far */ FILE *out; /* Write results here */ @@ -1185,6 +1186,42 @@ static int display_stats( return 0; } +/* +** Display scan stats. +*/ +static void display_scanstats( + sqlite3 *db, /* Database to query */ + ShellState *pArg /* Pointer to ShellState */ +){ +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + int i; + fprintf(pArg->out, "-------- scanstats --------\n"); + for(i=0; 1; i++){ + sqlite3_stmt *p = pArg->pStmt; + sqlite3_int64 nEst, nLoop, nVisit; + const char *zExplain; + if( sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_NLOOP, (void*)&nLoop) ){ + break; + } + sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_NVISIT, (void*)&nVisit); + sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_EST, (void*)&nEst); + sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_EXPLAIN, (void*)&zExplain); + + fprintf(pArg->out, "Loop %d: \"%s\"\n", i, zExplain); + fprintf(pArg->out, " nLoop=%-8lld nVisit=%-8lld nEst=%-8lld\n", + nLoop, nVisit, nEst + ); + } +#else + fprintf(pArg->out, "-------- scanstats --------\n"); + fprintf(pArg->out, + "sqlite3_stmt_scanstatus() unavailable - " + "rebuild with SQLITE_ENABLE_STMT_SCANSTATUS\n" + ); +#endif + fprintf(pArg->out, "---------------------------\n"); +} + /* ** Parameter azArray points to a zero-terminated array of strings. zStr ** points to a single nul-terminated string. Return non-zero if zStr @@ -1423,6 +1460,11 @@ static int shell_exec( display_stats(db, pArg, 0); } + /* print loop-counters if required */ + if( pArg && pArg->scanstatsOn ){ + display_scanstats(db, pArg); + } + /* Finalize the statement just executed. If this fails, save a ** copy of the error message. Otherwise, set zSql to point to the ** next statement to execute. */ @@ -3014,6 +3056,16 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_close(pSrc); }else + + if( c=='s' && strncmp(azArg[0], "scanstats", n)==0 ){ + if( nArg==2 ){ + p->scanstatsOn = booleanValue(azArg[1]); + }else{ + fprintf(stderr, "Usage: .scanstats on|off\n"); + rc = 1; + } + }else + if( c=='s' && strncmp(azArg[0], "schema", n)==0 ){ ShellState data; char *zErrMsg = 0; @@ -4140,6 +4192,8 @@ int main(int argc, char **argv){ data.autoEQP = 1; }else if( strcmp(z,"-stats")==0 ){ data.statsOn = 1; + }else if( strcmp(z,"-scanstats")==0 ){ + data.scanstatsOn = 1; }else if( strcmp(z,"-bail")==0 ){ bail_on_error = 1; }else if( strcmp(z,"-version")==0 ){ From e0de876e279cc6cfd9f809a145f1172cbae9837e Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 5 Nov 2014 13:13:13 +0000 Subject: [PATCH 077/133] Enhance whereLoopCheaperProperSubset(X,Y) so that it does not report true if X uses skip-scan less than Y, since in that case X might deserve to be cheaper even if it is a proper subset. FossilOrigin-Name: c106b755369c1f8546e897ecd2ac56fd09d6e885 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 12 +++++++----- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 29e1d31cf2..8e0066d70e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s".scanstats\son"\scommand\sto\sthe\sshell\stool.\sExecuting\sthis\scommand\scauses\sthe\sshell\stool\sto\sprint\svalues\sfrom\ssqlite3_stmt_scanstatus()\safter\seach\squery\sis\srun. -D 2014-11-05T09:07:28.365 +C Enhance\swhereLoopCheaperProperSubset(X,Y)\sso\sthat\sit\sdoes\snot\sreport\strue\nif\sX\suses\sskip-scan\sless\sthan\sY,\ssince\sin\sthat\scase\sX\smight\ndeserve\sto\sbe\scheaper\seven\sif\sit\sis\sa\sproper\ssubset. +D 2014-11-05T13:13:13.983 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -302,7 +302,7 @@ F src/vtab.c 2a30791bbd7926b589401bd09c3abb33de563793 F src/wal.c 825c948066c7604a07d56e67958cdab210749016 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c d5fa1081bf7cb70478ed06489e063695c92ee1e1 +F src/where.c 240961041f35862ebcafd260587c79a1ab7347f8 F src/whereInt.h d3633e9b592103241b74b0ec76185f3e5b8b62e0 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1211,7 +1211,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 7df82c46da437bc743576358c25e758280067df8 -R f4931dcb1177b970d566df3dbaa382ec -U dan -Z 44baa22d0450fc9fcacf5946d93d71c5 +P 7974c0ed10ffdc960a43fed89845c2bed428958d +R 418c93fe7ac7140cffa87d5f3271cb48 +U drh +Z 68c42aaa0a73598a09ed4215a5cfd466 diff --git a/manifest.uuid b/manifest.uuid index ccd696efe4..da46c6359a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7974c0ed10ffdc960a43fed89845c2bed428958d \ No newline at end of file +c106b755369c1f8546e897ecd2ac56fd09d6e885 \ No newline at end of file diff --git a/src/where.c b/src/where.c index d7966fdfbc..34614e6bb3 100644 --- a/src/where.c +++ b/src/where.c @@ -4012,10 +4012,11 @@ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ } /* -** Return TRUE if both of the following are true: +** Return TRUE if all of the following are true: ** ** (1) X has the same or lower cost that Y ** (2) X is a proper subset of Y +** (3) X skips at least as many columns as Y ** ** By "proper subset" we mean that X uses fewer WHERE clause terms ** than Y and that every WHERE clause term used by X is also used @@ -4023,7 +4024,9 @@ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ ** ** If X is a proper subset of Y then Y is a better choice and ought ** to have a lower cost. This routine returns TRUE when that cost -** relationship is inverted and needs to be adjusted. +** relationship is inverted and needs to be adjusted. The third rule +** was added because if X uses skip-scan less than Y it still might +** deserve a lower cost even if it is a proper subset of Y. */ static int whereLoopCheaperProperSubset( const WhereLoop *pX, /* First WhereLoop to compare */ @@ -4033,6 +4036,7 @@ static int whereLoopCheaperProperSubset( if( pX->nLTerm-pX->nSkip >= pY->nLTerm-pY->nSkip ){ return 0; /* X is not a subset of Y */ } + if( pY->nSkip > pX->nSkip ) return 0; if( pX->rRun >= pY->rRun ){ if( pX->rRun > pY->rRun ) return 0; /* X costs more than Y */ if( pX->nOut > pY->nOut ) return 0; /* X costs more than Y */ @@ -4068,9 +4072,7 @@ static void whereLoopAdjustCost(const WhereLoop *p, WhereLoop *pTemplate){ if( (p->wsFlags & WHERE_INDEXED)==0 ) continue; if( whereLoopCheaperProperSubset(p, pTemplate) ){ /* Adjust pTemplate cost downward so that it is cheaper than its - ** subset p. Except, do not adjust the cost estimate downward for - ** a loop that skips more columns. */ - if( pTemplate->nSkip>p->nSkip ) continue; + ** subset p. */ WHERETRACE(0x80,("subset cost adjustment %d,%d to %d,%d\n", pTemplate->rRun, pTemplate->nOut, p->rRun, p->nOut-1)); pTemplate->rRun = p->rRun; From 937994aa6596a0a77504f06d86d11f30bdc82fb3 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 5 Nov 2014 14:19:05 +0000 Subject: [PATCH 078/133] Add a test case to check that the previous commit is effective. FossilOrigin-Name: 948d6e5d07bc14b6de32ec2144c716a5532f894c --- manifest | 14 ++++++------ manifest.uuid | 2 +- test/skipscan6.test | 55 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 8e0066d70e..c849d3cf1c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\swhereLoopCheaperProperSubset(X,Y)\sso\sthat\sit\sdoes\snot\sreport\strue\nif\sX\suses\sskip-scan\sless\sthan\sY,\ssince\sin\sthat\scase\sX\smight\ndeserve\sto\sbe\scheaper\seven\sif\sit\sis\sa\sproper\ssubset. -D 2014-11-05T13:13:13.983 +C Add\sa\stest\scase\sto\scheck\sthat\sthe\sprevious\scommit\sis\seffective. +D 2014-11-05T14:19:05.905 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -849,7 +849,7 @@ F test/skipscan1.test 7e15e1cc524524e7b2c4595ec85c75501d22f4ff F test/skipscan2.test d1d1450952b7275f0b0a3a981f0230532743951a F test/skipscan3.test ec5bab3f81c7038b43450e7b3062e04a198bdbb5 F test/skipscan5.test 67817a4b6857c47e0e33ba3e506da6f23ef68de2 -F test/skipscan6.test 3a891b45d6df266ced861a2ad9d03fca2bc7fcc5 +F test/skipscan6.test 5866039d03a56f5bd0b3d172a012074a1d90a15b F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24 F test/sort.test c4400e7533748f6bd7413851ff148645e82b9e2d @@ -1211,7 +1211,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 7974c0ed10ffdc960a43fed89845c2bed428958d -R 418c93fe7ac7140cffa87d5f3271cb48 -U drh -Z 68c42aaa0a73598a09ed4215a5cfd466 +P c106b755369c1f8546e897ecd2ac56fd09d6e885 +R 27944d6d7573adcf0b31a08e199961ff +U dan +Z 1a8729ffc04f45ef7b13c20540a14cbb diff --git a/manifest.uuid b/manifest.uuid index da46c6359a..911dbed051 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c106b755369c1f8546e897ecd2ac56fd09d6e885 \ No newline at end of file +948d6e5d07bc14b6de32ec2144c716a5532f894c \ No newline at end of file diff --git a/test/skipscan6.test b/test/skipscan6.test index 9eda9a66f3..026c4d7b00 100644 --- a/test/skipscan6.test +++ b/test/skipscan6.test @@ -141,5 +141,60 @@ do_execsql_test 2.2 { } {/INDEX good .bb=. AND aa=. AND dd>. AND dd<../} +# Create a table containing 100 rows. Column "a" contains a copy of the +# rowid value - sequentially increasing integers from 1 to 100. Column +# "b" contains the value of (a % 5). Columns "c" and "d" both contain +# constant values (i.e. the same for every row). +# +# Then create a second table t2. t2 is the same as t3 except for the +# order in which the indexes are created. +# +do_execsql_test 3.0 { + CREATE TABLE t3(a, b, c, d); + CREATE INDEX t3_ba ON t3(b, a, c); + CREATE INDEX t3_a ON t3(a); + + WITH d(a, b) AS ( + SELECT 1, 1 + UNION ALL + SELECT a+1, (a+1) % 5 FROM d WHERE a<100 + ) + INSERT INTO t3 SELECT a, b, 'c', 'd' FROM d; + + CREATE TABLE t2(a, b, c, d); + CREATE INDEX t2_a ON t2(a); + CREATE INDEX t2_ba ON t2(b, a, c); + INSERT INTO t2 SELECT * FROM t3; + + ANALYZE; + SELECT * FROM sqlite_stat1; +} { + t2 t2_ba {100 20 1 1} + t2 t2_a {100 1} + t3 t3_a {100 1} + t3 t3_ba {100 20 1 1} +} + +# Use index "t3_a", as (a=?) is expected to match only a single row. +# +do_eqp_test 3.1 { + SELECT * FROM t3 WHERE a = ? AND c = ? +} { + 0 0 0 {SEARCH TABLE t3 USING INDEX t3_a (a=?)} +} + +# The same query on table t2. This should use index "t2_a", for the +# same reason. At one point though, it was mistakenly using a skip-scan. +# +do_eqp_test 3.2 { + SELECT * FROM t2 WHERE a = ? AND c = ? +} { + 0 0 0 {SEARCH TABLE t2 USING INDEX t2_a (a=?)} +} + +finish_test + + + finish_test From 6b5631e02f2204da55742ec6e89b3633d36f51cb Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 5 Nov 2014 15:57:39 +0000 Subject: [PATCH 079/133] Make sure that NULL results from OP_Column are fully and completely NULL and do not have the MEM_Ephem bit set. Fix for ticket [094d39a4c95ee4]. FossilOrigin-Name: 42705fd7d892c4fdfb95fbbb468c99569beece25 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/vdbe.c | 2 +- test/table.test | 18 +++++++++++++++++- 4 files changed, 27 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index c849d3cf1c..29f44c63b9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\stest\scase\sto\scheck\sthat\sthe\sprevious\scommit\sis\seffective. -D 2014-11-05T14:19:05.905 +C Make\ssure\sthat\sNULL\sresults\sfrom\sOP_Column\sare\sfully\sand\scompletely\sNULL\nand\sdo\snot\shave\sthe\sMEM_Ephem\sbit\sset.\s\sFix\sfor\sticket\s[094d39a4c95ee4]. +D 2014-11-05T15:57:39.984 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/update.c 3c4ecc282accf12d39edb8d524cf089645e55a13 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 3b627daa45c7308c1e36e3dbaa3f9ce7e5c7fa73 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c 175a360c56e75ce4eb2b60704fd7c011b93926f5 +F src/vdbe.c 3fd4ebd3e87b63175bfd2be747608bae1670b4df F src/vdbe.h d412bd01e89f0d69991b8f46601f96bc169d28f4 F src/vdbeInt.h 539ba284790e871f98be74a78cbdfcedfae22639 F src/vdbeapi.c 900259bdd85cd66a9f210d8ec08147a9034593bd @@ -879,7 +879,7 @@ F test/superlock.test 1cde669f68d2dd37d6c9bd35eee1d95491ae3fc2 F test/sync.test a34cd43e98b7fb84eabbf38f7ed8f7349b3f3d85 F test/syscall.test d2fdaad713f103ac611fe7ef9b724c7b69f8149c F test/sysfault.test fa776e60bf46bdd3ae69f0b73e46ee3977a58ae6 -F test/table.test 2a1d2fa52c531de5915f28023747d9a8c27b6f31 +F test/table.test 06271d61eb13871490d38168433c1ef3dd82bb2a F test/tableapi.test 2674633fa95d80da917571ebdd759a14d9819126 F test/tableopts.test dba698ba97251017b7c80d738c198d39ab747930 F test/tclsqlite.test 37a61c2da7e3bfe3b8c1a2867199f6b860df5d43 @@ -1211,7 +1211,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P c106b755369c1f8546e897ecd2ac56fd09d6e885 -R 27944d6d7573adcf0b31a08e199961ff -U dan -Z 1a8729ffc04f45ef7b13c20540a14cbb +P 948d6e5d07bc14b6de32ec2144c716a5532f894c +R 4628888308aa60120db393287d665a20 +U drh +Z e07070b00a71f6116b6f23e15e08fb5e diff --git a/manifest.uuid b/manifest.uuid index 911dbed051..8df21c81cd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -948d6e5d07bc14b6de32ec2144c716a5532f894c \ No newline at end of file +42705fd7d892c4fdfb95fbbb468c99569beece25 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 62470a743b..bf9b233a26 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2301,7 +2301,7 @@ case OP_Column: { pC->payloadSize = pC->szRow = avail = pReg->n; pC->aRow = (u8*)pReg->z; }else{ - MemSetTypeFlag(pDest, MEM_Null); + sqlite3VdbeMemSetNull(pDest); goto op_column_out; } }else{ diff --git a/test/table.test b/test/table.test index 656884ca73..69f105aa6c 100644 --- a/test/table.test +++ b/test/table.test @@ -11,7 +11,6 @@ # This file implements regression tests for SQLite library. The # focus of this file is testing the CREATE TABLE statement. # -# $Id: table.test,v 1.53 2009/06/05 17:09:12 drh Exp $ set testdir [file dirname $argv0] source $testdir/tester.tcl @@ -773,4 +772,21 @@ do_catchsql_test table-16.7 { INSERT INTO t16 DEFAULT VALUES; } {1 {unknown function: group_concat()}} +# Ticket [https://www.sqlite.org/src/info/094d39a4c95ee4abbc417f04214617675ba15c63] +# describes a assertion fault that occurs on a CREATE TABLE .. AS SELECT statement. +# the following test verifies that the problem has been fixed. +# +do_execsql_test table-17.1 { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(a TEXT); + INSERT INTO t1(a) VALUES(1),(2); + DROP TABLE IF EXISTS t2; + CREATE TABLE t2(x TEXT, y TEXT); + INSERT INTO t2(x,y) VALUES(3,4); + DROP TABLE IF EXISTS t3; + CREATE TABLE t3 AS + SELECT a AS p, coalesce(y,a) AS q FROM t1 LEFT JOIN t2 ON a=x; + SELECT p, q, '|' FROM t3 ORDER BY p; +} {1 1 | 2 2 |} + finish_test From 6d3f91d040ae42394d7b1534cce8ead374cc0f7c Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 5 Nov 2014 19:26:12 +0000 Subject: [PATCH 080/133] Change the query planner to do a better job of estimating the number rows selected by a BETWEEN operator using STAT4 when both upper and lower bounds are contained within the same sample. FossilOrigin-Name: 2d36be5d9a1cdd4fd2d54fc4eeece32a81cbacc1 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 34 +++++++++++++++++++++------------- test/analyze8.test | 8 ++++---- 4 files changed, 33 insertions(+), 25 deletions(-) diff --git a/manifest b/manifest index 29f44c63b9..d1dcbe649d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\ssure\sthat\sNULL\sresults\sfrom\sOP_Column\sare\sfully\sand\scompletely\sNULL\nand\sdo\snot\shave\sthe\sMEM_Ephem\sbit\sset.\s\sFix\sfor\sticket\s[094d39a4c95ee4]. -D 2014-11-05T15:57:39.984 +C Change\sthe\squery\splanner\sto\sdo\sa\sbetter\sjob\sof\sestimating\sthe\snumber\srows\nselected\sby\sa\sBETWEEN\soperator\susing\sSTAT4\swhen\sboth\supper\sand\slower\sbounds\nare\scontained\swithin\sthe\ssame\ssample. +D 2014-11-05T19:26:12.741 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -302,7 +302,7 @@ F src/vtab.c 2a30791bbd7926b589401bd09c3abb33de563793 F src/wal.c 825c948066c7604a07d56e67958cdab210749016 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 240961041f35862ebcafd260587c79a1ab7347f8 +F src/where.c 2c2081c546c90227577c502765611555503ce3f7 F src/whereInt.h d3633e9b592103241b74b0ec76185f3e5b8b62e0 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -321,7 +321,7 @@ F test/analyze4.test eff2df19b8dd84529966420f29ea52edc6b56213 F test/analyze5.test 765c4e284aa69ca172772aa940946f55629bc8c4 F test/analyze6.test f1c552ce39cca4ec922a7e4e0e5d0203d6b3281f F test/analyze7.test bb1409afc9e8629e414387ef048b8e0e3e0bdc4f -F test/analyze8.test 093d15c1c888eed5034304a98c992f7360130b88 +F test/analyze8.test c05a461d0a6b05991106467d0c47480f2e709c82 F test/analyze9.test 72795c8113604b5dcd47a1498a61d6d7fb5d041a F test/analyzeA.test 3335697f6700c7052295cfd0067fc5b2aacddf9a F test/analyzeB.test 8bf35ee0a548aea831bf56762cb8e7fdb1db083d @@ -1211,7 +1211,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 948d6e5d07bc14b6de32ec2144c716a5532f894c -R 4628888308aa60120db393287d665a20 +P 42705fd7d892c4fdfb95fbbb468c99569beece25 +R b31ea7cd782cb3cc21bda8e8c039f16c U drh -Z e07070b00a71f6116b6f23e15e08fb5e +Z 7c6a788d1b7b1ae57295ef90b3d9da88 diff --git a/manifest.uuid b/manifest.uuid index 8df21c81cd..64b0a08a4b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -42705fd7d892c4fdfb95fbbb468c99569beece25 \ No newline at end of file +2d36be5d9a1cdd4fd2d54fc4eeece32a81cbacc1 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 34614e6bb3..92f783b2e9 100644 --- a/src/where.c +++ b/src/where.c @@ -1897,7 +1897,6 @@ static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){ } #endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) */ - #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 /* ** Estimate the location of a particular key among all keys in an @@ -1906,9 +1905,10 @@ static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){ ** aStat[0] Est. number of rows less than pVal ** aStat[1] Est. number of rows equal to pVal ** -** Return SQLITE_OK on success. +** Return the index of the sample that is the smallest sample that +** is greater than or equal to pRec. */ -static void whereKeyStats( +static int whereKeyStats( Parse *pParse, /* Database connection */ Index *pIdx, /* Index to consider domain of */ UnpackedRecord *pRec, /* Vector of values to consider */ @@ -1990,6 +1990,7 @@ static void whereKeyStats( } aStat[0] = iLower + iGap; } + return i; } #endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ @@ -2140,7 +2141,7 @@ static int whereRangeSkipScanEst( ** If either of the upper or lower bound is not present, then NULL is passed in ** place of the corresponding WhereTerm. ** -** The value in (pBuilder->pNew->u.btree.nEq) is the index of the index +** The value in (pBuilder->pNew->u.btree.nEq) is the number of the index ** column subject to the range constraint. Or, equivalently, the number of ** equality constraints optimized by the proposed index scan. For example, ** assuming index p is on t1(a, b), and the SQL query is: @@ -2156,7 +2157,7 @@ static int whereRangeSkipScanEst( ** ** When this function is called, *pnOut is set to the sqlite3LogEst() of the ** number of rows that the index scan is expected to visit without -** considering the range constraints. If nEq is 0, this is the number of +** considering the range constraints. If nEq is 0, then *pnOut is the number of ** rows in the index. Assuming no error occurs, *pnOut is adjusted (reduced) ** to account for the range constraints pLower and pUpper. ** @@ -2180,9 +2181,7 @@ static int whereRangeScanEst( Index *p = pLoop->u.btree.pIndex; int nEq = pLoop->u.btree.nEq; - if( p->nSample>0 - && nEqnSampleCol - ){ + if( p->nSample>0 && nEqnSampleCol ){ if( nEq==pBuilder->nRecValid ){ UnpackedRecord *pRec = pBuilder->pRec; tRowcnt a[2]; @@ -2198,15 +2197,19 @@ static int whereRangeScanEst( ** is not a simple variable or literal value), the lower bound of the ** range is $P. Due to a quirk in the way whereKeyStats() works, even ** if $L is available, whereKeyStats() is called for both ($P) and - ** ($P:$L) and the larger of the two returned values used. + ** ($P:$L) and the larger of the two returned values is used. ** ** Similarly, iUpper is to be set to the estimate of the number of rows ** less than the upper bound of the range query. Where the upper bound ** is either ($P) or ($P:$U). Again, even if $U is available, both values ** of iUpper are requested of whereKeyStats() and the smaller used. + ** + ** The number of rows between the two bounds is then just iUpper-iLower. */ - tRowcnt iLower; - tRowcnt iUpper; + tRowcnt iLower; /* Rows less than the lower bound */ + tRowcnt iUpper; /* Rows less than the upper bound */ + int iLwrIdx = -2; /* aSample[] for the lower bound */ + int iUprIdx = -1; /* aSample[] for the upper bound */ if( pRec ){ testcase( pRec->nField!=pBuilder->nRecValid ); @@ -2244,7 +2247,7 @@ static int whereRangeScanEst( rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq, &bOk); if( rc==SQLITE_OK && bOk ){ tRowcnt iNew; - whereKeyStats(pParse, p, pRec, 0, a); + iLwrIdx = whereKeyStats(pParse, p, pRec, 0, a); iNew = a[0] + ((pLower->eOperator & (WO_GT|WO_LE)) ? a[1] : 0); if( iNew>iLower ) iLower = iNew; nOut--; @@ -2259,7 +2262,7 @@ static int whereRangeScanEst( rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq, &bOk); if( rc==SQLITE_OK && bOk ){ tRowcnt iNew; - whereKeyStats(pParse, p, pRec, 1, a); + iUprIdx = whereKeyStats(pParse, p, pRec, 1, a); iNew = a[0] + ((pUpper->eOperator & (WO_GT|WO_LE)) ? a[1] : 0); if( iNewiLower ){ nNew = sqlite3LogEst(iUpper - iLower); + /* TUNING: If both iUpper and iLower are derived from the same + ** sample, then assume they are 4x more selective. This brings + ** the estimated selectivity more in line with what it would be + ** if estimated without the use of STAT3/4 tables. */ + if( iLwrIdx==iUprIdx ) nNew -= 20; assert( 20==sqlite3LogEst(4) ); }else{ nNew = 10; assert( 10==sqlite3LogEst(2) ); } diff --git a/test/analyze8.test b/test/analyze8.test index 4384c39676..1079e68080 100644 --- a/test/analyze8.test +++ b/test/analyze8.test @@ -86,23 +86,23 @@ do_test 2.1 { # range. # # Test 3.2 is a little unstable. It depends on the planner estimating -# that (b BETWEEN 50 AND 54) will match more rows than (c BETWEEN +# that (b BETWEEN 30 AND 34) will match more rows than (c BETWEEN # 800000 AND 900000). Which is a pretty close call (50 vs. 32), so # the planner could get it wrong with an unlucky set of samples. This # case happens to work, but others ("b BETWEEN 40 AND 44" for example) # will fail. # do_execsql_test 3.0 { - SELECT count(*) FROM t1 WHERE b BETWEEN 50 AND 54; + SELECT count(*) FROM t1 WHERE b BETWEEN 30 AND 34; SELECT count(*) FROM t1 WHERE c BETWEEN 0 AND 100000; SELECT count(*) FROM t1 WHERE c BETWEEN 800000 AND 900000; } {50 376 32} do_test 3.1 { - eqp {SELECT * FROM t1 WHERE b BETWEEN 50 AND 54 AND c BETWEEN 0 AND 100000} + eqp {SELECT * FROM t1 WHERE b BETWEEN 30 AND 34 AND c BETWEEN 0 AND 100000} } {0 0 0 {SEARCH TABLE t1 USING INDEX t1b (b>? AND b? AND c Date: Wed, 5 Nov 2014 21:21:08 +0000 Subject: [PATCH 081/133] Fix harmless compiler warnings in the new balance_nonroot() routine. FossilOrigin-Name: 83a1e5db926b3a6d40f4a5cf9a8e6852b9bac9ac --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 19 ++++++++++++------- 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index d1dcbe649d..6994d5e8ec 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\squery\splanner\sto\sdo\sa\sbetter\sjob\sof\sestimating\sthe\snumber\srows\nselected\sby\sa\sBETWEEN\soperator\susing\sSTAT4\swhen\sboth\supper\sand\slower\sbounds\nare\scontained\swithin\sthe\ssame\ssample. -D 2014-11-05T19:26:12.741 +C Fix\sharmless\scompiler\swarnings\sin\sthe\snew\sbalance_nonroot()\sroutine. +D 2014-11-05T21:21:08.046 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 7f841396adfd47507ff670a471162d2bfcda3136 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 11d1262110c2d459b68121833fa3ec6625b1d022 +F src/btree.c 4a126e2066076872ab6f37f9ad116eb5f651cd38 F src/btree.h 49b408be9c1cd41249076898e0673711071205d8 F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 67bb05b1077e0cdaccb2e36bfcbe7a5df9ed31e8 @@ -1211,7 +1211,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 42705fd7d892c4fdfb95fbbb468c99569beece25 -R b31ea7cd782cb3cc21bda8e8c039f16c +P 2d36be5d9a1cdd4fd2d54fc4eeece32a81cbacc1 +R e9a3ba77188e96b62213debbd39f0d21 U drh -Z 7c6a788d1b7b1ae57295ef90b3d9da88 +Z dda139af7d076df31a05d3458397d7d6 diff --git a/manifest.uuid b/manifest.uuid index 64b0a08a4b..82de0553f1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2d36be5d9a1cdd4fd2d54fc4eeece32a81cbacc1 \ No newline at end of file +83a1e5db926b3a6d40f4a5cf9a8e6852b9bac9ac \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index fe15c922ca..8b4a2a4f3b 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1297,8 +1297,8 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ const int hdr = pPage->hdrOffset; /* Local cache of pPage->hdrOffset */ u8 * const data = pPage->aData; /* Local cache of pPage->aData */ int top; /* First byte of cell content area */ + int rc = SQLITE_OK; /* Integer return code */ int gap; /* First byte of gap between cell pointers and cell content */ - int rc; /* Integer return code */ assert( sqlite3PagerIswriteable(pPage->pDbPage) ); assert( pPage->pBt ); @@ -1328,13 +1328,13 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ testcase( gap+1==top ); testcase( gap==top ); if( gap+2<=top && (data[hdr+1] || data[hdr+2]) ){ - int rc = SQLITE_OK; int bDefrag = 0; u8 *pSpace = pageFindSlot(pPage, nByte, &rc, &bDefrag); if( rc ) return rc; if( bDefrag ) goto defragment_page; if( pSpace ){ - *pIdx = pSpace - data; + assert( pSpace>=data && (pSpace - data)<65536 ); + *pIdx = (int)(pSpace - data); return SQLITE_OK; } } @@ -6117,7 +6117,10 @@ static int pageFreeArray( if( pCell>=pStart && pCellaData && (pFree - aData)<65536 ); + freeSpace(pPg, (u16)(pFree - aData), szFree); + } pFree = pCell; szFree = sz; if( pFree+sz>pEnd ) return 0; @@ -6128,7 +6131,10 @@ static int pageFreeArray( nRet++; } } - if( pFree ) freeSpace(pPg, pFree - aData, szFree); + if( pFree ){ + assert( pFree>aData && (pFree - aData)<65536 ); + freeSpace(pPg, (u16)(pFree - aData), szFree); + } return nRet; } @@ -6192,7 +6198,7 @@ static void editPage( for(i=0; inOverflow; i++){ int iCell = (iOld + pPg->aiOvfl[i]) - iNew; if( iCell>=0 && iCellaCellIdx[iCell * 2]; + pCellptr = &pPg->aCellIdx[iCell * 2]; memmove(&pCellptr[2], pCellptr, (nCell - iCell) * 2); nCell++; if( pageInsertArray( @@ -6881,7 +6887,6 @@ static int balance_nonroot( } for(i=0; i Date: Thu, 6 Nov 2014 03:55:10 +0000 Subject: [PATCH 082/133] Change the SQLITE_SCANSTAT_EST parameter so that it returns a double for the estimated number of output rows per loop, rather than a 64-bit integer. Revise the output format for the ".scanstats on" in the shell to make use of this new capability. FossilOrigin-Name: f9684000665ae7ef6f89c3773612b8286b8f545a --- manifest | 32 ++++++++++--------- manifest.uuid | 2 +- src/shell.c | 13 +++++--- src/sqlite.h.in | 12 ++++--- src/test1.c | 6 ++-- src/vdbe.h | 2 +- src/vdbeInt.h | 2 +- src/vdbeapi.c | 8 ++++- src/vdbeaux.c | 2 +- src/where.c | 6 +--- test/scanstatus.test | 76 ++++++++++++++++++++++---------------------- 11 files changed, 85 insertions(+), 76 deletions(-) diff --git a/manifest b/manifest index a8453434a2..75d37178fb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fixes\sto\sthe\sWindows\sVFS\sto\sallow\smemory\smapped\sfiles\sto\swork\swithout\sWAL\ssupport. -D 2014-11-05T21:34:56.096 +C Change\sthe\sSQLITE_SCANSTAT_EST\sparameter\sso\sthat\sit\sreturns\sa\sdouble\sfor\nthe\sestimated\snumber\sof\soutput\srows\sper\sloop,\srather\sthan\sa\s64-bit\sinteger.\nRevise\sthe\soutput\sformat\sfor\sthe\s".scanstats\son"\sin\sthe\sshell\sto\smake\suse\nof\sthis\snew\scapability. +D 2014-11-06T03:55:10.745 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -228,8 +228,8 @@ F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b -F src/shell.c 5ad1eb4dfcd7a57e15825207a9bd559415bf34b1 -F src/sqlite.h.in 6e9af739d79f0bea2584b70fb1c54d3bb1a2eab6 +F src/shell.c 908ff96ef1551b28b940aaf4c886ba2681057209 +F src/sqlite.h.in e13a7b64efa8d6a591577e6a5281fb22783c0133 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 8f67ca79e957b8ece7453b8e320b6a996e1b4761 @@ -237,7 +237,7 @@ F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 81712116e826b0089bb221b018929536b2b5406f F src/table.c f142bba7903e93ca8d113a5b8877a108ad1a27dc F src/tclsqlite.c 7cdd4dd3c2a4183483feca260070d73d6e22cd47 -F src/test1.c 5890094c09691fe9564cf0f0d5b22d35b3218c47 +F src/test1.c ac7f3bad83ef4508d5efc85b32e86da48db8ed7e F src/test2.c 98049e51a17dc62606a99a9eb95ee477f9996712 F src/test3.c 1c0e5d6f080b8e33c1ce8b3078e7013fdbcd560c F src/test4.c 9b32d22f5f150abe23c1830e2057c4037c45b3df @@ -290,10 +290,10 @@ F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 3b627daa45c7308c1e36e3dbaa3f9ce7e5c7fa73 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a F src/vdbe.c 3fd4ebd3e87b63175bfd2be747608bae1670b4df -F src/vdbe.h d412bd01e89f0d69991b8f46601f96bc169d28f4 -F src/vdbeInt.h 539ba284790e871f98be74a78cbdfcedfae22639 -F src/vdbeapi.c 900259bdd85cd66a9f210d8ec08147a9034593bd -F src/vdbeaux.c cf6b8152dd22155201d57c216e6266866b61da59 +F src/vdbe.h 6fc69d9c5e146302c56e163cb4b31d1ee64a18c3 +F src/vdbeInt.h c32c1de25e3821a5b53d73abdb23ccc644ec5b63 +F src/vdbeapi.c 6a126fd8ed297ff0542bfbf7891b92977b5ed653 +F src/vdbeaux.c 9b0a251b6dfab349dd6c6efb40062eb7386b26f5 F src/vdbeblob.c 8b5442ff0954c44b45cbabbe2e94091a2e16fdef F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f F src/vdbesort.c 87f3923483113d1c95d84640becb4e4946f27d9a @@ -302,7 +302,7 @@ F src/vtab.c 2a30791bbd7926b589401bd09c3abb33de563793 F src/wal.c 825c948066c7604a07d56e67958cdab210749016 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 2c2081c546c90227577c502765611555503ce3f7 +F src/where.c 3862a1173ae2716bde12f1ab3fb649f1d85b05c2 F src/whereInt.h d3633e9b592103241b74b0ec76185f3e5b8b62e0 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -801,7 +801,7 @@ F test/savepoint4.test c8f8159ade6d2acd9128be61e1230f1c1edc6cc0 F test/savepoint5.test 0735db177e0ebbaedc39812c8d065075d563c4fd F test/savepoint6.test f41279c5e137139fa5c21485773332c7adb98cd7 F test/savepoint7.test fbf319a7b2dda089ec5be30a424a0e95f121d423 -F test/scanstatus.test 01afb2220f18ce85f9e338c20684f428d56e5c01 +F test/scanstatus.test a6dd739bc4d9638e8f5c2493b518057f2b681655 F test/schema.test 8f7999be894260f151adf15c2c7540f1c6d6a481 F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5 F test/schema3.test 1bc1008e1f8cb5654b248c55f27249366eb7ed38 @@ -1211,8 +1211,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 83a1e5db926b3a6d40f4a5cf9a8e6852b9bac9ac 6fc4ead26d19b9348bbda34c3053ae1e066abc32 -R 3ad47b2ac5693ba63ff5c591cf540d89 -T +closed 6fc4ead26d19b9348bbda34c3053ae1e066abc32 +P 272fddc14cc322655eeba670bc0f9fc30e5a804c +R c0f850c2227a7a137fbc73d5c1c5136e +T *branch * scanstatus +T *sym-scanstatus * +T -sym-trunk * U drh -Z ee3b32335a4c8630acaf81919b28b64e +Z f39cd1607e4aacbb491c044bd0bb317e diff --git a/manifest.uuid b/manifest.uuid index b558f09274..d125168b7d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -272fddc14cc322655eeba670bc0f9fc30e5a804c \ No newline at end of file +f9684000665ae7ef6f89c3773612b8286b8f545a \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index 3423bb0143..4dd0ffa1a1 100644 --- a/src/shell.c +++ b/src/shell.c @@ -1195,21 +1195,24 @@ static void display_scanstats( ){ #ifdef SQLITE_ENABLE_STMT_SCANSTATUS int i; + double rEstLoop = 1.0; fprintf(pArg->out, "-------- scanstats --------\n"); for(i=0; 1; i++){ sqlite3_stmt *p = pArg->pStmt; - sqlite3_int64 nEst, nLoop, nVisit; + sqlite3_int64 nLoop, nVisit; + double rEst; const char *zExplain; if( sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_NLOOP, (void*)&nLoop) ){ break; } sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_NVISIT, (void*)&nVisit); - sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_EST, (void*)&nEst); + sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_EST, (void*)&rEst); sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_EXPLAIN, (void*)&zExplain); - fprintf(pArg->out, "Loop %d: \"%s\"\n", i, zExplain); - fprintf(pArg->out, " nLoop=%-8lld nVisit=%-8lld nEst=%-8lld\n", - nLoop, nVisit, nEst + fprintf(pArg->out, "Loop %2d: \"%s\"\n", i, zExplain); + rEstLoop *= rEst; + fprintf(pArg->out, " nLoop=%-8lld nRow=%-8lld estRow=%-8lld estRow/Loop=%-8g\n", + nLoop, nVisit, (sqlite3_int64)rEstLoop, rEst ); } #else diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 44f7800ca2..c309dc1c5b 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -7440,13 +7440,15 @@ int sqlite3_vtab_on_conflict(sqlite3 *); ** ** [[SQLITE_SCANSTAT_NVISIT]]

SQLITE_SCANSTAT_NVISIT
**
^The [sqlite3_int64] variable pointed to by the T parameter will be set to the -** total number of rows visited by the X-th loop.
+** total number of rows examined by all iterations of the X-th loop. ** ** [[SQLITE_SCANSTAT_EST]]
SQLITE_SCANSTAT_EST
-**
^The [sqlite3_int64] variable pointed to by the T parameter will be set to the -** query planner's estimate for the number of rows visited for each -** iteration of the X-th loop. If the query planner's estimate was accurate, -** then this value should be approximately NVISIT/NLOOP. +**
^The "double" variable pointed to by the T parameter will be set to the +** query planner's estimate for the average number of rows output from each +** iteration of the X-th loop. If the query planner's estimates was accurate, +** then this value will approximate the quotient NVISIT/NLOOP and the +** product of this value for the first N-1 loops will approximate +** the NLOOP value for the N-th loop. ** ** [[SQLITE_SCANSTAT_NAME]]
SQLITE_SCANSTAT_NAME
**
^The "const char *" variable pointed to by the T parameter will be set to diff --git a/src/test1.c b/src/test1.c index 5e526b3013..fcc1a23de1 100644 --- a/src/test1.c +++ b/src/test1.c @@ -2318,7 +2318,7 @@ static int test_stmt_scanstatus( const char *zExplain; sqlite3_int64 nLoop; sqlite3_int64 nVisit; - sqlite3_int64 nEst; + double rEst; int res; if( objc!=3 ){ @@ -2336,9 +2336,9 @@ static int test_stmt_scanstatus( sqlite3_stmt_scanstatus(pStmt, idx, SQLITE_SCANSTAT_NVISIT, (void*)&nVisit); Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj("nVisit", -1)); Tcl_ListObjAppendElement(0, pRet, Tcl_NewWideIntObj(nVisit)); - sqlite3_stmt_scanstatus(pStmt, idx, SQLITE_SCANSTAT_EST, (void*)&nEst); + sqlite3_stmt_scanstatus(pStmt, idx, SQLITE_SCANSTAT_EST, (void*)&rEst); Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj("nEst", -1)); - Tcl_ListObjAppendElement(0, pRet, Tcl_NewWideIntObj(nEst)); + Tcl_ListObjAppendElement(0, pRet, Tcl_NewDoubleObj(rEst)); sqlite3_stmt_scanstatus(pStmt, idx, SQLITE_SCANSTAT_NAME, (void*)&zName); Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj("zName", -1)); Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zName, -1)); diff --git a/src/vdbe.h b/src/vdbe.h index 1b9ad8b6bf..b715241b41 100644 --- a/src/vdbe.h +++ b/src/vdbe.h @@ -283,7 +283,7 @@ void sqlite3VdbeLinkSubProgram(Vdbe *, SubProgram *); #endif #ifdef SQLITE_ENABLE_STMT_SCANSTATUS -void sqlite3VdbeScanStatus(Vdbe*, int, int, int, i64, const char*); +void sqlite3VdbeScanStatus(Vdbe*, int, int, int, LogEst, const char*); #else # define sqlite3VdbeScanStatus(a,b,c,d,e) #endif diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 29117dd064..a42ca3dffc 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -302,7 +302,7 @@ struct ScanStatus { int addrExplain; /* OP_Explain for loop */ int addrLoop; /* Address of "loops" counter */ int addrVisit; /* Address of "rows visited" counter */ - i64 nEst; /* Estimated rows per loop */ + LogEst nEst; /* Estimated output rows per loop */ char *zName; /* Name of table or index */ }; diff --git a/src/vdbeapi.c b/src/vdbeapi.c index d472004590..d907afee00 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -1500,7 +1500,13 @@ int sqlite3_stmt_scanstatus( break; } case SQLITE_SCANSTAT_EST: { - *(sqlite3_int64*)pOut = pScan->nEst; + double r = 1.0; + LogEst x = pScan->nEst; + while( x<100 ){ + x += 10; + r *= 0.5; + } + *(double*)pOut = r*sqlite3LogEstToInt(x); break; } case SQLITE_SCANSTAT_NAME: { diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 7cf996ce5c..d8ee5c8e8c 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -606,7 +606,7 @@ void sqlite3VdbeScanStatus( int addrExplain, /* Address of OP_Explain (or 0) */ int addrLoop, /* Address of loop counter */ int addrVisit, /* Address of rows visited counter */ - i64 nEst, /* Estimated number of rows */ + LogEst nEst, /* Estimated number of output rows */ const char *zName /* Name of table or index being scanned */ ){ int nByte = (p->nScan+1) * sizeof(ScanStatus); diff --git a/src/where.c b/src/where.c index 92f783b2e9..c3641c7cc7 100644 --- a/src/where.c +++ b/src/where.c @@ -2946,18 +2946,14 @@ static void addScanStatus( int addrExplain /* Address of OP_Explain (or 0) */ ){ const char *zObj = 0; - i64 nEst = 1; WhereLoop *pLoop = pLvl->pWLoop; if( (pLoop->wsFlags & (WHERE_IPK|WHERE_VIRTUALTABLE))==0 ){ zObj = pLoop->u.btree.pIndex->zName; }else{ zObj = pSrclist->a[pLvl->iFrom].zName; } - if( pLoop->nOut>=10 ){ - nEst = sqlite3LogEstToInt(pLoop->nOut); - } sqlite3VdbeScanStatus( - v, addrExplain, pLvl->addrBody, pLvl->addrVisit, nEst, zObj + v, addrExplain, pLvl->addrBody, pLvl->addrVisit, pLoop->nOut, zObj ); } #else diff --git a/test/scanstatus.test b/test/scanstatus.test index 9b34506e91..7713bae5fc 100644 --- a/test/scanstatus.test +++ b/test/scanstatus.test @@ -45,8 +45,8 @@ proc do_scanstatus_test {tn res} { do_execsql_test 1.1 { SELECT count(*) FROM t1, t2; } 6 do_scanstatus_test 1.2 { - nLoop 1 nVisit 2 nEst 1048576 zName t1 zExplain {SCAN TABLE t1} - nLoop 2 nVisit 6 nEst 1048576 zName t2 zExplain {SCAN TABLE t2} + nLoop 1 nVisit 2 nEst 1048576.0 zName t1 zExplain {SCAN TABLE t1} + nLoop 2 nVisit 6 nEst 1048576.0 zName t2 zExplain {SCAN TABLE t2} } do_execsql_test 1.3 { @@ -54,8 +54,8 @@ do_execsql_test 1.3 { SELECT count(*) FROM t1, t2; } 6 do_scanstatus_test 1.4 { - nLoop 1 nVisit 2 nEst 2 zName t1 zExplain {SCAN TABLE t1} - nLoop 2 nVisit 6 nEst 3 zName t2 zExplain {SCAN TABLE t2} + nLoop 1 nVisit 2 nEst 2.0 zName t1 zExplain {SCAN TABLE t1} + nLoop 2 nVisit 6 nEst 3.0 zName t2 zExplain {SCAN TABLE t2} } do_execsql_test 1.5 { ANALYZE } @@ -63,9 +63,9 @@ do_execsql_test 1.6 { SELECT count(*) FROM t1, t2 WHERE t2.rowid>1; } 4 do_scanstatus_test 1.7 { - nLoop 1 nVisit 2 nEst 2 zName t2 zExplain + nLoop 1 nVisit 2 nEst 2.0 zName t2 zExplain {SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid>?)} - nLoop 2 nVisit 4 nEst 2 zName t1 zExplain {SCAN TABLE t1} + nLoop 2 nVisit 4 nEst 2.0 zName t1 zExplain {SCAN TABLE t1} } do_execsql_test 1.8 { @@ -73,9 +73,9 @@ do_execsql_test 1.8 { } 4 do_scanstatus_test 1.9 { - nLoop 2 nVisit 4 nEst 2 zName t2 zExplain + nLoop 2 nVisit 4 nEst 2.0 zName t2 zExplain {SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid>?)} - nLoop 4 nVisit 8 nEst 2 zName t1 zExplain {SCAN TABLE t1} + nLoop 4 nVisit 8 nEst 2.0 zName t1 zExplain {SCAN TABLE t1} } do_test 1.9 { @@ -83,9 +83,9 @@ do_test 1.9 { } {} do_scanstatus_test 1.10 { - nLoop 0 nVisit 0 nEst 2 zName t2 zExplain + nLoop 0 nVisit 0 nEst 2.0 zName t2 zExplain {SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid>?)} - nLoop 0 nVisit 0 nEst 2 zName t1 zExplain {SCAN TABLE t1} + nLoop 0 nVisit 0 nEst 2.0 zName t1 zExplain {SCAN TABLE t1} } #------------------------------------------------------------------------- @@ -104,7 +104,7 @@ do_execsql_test 2.1 { } {2 two} do_scanstatus_test 2.2 { - nLoop 1 nVisit 1 nEst 1 zName x1 + nLoop 1 nVisit 1 nEst 1.0 zName x1 zExplain {SEARCH TABLE x1 USING INTEGER PRIMARY KEY (rowid=?)} } @@ -112,7 +112,7 @@ do_execsql_test 2.3.1 { SELECT * FROM x1 WHERE j='two' } {2 two} do_scanstatus_test 2.3.2 { - nLoop 1 nVisit 1 nEst 10 zName x1j + nLoop 1 nVisit 1 nEst 10.0 zName x1j zExplain {SEARCH TABLE x1 USING COVERING INDEX x1j (j=?)} } @@ -120,7 +120,7 @@ do_execsql_test 2.4.1 { SELECT * FROM x1 WHERE j<'two' } {4 four 1 one 3 three} do_scanstatus_test 2.4.2 { - nLoop 1 nVisit 3 nEst 262144 zName x1j + nLoop 1 nVisit 3 nEst 262144.0 zName x1j zExplain {SEARCH TABLE x1 USING COVERING INDEX x1j (j='two' } {2 two} do_scanstatus_test 2.5.2 { - nLoop 1 nVisit 1 nEst 262144 zName x1j + nLoop 1 nVisit 1 nEst 262144.0 zName x1j zExplain {SEARCH TABLE x1 USING COVERING INDEX x1j (j>?)} } @@ -136,7 +136,7 @@ do_execsql_test 2.6.1 { SELECT * FROM x1 WHERE j BETWEEN 'three' AND 'two' } {3 three 2 two} do_scanstatus_test 2.6.2 { - nLoop 1 nVisit 2 nEst 16384 zName x1j + nLoop 1 nVisit 2 nEst 16384.0 zName x1j zExplain {SEARCH TABLE x1 USING COVERING INDEX x1j (j>? AND j? AND j? AND a? AND b? AND b? AND a Date: Thu, 6 Nov 2014 04:42:20 +0000 Subject: [PATCH 083/133] Add the SQLITE_SCANSTAT_SELECTID metric. Use it to improve the ".stmtscan on" output in the shell. FossilOrigin-Name: 64ad5761a841f71530d41565b9fbe9d19c2d6aff --- manifest | 21 +++++++++------------ manifest.uuid | 2 +- src/shell.c | 22 +++++++++++++++++----- src/sqlite.h.in | 12 ++++++++++-- src/vdbeInt.h | 1 + src/vdbeapi.c | 8 ++++++++ 6 files changed, 46 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index 75d37178fb..25d04501fe 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\sSQLITE_SCANSTAT_EST\sparameter\sso\sthat\sit\sreturns\sa\sdouble\sfor\nthe\sestimated\snumber\sof\soutput\srows\sper\sloop,\srather\sthan\sa\s64-bit\sinteger.\nRevise\sthe\soutput\sformat\sfor\sthe\s".scanstats\son"\sin\sthe\sshell\sto\smake\suse\nof\sthis\snew\scapability. -D 2014-11-06T03:55:10.745 +C Add\sthe\sSQLITE_SCANSTAT_SELECTID\smetric.\s\sUse\sit\sto\simprove\sthe\n".stmtscan\son"\soutput\sin\sthe\sshell. +D 2014-11-06T04:42:20.310 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -228,8 +228,8 @@ F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b -F src/shell.c 908ff96ef1551b28b940aaf4c886ba2681057209 -F src/sqlite.h.in e13a7b64efa8d6a591577e6a5281fb22783c0133 +F src/shell.c 74768f90bd0f8880937d52e2eb756655dba0015a +F src/sqlite.h.in 087d30a4c7ec7ae19bcaa03a9db9d6ee7a73b0b3 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 8f67ca79e957b8ece7453b8e320b6a996e1b4761 @@ -291,8 +291,8 @@ F src/util.c 3b627daa45c7308c1e36e3dbaa3f9ce7e5c7fa73 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a F src/vdbe.c 3fd4ebd3e87b63175bfd2be747608bae1670b4df F src/vdbe.h 6fc69d9c5e146302c56e163cb4b31d1ee64a18c3 -F src/vdbeInt.h c32c1de25e3821a5b53d73abdb23ccc644ec5b63 -F src/vdbeapi.c 6a126fd8ed297ff0542bfbf7891b92977b5ed653 +F src/vdbeInt.h 9bb69ff2447c34b6ccc58b34ec35b615f86ead78 +F src/vdbeapi.c 07acb615d1e4170e71fc1b0d087f3c53a1ad8e83 F src/vdbeaux.c 9b0a251b6dfab349dd6c6efb40062eb7386b26f5 F src/vdbeblob.c 8b5442ff0954c44b45cbabbe2e94091a2e16fdef F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f @@ -1211,10 +1211,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 272fddc14cc322655eeba670bc0f9fc30e5a804c -R c0f850c2227a7a137fbc73d5c1c5136e -T *branch * scanstatus -T *sym-scanstatus * -T -sym-trunk * +P f9684000665ae7ef6f89c3773612b8286b8f545a +R 34f47ade33b56f41a439ae08635a112b U drh -Z f39cd1607e4aacbb491c044bd0bb317e +Z 216d325be8d3eddf6fd433a7ff464585 diff --git a/manifest.uuid b/manifest.uuid index d125168b7d..13ff805ffa 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f9684000665ae7ef6f89c3773612b8286b8f545a \ No newline at end of file +64ad5761a841f71530d41565b9fbe9d19c2d6aff \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index 4dd0ffa1a1..b6ae6e15a6 100644 --- a/src/shell.c +++ b/src/shell.c @@ -1195,12 +1195,14 @@ static void display_scanstats( ){ #ifdef SQLITE_ENABLE_STMT_SCANSTATUS int i; - double rEstLoop = 1.0; + double *arEstLoop = 0; + int nEstLoop = 0; fprintf(pArg->out, "-------- scanstats --------\n"); for(i=0; 1; i++){ sqlite3_stmt *p = pArg->pStmt; sqlite3_int64 nLoop, nVisit; - double rEst; + double rEst, rLoop; + int iSid; const char *zExplain; if( sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_NLOOP, (void*)&nLoop) ){ break; @@ -1208,13 +1210,23 @@ static void display_scanstats( sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_NVISIT, (void*)&nVisit); sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_EST, (void*)&rEst); sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_EXPLAIN, (void*)&zExplain); - + sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_SELECTID, (void*)&iSid); + if( iSid>=nEstLoop ){ + arEstLoop = sqlite3_realloc(arEstLoop, sizeof(arEstLoop[0])*(iSid+1) ); + while( nEstLoop<=iSid ) arEstLoop[nEstLoop++] = 1.0; + } + if( iSid>=0 ){ + arEstLoop[iSid] *= rEst; + rLoop = arEstLoop[iSid]; + }else{ + rLoop = rEst; + } fprintf(pArg->out, "Loop %2d: \"%s\"\n", i, zExplain); - rEstLoop *= rEst; fprintf(pArg->out, " nLoop=%-8lld nRow=%-8lld estRow=%-8lld estRow/Loop=%-8g\n", - nLoop, nVisit, (sqlite3_int64)rEstLoop, rEst + nLoop, nVisit, (sqlite3_int64)rLoop, rEst ); } + sqlite3_free(arEstLoop); #else fprintf(pArg->out, "-------- scanstats --------\n"); fprintf(pArg->out, diff --git a/src/sqlite.h.in b/src/sqlite.h.in index c309dc1c5b..00bf5e2d98 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -7447,8 +7447,8 @@ int sqlite3_vtab_on_conflict(sqlite3 *); ** query planner's estimate for the average number of rows output from each ** iteration of the X-th loop. If the query planner's estimates was accurate, ** then this value will approximate the quotient NVISIT/NLOOP and the -** product of this value for the first N-1 loops will approximate -** the NLOOP value for the N-th loop. +** product of this value for all prior loops with the same SELECTID will +** be the NLOOP value for the current loop. ** ** [[SQLITE_SCANSTAT_NAME]]
SQLITE_SCANSTAT_NAME
**
^The "const char *" variable pointed to by the T parameter will be set to @@ -7459,6 +7459,13 @@ int sqlite3_vtab_on_conflict(sqlite3 *); **
^The "const char *" variable pointed to by the T parameter will be set to ** a zero-terminated UTF-8 string containing the [EXPLAIN QUERY PLAN] description ** for the X-th loop. +** +** [[SQLITE_SCANSTAT_SELECTID]]
SQLITE_SCANSTAT_SELECT
+**
^The "int" variable pointed to by the T parameter will be set to the +** "select-id" for the X-th loop. The select-id identifies which query or +** subquery the loop is part of. The main query has a select-id of zero. +** The select-id is the same value as is output in the first column +** of an [EXPLAIN QUERY PLAN] query. ** */ #define SQLITE_SCANSTAT_NLOOP 0 @@ -7466,6 +7473,7 @@ int sqlite3_vtab_on_conflict(sqlite3 *); #define SQLITE_SCANSTAT_EST 2 #define SQLITE_SCANSTAT_NAME 3 #define SQLITE_SCANSTAT_EXPLAIN 4 +#define SQLITE_SCANSTAT_SELECTID 5 /* ** CAPI3REF: Prepared Statement Scan Status diff --git a/src/vdbeInt.h b/src/vdbeInt.h index a42ca3dffc..1a7297e946 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -302,6 +302,7 @@ struct ScanStatus { int addrExplain; /* OP_Explain for loop */ int addrLoop; /* Address of "loops" counter */ int addrVisit; /* Address of "rows visited" counter */ + int iSelectID; /* The "Select-ID" for this loop */ LogEst nEst; /* Estimated output rows per loop */ char *zName; /* Name of table or index */ }; diff --git a/src/vdbeapi.c b/src/vdbeapi.c index d907afee00..5744c28632 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -1521,6 +1521,14 @@ int sqlite3_stmt_scanstatus( } break; } + case SQLITE_SCANSTAT_SELECTID: { + if( pScan->addrExplain ){ + *(int*)pOut = p->aOp[ pScan->addrExplain ].p1; + }else{ + *(int*)pOut = -1; + } + break; + } default: { return 1; } From 42f30bce1158008097d271fd06c31398cf631aea Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 6 Nov 2014 12:08:21 +0000 Subject: [PATCH 084/133] Changes the formatting of ".scanstats on" in the shell so that the stats for subqueries are grouped together and occur after the main query. FossilOrigin-Name: eacbbd8849db9b023eff15ef1cb42ec941299433 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c | 54 +++++++++++++++++++++++---------------------------- 3 files changed, 31 insertions(+), 37 deletions(-) diff --git a/manifest b/manifest index 25d04501fe..af1ed3e99c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sSQLITE_SCANSTAT_SELECTID\smetric.\s\sUse\sit\sto\simprove\sthe\n".stmtscan\son"\soutput\sin\sthe\sshell. -D 2014-11-06T04:42:20.310 +C Changes\sthe\sformatting\sof\s".scanstats\son"\sin\sthe\sshell\sso\sthat\sthe\sstats\sfor\nsubqueries\sare\sgrouped\stogether\sand\soccur\safter\sthe\smain\squery. +D 2014-11-06T12:08:21.237 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -228,7 +228,7 @@ F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b -F src/shell.c 74768f90bd0f8880937d52e2eb756655dba0015a +F src/shell.c 22c7c693f322091b26e9333a8fa50c56e4aba667 F src/sqlite.h.in 087d30a4c7ec7ae19bcaa03a9db9d6ee7a73b0b3 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d @@ -1211,7 +1211,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f9684000665ae7ef6f89c3773612b8286b8f545a -R 34f47ade33b56f41a439ae08635a112b +P 64ad5761a841f71530d41565b9fbe9d19c2d6aff +R 301cd68395834791ac0b2ebb68f39e69 U drh -Z 216d325be8d3eddf6fd433a7ff464585 +Z c6dcae9ced19f9079c51a47717359be1 diff --git a/manifest.uuid b/manifest.uuid index 13ff805ffa..6061834c99 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -64ad5761a841f71530d41565b9fbe9d19c2d6aff \ No newline at end of file +eacbbd8849db9b023eff15ef1cb42ec941299433 \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index b6ae6e15a6..3505e59d4b 100644 --- a/src/shell.c +++ b/src/shell.c @@ -1194,39 +1194,33 @@ static void display_scanstats( ShellState *pArg /* Pointer to ShellState */ ){ #ifdef SQLITE_ENABLE_STMT_SCANSTATUS - int i; - double *arEstLoop = 0; - int nEstLoop = 0; + int i, k, n = 1; fprintf(pArg->out, "-------- scanstats --------\n"); - for(i=0; 1; i++){ - sqlite3_stmt *p = pArg->pStmt; - sqlite3_int64 nLoop, nVisit; - double rEst, rLoop; - int iSid; - const char *zExplain; - if( sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_NLOOP, (void*)&nLoop) ){ - break; + for(k=0; n>0; k++){ + double rEstLoop = 1.0; + for(i=n=0; 1; i++){ + sqlite3_stmt *p = pArg->pStmt; + sqlite3_int64 nLoop, nVisit; + double rEst; + int iSid; + const char *zExplain; + if( sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_NLOOP, (void*)&nLoop) ){ + break; + } + sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_SELECTID, (void*)&iSid); + if( iSid!=k ) continue; + if( n==0 && k>0 ) fprintf(pArg->out, "-------- subquery %d --------\n", k); + n++; + sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_NVISIT, (void*)&nVisit); + sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_EST, (void*)&rEst); + sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_EXPLAIN, (void*)&zExplain); + fprintf(pArg->out, "Loop %2d: %s\n", n, zExplain); + rEstLoop *= rEst; + fprintf(pArg->out, " nLoop=%-8lld nRow=%-8lld estRow=%-8lld estRow/Loop=%-8g\n", + nLoop, nVisit, (sqlite3_int64)rEstLoop, rEst + ); } - sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_NVISIT, (void*)&nVisit); - sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_EST, (void*)&rEst); - sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_EXPLAIN, (void*)&zExplain); - sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_SELECTID, (void*)&iSid); - if( iSid>=nEstLoop ){ - arEstLoop = sqlite3_realloc(arEstLoop, sizeof(arEstLoop[0])*(iSid+1) ); - while( nEstLoop<=iSid ) arEstLoop[nEstLoop++] = 1.0; - } - if( iSid>=0 ){ - arEstLoop[iSid] *= rEst; - rLoop = arEstLoop[iSid]; - }else{ - rLoop = rEst; - } - fprintf(pArg->out, "Loop %2d: \"%s\"\n", i, zExplain); - fprintf(pArg->out, " nLoop=%-8lld nRow=%-8lld estRow=%-8lld estRow/Loop=%-8g\n", - nLoop, nVisit, (sqlite3_int64)rLoop, rEst - ); } - sqlite3_free(arEstLoop); #else fprintf(pArg->out, "-------- scanstats --------\n"); fprintf(pArg->out, From 179bac3a14924c42656efdfddc5bbfd8e8a2be32 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 6 Nov 2014 12:17:24 +0000 Subject: [PATCH 085/133] On the ".scanstats on" output in the shell, initialize the estimated count for the first loop of each subquery to the actual loop count. FossilOrigin-Name: d1c51c8455d5ce972a77720c2d56228646ced27c --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c | 5 ++++- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index af1ed3e99c..05212e91e7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Changes\sthe\sformatting\sof\s".scanstats\son"\sin\sthe\sshell\sso\sthat\sthe\sstats\sfor\nsubqueries\sare\sgrouped\stogether\sand\soccur\safter\sthe\smain\squery. -D 2014-11-06T12:08:21.237 +C On\sthe\s".scanstats\son"\soutput\sin\sthe\sshell,\sinitialize\sthe\sestimated\scount\sfor\nthe\sfirst\sloop\sof\seach\ssubquery\sto\sthe\sactual\sloop\scount. +D 2014-11-06T12:17:24.789 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -228,7 +228,7 @@ F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b -F src/shell.c 22c7c693f322091b26e9333a8fa50c56e4aba667 +F src/shell.c 64a941c079837fd1a0d920273832e6275b777402 F src/sqlite.h.in 087d30a4c7ec7ae19bcaa03a9db9d6ee7a73b0b3 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d @@ -1211,7 +1211,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 64ad5761a841f71530d41565b9fbe9d19c2d6aff -R 301cd68395834791ac0b2ebb68f39e69 +P eacbbd8849db9b023eff15ef1cb42ec941299433 +R c65acc2e5374aae8ca471af05c87c9aa U drh -Z c6dcae9ced19f9079c51a47717359be1 +Z b18ed09265a53348fca40701c374436f diff --git a/manifest.uuid b/manifest.uuid index 6061834c99..fc422bc396 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -eacbbd8849db9b023eff15ef1cb42ec941299433 \ No newline at end of file +d1c51c8455d5ce972a77720c2d56228646ced27c \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index 3505e59d4b..ca49e00a69 100644 --- a/src/shell.c +++ b/src/shell.c @@ -1209,7 +1209,10 @@ static void display_scanstats( } sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_SELECTID, (void*)&iSid); if( iSid!=k ) continue; - if( n==0 && k>0 ) fprintf(pArg->out, "-------- subquery %d --------\n", k); + if( n==0 ){ + rEstLoop = (double)nLoop; + if( k>0 ) fprintf(pArg->out, "-------- subquery %d --------\n", k); + } n++; sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_NVISIT, (void*)&nVisit); sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_EST, (void*)&rEst); From 15f23c2cf0da0ecd46f588a156af251fb58c3234 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 6 Nov 2014 12:46:16 +0000 Subject: [PATCH 086/133] Further improvements to the ".scanstats on" display in the shell. Be sure to show the results of all subqueries even if there are gaps in the SELECTID values. Add ".scanstats" to the ".help" output. FossilOrigin-Name: ee922682bb7235dbcd23a22fcfdfa188f6d3228a --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c | 20 ++++++++++---------- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index 05212e91e7..3bbd6d3bd3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C On\sthe\s".scanstats\son"\soutput\sin\sthe\sshell,\sinitialize\sthe\sestimated\scount\sfor\nthe\sfirst\sloop\sof\seach\ssubquery\sto\sthe\sactual\sloop\scount. -D 2014-11-06T12:17:24.789 +C Further\simprovements\sto\sthe\s".scanstats\son"\sdisplay\sin\sthe\sshell.\s\sBe\ssure\nto\sshow\sthe\sresults\sof\sall\ssubqueries\seven\sif\sthere\sare\sgaps\sin\sthe\s\nSELECTID\svalues.\s\sAdd\s".scanstats"\sto\sthe\s".help"\soutput. +D 2014-11-06T12:46:16.708 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -228,7 +228,7 @@ F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b -F src/shell.c 64a941c079837fd1a0d920273832e6275b777402 +F src/shell.c 48fe276aada42a15722aee2584e6321345ed4609 F src/sqlite.h.in 087d30a4c7ec7ae19bcaa03a9db9d6ee7a73b0b3 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d @@ -1211,7 +1211,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P eacbbd8849db9b023eff15ef1cb42ec941299433 -R c65acc2e5374aae8ca471af05c87c9aa +P d1c51c8455d5ce972a77720c2d56228646ced27c +R 1f66f00948a1acda590c099646f9e3d3 U drh -Z b18ed09265a53348fca40701c374436f +Z 2bc412ec02d784f50574dd160207db0a diff --git a/manifest.uuid b/manifest.uuid index fc422bc396..5ac436c534 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d1c51c8455d5ce972a77720c2d56228646ced27c \ No newline at end of file +ee922682bb7235dbcd23a22fcfdfa188f6d3228a \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index ca49e00a69..915b69263b 100644 --- a/src/shell.c +++ b/src/shell.c @@ -1194,9 +1194,10 @@ static void display_scanstats( ShellState *pArg /* Pointer to ShellState */ ){ #ifdef SQLITE_ENABLE_STMT_SCANSTATUS - int i, k, n = 1; + int i, k, n, mx; fprintf(pArg->out, "-------- scanstats --------\n"); - for(k=0; n>0; k++){ + mx = 0; + for(k=0; k<=mx; k++){ double rEstLoop = 1.0; for(i=n=0; 1; i++){ sqlite3_stmt *p = pArg->pStmt; @@ -1208,10 +1209,11 @@ static void display_scanstats( break; } sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_SELECTID, (void*)&iSid); + if( iSid>mx ) mx = iSid; if( iSid!=k ) continue; if( n==0 ){ rEstLoop = (double)nLoop; - if( k>0 ) fprintf(pArg->out, "-------- subquery %d --------\n", k); + if( k>0 ) fprintf(pArg->out, "-------- subquery %d -------\n", k); } n++; sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_NVISIT, (void*)&nVisit); @@ -1224,14 +1226,8 @@ static void display_scanstats( ); } } -#else - fprintf(pArg->out, "-------- scanstats --------\n"); - fprintf(pArg->out, - "sqlite3_stmt_scanstatus() unavailable - " - "rebuild with SQLITE_ENABLE_STMT_SCANSTATUS\n" - ); -#endif fprintf(pArg->out, "---------------------------\n"); +#endif } /* @@ -1687,6 +1683,7 @@ static char zHelp[] = ".read FILENAME Execute SQL in FILENAME\n" ".restore ?DB? FILE Restore content of DB (default \"main\") from FILE\n" ".save FILE Write in-memory database into FILE\n" + ".scanstats on|off Turn sqlite3_stmt_scanstatus() metrics on or off\n" ".schema ?TABLE? Show the CREATE statements\n" " If TABLE specified, only show tables matching\n" " LIKE pattern TABLE.\n" @@ -3072,6 +3069,9 @@ static int do_meta_command(char *zLine, ShellState *p){ if( c=='s' && strncmp(azArg[0], "scanstats", n)==0 ){ if( nArg==2 ){ p->scanstatsOn = booleanValue(azArg[1]); +#ifndef SQLITE_ENABLE_STMT_SCANSTATUS + fprintf(stderr, "Warning: .scanstats not available in this build.\n"); +#endif }else{ fprintf(stderr, "Usage: .scanstats on|off\n"); rc = 1; From 8790b6e860fb5c9dacc3ddc1ecb27b384dd71c2a Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 7 Nov 2014 01:43:56 +0000 Subject: [PATCH 087/133] Update documentation on sqlite3_config() and add corresponding evidence marks. FossilOrigin-Name: 360c8ca11c3315c8e08c7c52ff5468e3f723e562 --- manifest | 15 +++++++-------- manifest.uuid | 2 +- src/main.c | 32 ++++++++++++++++++++++---------- src/sqlite.h.in | 10 ++++++---- 4 files changed, 36 insertions(+), 23 deletions(-) diff --git a/manifest b/manifest index 277b0baf4c..2ad90a7f0d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Added\sSQLITE_SCANSTAT_SELECTID.\s\sChange\sthe\svalue\sreturned\sby\nSQLITE_SCANSTAT_EST\sfrom\ssqlite3_int64\sto\sdouble.\s\sEnhanced\sthe\sformatting\nand\sdisplay\sof\sscan\sstatistics\susing\sthe\s".scanstats\son"\scommand\sin\sthe\nshell. -D 2014-11-06T14:43:53.299 +C Update\sdocumentation\son\ssqlite3_config()\sand\sadd\scorresponding\sevidence\smarks. +D 2014-11-07T01:43:56.762 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -194,7 +194,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 F src/loadext.c de741e66e5ddc1598d904d7289239696e40ed994 -F src/main.c 42f857be3cef3e1f9752d8e46d61345f80932396 +F src/main.c f88ed28716cbbada0f3d81479e6d43823b553de6 F src/malloc.c 740db54387204c9a2eb67c6d98e68b08e9ef4eab F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c faf615aafd8be74a71494dfa027c113ea5c6615f @@ -229,7 +229,7 @@ F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c 48fe276aada42a15722aee2584e6321345ed4609 -F src/sqlite.h.in 087d30a4c7ec7ae19bcaa03a9db9d6ee7a73b0b3 +F src/sqlite.h.in 83e34312bc974a99f03e1b040854c18145d7b662 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 8f67ca79e957b8ece7453b8e320b6a996e1b4761 @@ -1211,8 +1211,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 272fddc14cc322655eeba670bc0f9fc30e5a804c ee922682bb7235dbcd23a22fcfdfa188f6d3228a -R 1f66f00948a1acda590c099646f9e3d3 -T +closed ee922682bb7235dbcd23a22fcfdfa188f6d3228a +P 20c7614addb8494cd7f40263a50fa6f428cce1c7 +R ccc709fd5070dead531e2a6ad6b5aa29 U drh -Z 1da4b43998d0836d48029478818e14c6 +Z 3b414dc01fd5a3ebe6b3ed14cc9e0f06 diff --git a/manifest.uuid b/manifest.uuid index 1e9e58656e..01580f60c0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -20c7614addb8494cd7f40263a50fa6f428cce1c7 \ No newline at end of file +360c8ca11c3315c8e08c7c52ff5468e3f723e562 \ No newline at end of file diff --git a/src/main.c b/src/main.c index 4c33bbde9d..ae3753c139 100644 --- a/src/main.c +++ b/src/main.c @@ -457,6 +457,9 @@ int sqlite3_config(int op, ...){ break; } +/* EVIDENCE-OF: R-06626-12911 The SQLITE_CONFIG_HEAP option is only +** available if SQLite is compiled with either SQLITE_ENABLE_MEMSYS3 or +** SQLITE_ENABLE_MEMSYS5 and returns SQLITE_ERROR if invoked otherwise. */ #if defined(SQLITE_ENABLE_MEMSYS3) || defined(SQLITE_ENABLE_MEMSYS5) case SQLITE_CONFIG_HEAP: { /* EVIDENCE-OF: R-19854-42126 There are three arguments to @@ -474,17 +477,19 @@ int sqlite3_config(int op, ...){ } if( sqlite3GlobalConfig.pHeap==0 ){ - /* If the heap pointer is NULL, then restore the malloc implementation - ** back to NULL pointers too. This will cause the malloc to go - ** back to its default implementation when sqlite3_initialize() is - ** run. + /* EVIDENCE-OF: R-49920-60189 If the first pointer (the memory pointer) + ** is NULL, then SQLite reverts to using its default memory allocator + ** (the system malloc() implementation), undoing any prior invocation of + ** SQLITE_CONFIG_MALLOC. + ** + ** Setting sqlite3GlobalConfig.m to all zeros will cause malloc to + ** revert to its default implementation when sqlite3_initialize() is run */ memset(&sqlite3GlobalConfig.m, 0, sizeof(sqlite3GlobalConfig.m)); }else{ - /* The heap pointer is not NULL, then install one of the - ** mem5.c/mem3.c methods. The enclosing #if guarantees at - ** least one of these methods is currently enabled. - */ + /* EVIDENCE-OF: R-61006-08918 If the memory pointer is not NULL then the + ** alternative memory allocator is engaged to handle all of SQLites + ** memory allocation needs. */ #ifdef SQLITE_ENABLE_MEMSYS3 sqlite3GlobalConfig.m = *sqlite3MemGetMemsys3(); #endif @@ -557,7 +562,13 @@ int sqlite3_config(int op, ...){ sqlite3_int64 szMmap = va_arg(ap, sqlite3_int64); sqlite3_int64 mxMmap = va_arg(ap, sqlite3_int64); /* EVIDENCE-OF: R-53367-43190 If either argument to this option is - ** negative, then that argument is changed to its compile-time default. */ + ** negative, then that argument is changed to its compile-time default. + ** + ** EVIDENCE-OF: R-34993-45031 The maximum allowed mmap size will be + ** silently truncated if necessary so that it does not exceed the + ** compile-time maximum mmap size set by the SQLITE_MAX_MMAP_SIZE + ** compile-time option. + */ if( mxMmap<0 || mxMmap>SQLITE_MAX_MMAP_SIZE ) mxMmap = SQLITE_MAX_MMAP_SIZE; if( szMmap<0 ) szMmap = SQLITE_DEFAULT_MMAP_SIZE; if( szMmap>mxMmap) szMmap = mxMmap; @@ -2389,7 +2400,8 @@ int sqlite3ParseUri( assert( *pzErrMsg==0 ); - if( ((flags & SQLITE_OPEN_URI) || sqlite3GlobalConfig.bOpenUri) + if( ((flags & SQLITE_OPEN_URI) /* IMP: R-48725-32206 */ + || sqlite3GlobalConfig.bOpenUri) /* IMP: R-51689-46548 */ && nUri>=5 && memcmp(zUri, "file:", 5)==0 /* IMP: R-57884-37496 */ ){ char *zOpt; diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 00bf5e2d98..192280e726 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -1582,14 +1582,16 @@ struct sqlite3_mem_methods { **
^The SQLITE_CONFIG_HEAP option specifies a static memory buffer ** that SQLite will use for all of its dynamic memory allocation needs ** beyond those provided for by [SQLITE_CONFIG_SCRATCH] and [SQLITE_CONFIG_PAGECACHE]. +** ^The SQLITE_CONFIG_HEAP option is only available if SQLite is compiled +** with either [SQLITE_ENABLE_MEMSYS3] or [SQLITE_ENABLE_MEMSYS5] and returns +** [SQLITE_ERROR] if invoked otherwise. ** ^There are three arguments to SQLITE_CONFIG_HEAP: ** An 8-byte aligned pointer to the memory, ** the number of bytes in the memory buffer, and the minimum allocation size. ** ^If the first pointer (the memory pointer) is NULL, then SQLite reverts ** to using its default memory allocator (the system malloc() implementation), ** undoing any prior invocation of [SQLITE_CONFIG_MALLOC]. ^If the -** memory pointer is not NULL and either [SQLITE_ENABLE_MEMSYS3] or -** [SQLITE_ENABLE_MEMSYS5] are defined, then the alternative memory +** memory pointer is not NULL then the alternative memory ** allocator is engaged to handle all of SQLites memory allocation needs. ** The first pointer (the memory pointer) must be aligned to an 8-byte ** boundary or subsequent behavior of SQLite will be undefined. @@ -1719,8 +1721,8 @@ struct sqlite3_mem_methods { ** ^The default setting can be overridden by each database connection using ** either the [PRAGMA mmap_size] command, or by using the ** [SQLITE_FCNTL_MMAP_SIZE] file control. ^(The maximum allowed mmap size -** cannot be changed at run-time. Nor may the maximum allowed mmap size -** exceed the compile-time maximum mmap size set by the +** will be silently truncated if necessary so that it does not exceed the +** compile-time maximum mmap size set by the ** [SQLITE_MAX_MMAP_SIZE] compile-time option.)^ ** ^If either argument to this option is negative, then that argument is ** changed to its compile-time default. From 6137138ded9c5ca02d65d62d0ec4f3ff7cde05d2 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 7 Nov 2014 11:39:16 +0000 Subject: [PATCH 088/133] Fix typo in sqlite3.h reported on the mailing list. FossilOrigin-Name: 402703212a8488f8b571ce170b3b6c7374bd7daa --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/sqlite.h.in | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 2ad90a7f0d..b80e3b8250 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sdocumentation\son\ssqlite3_config()\sand\sadd\scorresponding\sevidence\smarks. -D 2014-11-07T01:43:56.762 +C Fix\stypo\sin\ssqlite3.h\sreported\son\sthe\smailing\slist. +D 2014-11-07T11:39:16.680 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -229,7 +229,7 @@ F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c 48fe276aada42a15722aee2584e6321345ed4609 -F src/sqlite.h.in 83e34312bc974a99f03e1b040854c18145d7b662 +F src/sqlite.h.in f433227d7f619887a1064913fa66cefa3da4349a F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 8f67ca79e957b8ece7453b8e320b6a996e1b4761 @@ -1211,7 +1211,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 20c7614addb8494cd7f40263a50fa6f428cce1c7 -R ccc709fd5070dead531e2a6ad6b5aa29 +P 360c8ca11c3315c8e08c7c52ff5468e3f723e562 +R 367fba2464fa0866db1d5cae2a24beee U drh -Z 3b414dc01fd5a3ebe6b3ed14cc9e0f06 +Z 3885b799624859bdb0af87865d5681ed diff --git a/manifest.uuid b/manifest.uuid index 01580f60c0..185fd7e51d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -360c8ca11c3315c8e08c7c52ff5468e3f723e562 \ No newline at end of file +402703212a8488f8b571ce170b3b6c7374bd7daa \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 192280e726..fc741b7291 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -52,7 +52,7 @@ extern "C" { /* ** These no-op macros are used in front of interfaces to mark those ** interfaces as either deprecated or experimental. New applications -** should not use deprecated interfaces - they are support for backwards +** should not use deprecated interfaces - they are supported for backwards ** compatibility only. Application writers should be aware that ** experimental interfaces are subject to change in point releases. ** From 86a11b8a6a671c1f0d55c7adb47994f2cec37f01 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 7 Nov 2014 13:24:29 +0000 Subject: [PATCH 089/133] Fix harmless typos in comments. FossilOrigin-Name: 94c564da4c2cf5dffe58fdf7a180e9ba4cc3de69 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/date.c | 8 ++++---- src/global.c | 4 ++-- src/vacuum.c | 2 +- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index b80e3b8250..ace0366e33 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\stypo\sin\ssqlite3.h\sreported\son\sthe\smailing\slist. -D 2014-11-07T11:39:16.680 +C Fix\sharmless\stypos\sin\scomments. +D 2014-11-07T13:24:29.246 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -179,13 +179,13 @@ F src/build.c 67bb05b1077e0cdaccb2e36bfcbe7a5df9ed31e8 F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 F src/complete.c c4ba6e0626bb94bc77a0861735f3382fcf7cc818 F src/ctime.c df19848891c8a553c80e6f5a035e768280952d1a -F src/date.c 57a7f9ba9f6b4d5268f5e411739066a611f99036 +F src/date.c 93594514aae68de117ca4a2a0d6cc63eddf26744 F src/delete.c 0750b1eb4d96cd3fb2c798599a3a7c85e92f1417 F src/expr.c 0391a657df4959eaf2a2fd7d77de5ebe750686ee F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c da985ae673efef2c712caef825a5d2edb087ead7 F src/func.c ba47c1671ab3cfdafa6e9d6ee490939ea578adee -F src/global.c a50ad0b9ee328107a65aa8f5f3cd34905e74745c +F src/global.c 6ded36dda9466fc1c9a3c5492ded81d79bf3977d F src/hash.c 4263fbc955f26c2e8cdc0cf214bc42435aa4e4f5 F src/hash.h c8f3c31722cf3277d03713909761e152a5b81094 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 @@ -288,7 +288,7 @@ F src/trigger.c 25571661fdeae8c7f975ff40ffec205520a3f92f F src/update.c 3c4ecc282accf12d39edb8d524cf089645e55a13 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 3b627daa45c7308c1e36e3dbaa3f9ce7e5c7fa73 -F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a +F src/vacuum.c 9b30ec729337dd012ed88d4c292922c8ef9cf00c F src/vdbe.c 3fd4ebd3e87b63175bfd2be747608bae1670b4df F src/vdbe.h 6fc69d9c5e146302c56e163cb4b31d1ee64a18c3 F src/vdbeInt.h 9bb69ff2447c34b6ccc58b34ec35b615f86ead78 @@ -1211,7 +1211,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 360c8ca11c3315c8e08c7c52ff5468e3f723e562 -R 367fba2464fa0866db1d5cae2a24beee +P 402703212a8488f8b571ce170b3b6c7374bd7daa +R 9f78be6a67e175be94522748bee570f6 U drh -Z 3885b799624859bdb0af87865d5681ed +Z 1fff294369c9fe8b2c9ed9357ff2fdd6 diff --git a/manifest.uuid b/manifest.uuid index 185fd7e51d..3f84a89f53 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -402703212a8488f8b571ce170b3b6c7374bd7daa \ No newline at end of file +94c564da4c2cf5dffe58fdf7a180e9ba4cc3de69 \ No newline at end of file diff --git a/src/date.c b/src/date.c index 11b04ea004..10d9006263 100644 --- a/src/date.c +++ b/src/date.c @@ -16,7 +16,7 @@ ** sqlite3RegisterDateTimeFunctions() found at the bottom of the file. ** All other code has file scope. ** -** SQLite processes all times and dates as Julian Day numbers. The +** SQLite processes all times and dates as julian day numbers. The ** dates and times are stored as the number of days since noon ** in Greenwich on November 24, 4714 B.C. according to the Gregorian ** calendar system. @@ -31,7 +31,7 @@ ** ** The Gregorian calendar system is used for all dates and times, ** even those that predate the Gregorian calendar. Historians usually -** use the Julian calendar for dates prior to 1582-10-15 and for some +** use the julian calendar for dates prior to 1582-10-15 and for some ** dates afterwards, depending on locale. Beware of this difference. ** ** The conversion algorithms are implemented based on descriptions @@ -304,7 +304,7 @@ static int setDateTimeToCurrent(sqlite3_context *context, DateTime *p){ } /* -** Attempt to parse the given string into a Julian Day Number. Return +** Attempt to parse the given string into a julian day number. Return ** the number of errors. ** ** The following are acceptable forms for the input string: @@ -875,7 +875,7 @@ static void dateFunc( ** %f ** fractional seconds SS.SSS ** %H hour 00-24 ** %j day of year 000-366 -** %J ** Julian day number +** %J ** julian day number ** %m month 01-12 ** %M minute 00-59 ** %s seconds since 1970-01-01 diff --git a/src/global.c b/src/global.c index b5323e8df6..4bc8edb3bc 100644 --- a/src/global.c +++ b/src/global.c @@ -237,8 +237,8 @@ const Token sqlite3IntTokens[] = { ** ** IMPORTANT: Changing the pending byte to any value other than ** 0x40000000 results in an incompatible database file format! -** Changing the pending byte during operating results in undefined -** and dileterious behavior. +** Changing the pending byte during operation will result in undefined +** and incorrect behavior. */ #ifndef SQLITE_OMIT_WSD int sqlite3PendingByte = 0x40000000; diff --git a/src/vacuum.c b/src/vacuum.c index 4d0c0976a1..9df8e08b22 100644 --- a/src/vacuum.c +++ b/src/vacuum.c @@ -94,7 +94,7 @@ static int execExecSql(sqlite3 *db, char **pzErrMsg, const char *zSql){ ** overwriting the database with the vacuumed content. ** ** Only 1x temporary space and only 1x writes would be required if -** the copy of step (3) were replace by deleting the original database +** the copy of step (3) were replaced by deleting the original database ** and renaming the transient database as the original. But that will ** not work if other processes are attached to the original database. ** And a power loss in between deleting the original and renaming the From 9a06d30bb5266c008ad3a2493fecf5f65c36d56a Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 7 Nov 2014 13:52:44 +0000 Subject: [PATCH 090/133] In the ".scanstats on" output from the shell, round the estRows value to the nearest integer, rather than rounding toward zero. FossilOrigin-Name: 5700508535c35ac6b158b527e1d47e529e8e28ab --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index ace0366e33..b3d774824c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\stypos\sin\scomments. -D 2014-11-07T13:24:29.246 +C In\sthe\s".scanstats\son"\soutput\sfrom\sthe\sshell,\sround\sthe\sestRows\svalue\sto\nthe\snearest\sinteger,\srather\sthan\srounding\stoward\szero. +D 2014-11-07T13:52:44.718 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -228,7 +228,7 @@ F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b -F src/shell.c 48fe276aada42a15722aee2584e6321345ed4609 +F src/shell.c 0500307f63e3675dc4ab21112c6086b4d146fc08 F src/sqlite.h.in f433227d7f619887a1064913fa66cefa3da4349a F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d @@ -1211,7 +1211,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 402703212a8488f8b571ce170b3b6c7374bd7daa -R 9f78be6a67e175be94522748bee570f6 +P 94c564da4c2cf5dffe58fdf7a180e9ba4cc3de69 +R bf25c89aa0a2170169c258f353b46c64 U drh -Z 1fff294369c9fe8b2c9ed9357ff2fdd6 +Z 9c39d53d1fb35cc72c3ed2e42a4f819b diff --git a/manifest.uuid b/manifest.uuid index 3f84a89f53..7b8b4f079c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -94c564da4c2cf5dffe58fdf7a180e9ba4cc3de69 \ No newline at end of file +5700508535c35ac6b158b527e1d47e529e8e28ab \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index 915b69263b..a33e65b1f8 100644 --- a/src/shell.c +++ b/src/shell.c @@ -1222,7 +1222,7 @@ static void display_scanstats( fprintf(pArg->out, "Loop %2d: %s\n", n, zExplain); rEstLoop *= rEst; fprintf(pArg->out, " nLoop=%-8lld nRow=%-8lld estRow=%-8lld estRow/Loop=%-8g\n", - nLoop, nVisit, (sqlite3_int64)rEstLoop, rEst + nLoop, nVisit, (sqlite3_int64)(rEstLoop+0.5), rEst ); } } From 97d3898335b39b07e8753d0d705dd4b396bd85a2 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 7 Nov 2014 14:37:32 +0000 Subject: [PATCH 091/133] Fix another harmless comment typo. FossilOrigin-Name: b45bc80bb16f07192d84fd14433bb724a84d4146 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/sqliteInt.h | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index b3d774824c..579589c94b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\s".scanstats\son"\soutput\sfrom\sthe\sshell,\sround\sthe\sestRows\svalue\sto\nthe\snearest\sinteger,\srather\sthan\srounding\stoward\szero. -D 2014-11-07T13:52:44.718 +C Fix\sanother\sharmless\scomment\stypo. +D 2014-11-07T14:37:32.349 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -232,7 +232,7 @@ F src/shell.c 0500307f63e3675dc4ab21112c6086b4d146fc08 F src/sqlite.h.in f433227d7f619887a1064913fa66cefa3da4349a F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d -F src/sqliteInt.h 8f67ca79e957b8ece7453b8e320b6a996e1b4761 +F src/sqliteInt.h 71b0bf1a7fc55b5cb374f7579fd140e730a6e0f4 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 81712116e826b0089bb221b018929536b2b5406f F src/table.c f142bba7903e93ca8d113a5b8877a108ad1a27dc @@ -1211,7 +1211,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 94c564da4c2cf5dffe58fdf7a180e9ba4cc3de69 -R bf25c89aa0a2170169c258f353b46c64 +P 5700508535c35ac6b158b527e1d47e529e8e28ab +R 95ce072bedb368f2e0f6da4bf10906cf U drh -Z 9c39d53d1fb35cc72c3ed2e42a4f819b +Z 8c052ffec320f3367f2daa07942512b6 diff --git a/manifest.uuid b/manifest.uuid index 7b8b4f079c..13afcc03f7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5700508535c35ac6b158b527e1d47e529e8e28ab \ No newline at end of file +b45bc80bb16f07192d84fd14433bb724a84d4146 \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index e9715efcf7..4d272b06a1 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -562,7 +562,7 @@ typedef INT8_TYPE i8; /* 1-byte signed integer */ ** gives a possible range of values of approximately 1.0e986 to 1e-986. ** But the allowed values are "grainy". Not every value is representable. ** For example, quantities 16 and 17 are both represented by a LogEst -** of 40. However, since LogEst quantaties are suppose to be estimates, +** of 40. However, since LogEst quantities are suppose to be estimates, ** not exact values, this imprecision is not a problem. ** ** "LogEst" is short for "Logarithmic Estimate". From b391b944319fad869ec86b4798a0e62daf04d48c Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 7 Nov 2014 14:41:11 +0000 Subject: [PATCH 092/133] Add new test file e_blobopen.test, containing tests for sqlite3_blob_open(). FossilOrigin-Name: ecbccd0e594d22b3ae7fabc8037951dc49570bc3 --- main.mk | 1 + manifest | 30 ++- manifest.uuid | 2 +- src/sqlite.h.in | 58 +++-- src/tclsqlite.c | 2 + src/test1.c | 152 ----------- src/test_blob.c | 319 +++++++++++++++++++++++ src/test_config.c | 6 + src/vdbe.c | 5 +- test/e_blobopen.test | 549 +++++++++++++++++++++++++++++++++++++++ test/fkey7.test | 17 ++ test/without_rowid5.test | 3 +- 12 files changed, 950 insertions(+), 194 deletions(-) create mode 100644 src/test_blob.c create mode 100644 test/e_blobopen.test diff --git a/main.mk b/main.mk index 4a7ac02710..c8bf72b48b 100644 --- a/main.mk +++ b/main.mk @@ -246,6 +246,7 @@ TESTSRC = \ $(TOP)/src/test_autoext.c \ $(TOP)/src/test_async.c \ $(TOP)/src/test_backup.c \ + $(TOP)/src/test_blob.c \ $(TOP)/src/test_btree.c \ $(TOP)/src/test_config.c \ $(TOP)/src/test_demovfs.c \ diff --git a/manifest b/manifest index 579589c94b..e08707974e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sanother\sharmless\scomment\stypo. -D 2014-11-07T14:37:32.349 +C Add\snew\stest\sfile\se_blobopen.test,\scontaining\stests\sfor\ssqlite3_blob_open(). +D 2014-11-07T14:41:11.404 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -151,7 +151,7 @@ F ext/userauth/userauth.c 5fa3bdb492f481bbc1709fc83c91ebd13460c69e F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk bbc8b6000ed143a1a8d31d3b4995c359a3188fa1 +F main.mk 3fececc835c5c23637c3e0af970e27a81a9ba476 F mkopcodec.awk c2ff431854d702cdd2d779c9c0d1f58fa16fa4ea F mkopcodeh.awk c6b3fa301db6ef7ac916b14c60868aeaec1337b5 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 @@ -229,15 +229,15 @@ F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c 0500307f63e3675dc4ab21112c6086b4d146fc08 -F src/sqlite.h.in f433227d7f619887a1064913fa66cefa3da4349a +F src/sqlite.h.in 5abfd7d6c9bd2054105820de5ce12b1e491b1dc9 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 71b0bf1a7fc55b5cb374f7579fd140e730a6e0f4 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 81712116e826b0089bb221b018929536b2b5406f F src/table.c f142bba7903e93ca8d113a5b8877a108ad1a27dc -F src/tclsqlite.c 7cdd4dd3c2a4183483feca260070d73d6e22cd47 -F src/test1.c ac7f3bad83ef4508d5efc85b32e86da48db8ed7e +F src/tclsqlite.c 0a874655dd39a9875e39c5d3c464db662171d228 +F src/test1.c cce6ad0effe8c66a62d2634d9714ffdc7372ef11 F src/test2.c 98049e51a17dc62606a99a9eb95ee477f9996712 F src/test3.c 1c0e5d6f080b8e33c1ce8b3078e7013fdbcd560c F src/test4.c 9b32d22f5f150abe23c1830e2057c4037c45b3df @@ -249,8 +249,9 @@ F src/test9.c bea1e8cf52aa93695487badedd6e1886c321ea60 F src/test_async.c 21e11293a2f72080eda70e1124e9102044531cd8 F src/test_autoext.c dea8a01a7153b9adc97bd26161e4226329546e12 F src/test_backup.c 3875e899222b651e18b662f86e0e50daa946344e +F src/test_blob.c 1f2e3e25255b731c4fcf15ee7990d06347cb6c09 F src/test_btree.c 2e9978eca99a9a4bfa8cae949efb00886860a64f -F src/test_config.c c8b8b50bf2fe5102de10e4c7100b746d7f6bf62f +F src/test_config.c 035c17a173937d019b8dfc1d524f9d3fc8123504 F src/test_demovfs.c 69b2085076654ebc18014cbc6386f04409c959a9 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc F src/test_fs.c ced436e3d4b8e4681328409b8081051ce614e28f @@ -289,7 +290,7 @@ F src/update.c 3c4ecc282accf12d39edb8d524cf089645e55a13 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 3b627daa45c7308c1e36e3dbaa3f9ce7e5c7fa73 F src/vacuum.c 9b30ec729337dd012ed88d4c292922c8ef9cf00c -F src/vdbe.c 3fd4ebd3e87b63175bfd2be747608bae1670b4df +F src/vdbe.c d5dab22208e36e5689e9fb553aea3613921054ec F src/vdbe.h 6fc69d9c5e146302c56e163cb4b31d1ee64a18c3 F src/vdbeInt.h 9bb69ff2447c34b6ccc58b34ec35b615f86ead78 F src/vdbeapi.c 07acb615d1e4170e71fc1b0d087f3c53a1ad8e83 @@ -451,6 +452,7 @@ F test/descidx3.test 09ddbe3f5295f482d2f8b687cf6db8bad7acd9a2 F test/diskfull.test 106391384780753ea6896b7b4f005d10e9866b6e F test/distinct.test 086e70c765f172e8974e9f83b9ac5ca03c154e77 F test/distinctagg.test 1a6ef9c87a58669438fc771450d7a72577417376 +F test/e_blobopen.test 234f960d90235a9b51ec3ca1e062e8541dd558d8 w test/e_blob.test F test/e_changes.test fd66105385153dbf21fdb35eb8ef6c3e1eade579 F test/e_createtable.test c7e67b49e6cf92473c8fb30ab26143e9e2128cf7 F test/e_delete.test d5186e2f5478b659f16a2c8b66c09892823e542a @@ -491,7 +493,7 @@ F test/fkey3.test 76d475c80b84ee7a5d062e56ccb6ea68882e2b49 F test/fkey4.test 86446017011273aad8f9a99c1a65019e7bd9ca9d F test/fkey5.test 8a1fde4e7721ae00b05b3178888833726ca2df8d F test/fkey6.test abb59f866c1b44926fd02d1fdd217d831fe04f48 -F test/fkey7.test e31d0e71a41c1d29349a16448d6c420e2c53a8fc +F test/fkey7.test 72e915890ee4a005daaf3002cb208e8fe973ac13 F test/fkey_malloc.test 594a7ea1fbab553c036c70813cd8bd9407d63749 F test/format4.test 1f0cac8ff3895e9359ed87e41aaabee982a812eb F test/fts-9fd058691.test 78b887e30ae6816df0e1fed6259de4b5a64ad33c @@ -1155,7 +1157,7 @@ F test/without_rowid1.test 7862e605753c8d25329f665fa09072e842183151 F test/without_rowid2.test af260339f79d13cb220288b67cd287fbcf81ad99 F test/without_rowid3.test 1081aabf60a1e1123b7f9a8f6ae19954351843b0 F test/without_rowid4.test 4e08bcbaee0399f35d58b5581881e7a6243d458a -F test/without_rowid5.test b4a639a367f04d382d20e8f44fc1be4f2d57d107 +F test/without_rowid5.test 61256715b686359df48ca1742db50cc7e3e7b862 F test/wordcount.c 9915e06cb33d8ca8109b8700791afe80d305afda F test/zeroblob.test caaecfb4f908f7bc086ed238668049f96774d688 F test/zerodamage.test cf6748bad89553cc1632be51a6f54e487e4039ac @@ -1211,7 +1213,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 5700508535c35ac6b158b527e1d47e529e8e28ab -R 95ce072bedb368f2e0f6da4bf10906cf -U drh -Z 8c052ffec320f3367f2daa07942512b6 +P b45bc80bb16f07192d84fd14433bb724a84d4146 +R 0ead7b89f3ed1286937ae097bba351c9 +U dan +Z d785e78de9d268050b7beac3067db32e diff --git a/manifest.uuid b/manifest.uuid index 13afcc03f7..6227627e36 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b45bc80bb16f07192d84fd14433bb724a84d4146 \ No newline at end of file +ecbccd0e594d22b3ae7fabc8037951dc49570bc3 \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index fc741b7291..bed64bfae2 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -5657,26 +5657,42 @@ typedef struct sqlite3_blob sqlite3_blob; ** SELECT zColumn FROM zDb.zTable WHERE [rowid] = iRow; ** )^ ** +** ^(Parameter zDb is not the filename that contains the database, but +** rather the symbolic name of the database. For attached databases, this is +** the name that appears after the AS keyword in the [ATTACH] statement. +** For the main database file, the database name is "main". For TEMP +** tables, the database name is "temp".)^ +** ** ^If the flags parameter is non-zero, then the BLOB is opened for read -** and write access. ^If it is zero, the BLOB is opened for read access. -** ^It is not possible to open a column that is part of an index or primary -** key for writing. ^If [foreign key constraints] are enabled, it is -** not possible to open a column that is part of a [child key] for writing. +** and write access. ^If the flags parameter is zero, the BLOB is opened for +** read-only access. ** -** ^Note that the database name is not the filename that contains -** the database but rather the symbolic name of the database that -** appears after the AS keyword when the database is connected using [ATTACH]. -** ^For the main database file, the database name is "main". -** ^For TEMP tables, the database name is "temp". +** ^(On success, [SQLITE_OK] is returned and the new [BLOB handle] is stored +** in *ppBlob. Otherwise an [error code] is returned and, unless the error +** code is SQLITE_MISUSE, *ppBlob is set to NULL.)^ ^This means that, provided +** the API is not misused, it is always safe to call [sqlite3_blob_close()] +** on *ppBlob after this function it returns. +** +** This function fails with SQLITE_ERROR if any of the following are true: +**
    +**
  • ^(Database zDb does not exist)^, +**
  • ^(Table zTable does not exist within database zDb)^, +**
  • ^(Table zTable is a WITHOUT ROWID table)^, +**
  • ^(Column zColumn does not exist)^, +**
  • ^(Row iRow is not present in the table)^, +**
  • ^(The specified column of row iRow contains a value that is not +** a TEXT or BLOB value)^, +**
  • ^(Column zColumn is part of an index, PRIMARY KEY or UNIQUE +** constraint and the blob is being opened for read/write access)^, +**
  • ^([foreign key constraints | Foreign key constraints] are enabled, +** column zColumn is part of a [child key] definition and the blob is +** being opened for read/write access)^. +**
+** +** ^Unless it returns SQLITE_MISUSE, this function sets the +** [database connection] error code and message accessible via +** [sqlite3_errcode()] and [sqlite3_errmsg()] and related functions. ** -** ^(On success, [SQLITE_OK] is returned and the new [BLOB handle] is written -** to *ppBlob. Otherwise an [error code] is returned and *ppBlob is set -** to be a null pointer.)^ -** ^This function sets the [database connection] error code and message -** accessible via [sqlite3_errcode()] and [sqlite3_errmsg()] and related -** functions. ^Note that the *ppBlob variable is always initialized in a -** way that makes it safe to invoke [sqlite3_blob_close()] on *ppBlob -** regardless of the success or failure of this routine. ** ** ^(If the row that a BLOB handle points to is modified by an ** [UPDATE], [DELETE], or by [ON CONFLICT] side-effects @@ -5694,13 +5710,9 @@ typedef struct sqlite3_blob sqlite3_blob; ** interface. Use the [UPDATE] SQL command to change the size of a ** blob. ** -** ^The [sqlite3_blob_open()] interface will fail for a [WITHOUT ROWID] -** table. Incremental BLOB I/O is not possible on [WITHOUT ROWID] tables. -** ** ^The [sqlite3_bind_zeroblob()] and [sqlite3_result_zeroblob()] interfaces -** and the built-in [zeroblob] SQL function can be used, if desired, -** to create an empty, zero-filled blob in which to read or write using -** this interface. +** and the built-in [zeroblob] SQL function may be used to create a +** zero-filled blob to read or write using the incremental-blob interface. ** ** To avoid a resource leak, every open [BLOB handle] should eventually ** be released by a call to [sqlite3_blob_close()]. diff --git a/src/tclsqlite.c b/src/tclsqlite.c index bff4a92421..4fb78a5d12 100644 --- a/src/tclsqlite.c +++ b/src/tclsqlite.c @@ -3725,6 +3725,7 @@ static void init_all(Tcl_Interp *interp){ extern int Sqlitetest9_Init(Tcl_Interp*); extern int Sqlitetestasync_Init(Tcl_Interp*); extern int Sqlitetest_autoext_Init(Tcl_Interp*); + extern int Sqlitetest_blob_Init(Tcl_Interp*); extern int Sqlitetest_demovfs_Init(Tcl_Interp *); extern int Sqlitetest_func_Init(Tcl_Interp*); extern int Sqlitetest_hexio_Init(Tcl_Interp*); @@ -3768,6 +3769,7 @@ static void init_all(Tcl_Interp *interp){ Sqlitetest9_Init(interp); Sqlitetestasync_Init(interp); Sqlitetest_autoext_Init(interp); + Sqlitetest_blob_Init(interp); Sqlitetest_demovfs_Init(interp); Sqlitetest_func_Init(interp); Sqlitetest_hexio_Init(interp); diff --git a/src/test1.c b/src/test1.c index fcc1a23de1..e7a5929323 100644 --- a/src/test1.c +++ b/src/test1.c @@ -1651,154 +1651,6 @@ static int blobHandleFromObj( return TCL_OK; } -/* -** sqlite3_blob_bytes CHANNEL -*/ -static int test_blob_bytes( - ClientData clientData, /* Not used */ - Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ - int objc, /* Number of arguments */ - Tcl_Obj *CONST objv[] /* Command arguments */ -){ - sqlite3_blob *pBlob; - int nByte; - - if( objc!=2 ){ - Tcl_WrongNumArgs(interp, 1, objv, "CHANNEL"); - return TCL_ERROR; - } - - if( blobHandleFromObj(interp, objv[1], &pBlob) ) return TCL_ERROR; - nByte = sqlite3_blob_bytes(pBlob); - Tcl_SetObjResult(interp, Tcl_NewIntObj(nByte)); - - return TCL_OK; -} - -/* -** sqlite3_blob_close CHANNEL -*/ -static int test_blob_close( - ClientData clientData, /* Not used */ - Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ - int objc, /* Number of arguments */ - Tcl_Obj *CONST objv[] /* Command arguments */ -){ - sqlite3_blob *pBlob; - - if( objc!=2 ){ - Tcl_WrongNumArgs(interp, 1, objv, "CHANNEL"); - return TCL_ERROR; - } - - if( blobHandleFromObj(interp, objv[1], &pBlob) ) return TCL_ERROR; - sqlite3_blob_close(pBlob); - - return TCL_OK; -} - -/* -** sqlite3_blob_read CHANNEL OFFSET N -** -** This command is used to test the sqlite3_blob_read() in ways that -** the Tcl channel interface does not. The first argument should -** be the name of a valid channel created by the [incrblob] method -** of a database handle. This function calls sqlite3_blob_read() -** to read N bytes from offset OFFSET from the underlying SQLite -** blob handle. -** -** On success, a byte-array object containing the read data is -** returned. On failure, the interpreter result is set to the -** text representation of the returned error code (i.e. "SQLITE_NOMEM") -** and a Tcl exception is thrown. -*/ -static int test_blob_read( - ClientData clientData, /* Not used */ - Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ - int objc, /* Number of arguments */ - Tcl_Obj *CONST objv[] /* Command arguments */ -){ - sqlite3_blob *pBlob; - int nByte; - int iOffset; - unsigned char *zBuf = 0; - int rc; - - if( objc!=4 ){ - Tcl_WrongNumArgs(interp, 1, objv, "CHANNEL OFFSET N"); - return TCL_ERROR; - } - - if( blobHandleFromObj(interp, objv[1], &pBlob) ) return TCL_ERROR; - if( TCL_OK!=Tcl_GetIntFromObj(interp, objv[2], &iOffset) - || TCL_OK!=Tcl_GetIntFromObj(interp, objv[3], &nByte) - ){ - return TCL_ERROR; - } - - if( nByte>0 ){ - zBuf = (unsigned char *)Tcl_Alloc(nByte); - } - rc = sqlite3_blob_read(pBlob, zBuf, nByte, iOffset); - if( rc==SQLITE_OK ){ - Tcl_SetObjResult(interp, Tcl_NewByteArrayObj(zBuf, nByte)); - }else{ - Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE); - } - Tcl_Free((char *)zBuf); - - return (rc==SQLITE_OK ? TCL_OK : TCL_ERROR); -} - -/* -** sqlite3_blob_write CHANNEL OFFSET DATA ?NDATA? -** -** This command is used to test the sqlite3_blob_write() in ways that -** the Tcl channel interface does not. The first argument should -** be the name of a valid channel created by the [incrblob] method -** of a database handle. This function calls sqlite3_blob_write() -** to write the DATA byte-array to the underlying SQLite blob handle. -** at offset OFFSET. -** -** On success, an empty string is returned. On failure, the interpreter -** result is set to the text representation of the returned error code -** (i.e. "SQLITE_NOMEM") and a Tcl exception is thrown. -*/ -static int test_blob_write( - ClientData clientData, /* Not used */ - Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ - int objc, /* Number of arguments */ - Tcl_Obj *CONST objv[] /* Command arguments */ -){ - sqlite3_blob *pBlob; - int iOffset; - int rc; - - unsigned char *zBuf; - int nBuf; - - if( objc!=4 && objc!=5 ){ - Tcl_WrongNumArgs(interp, 1, objv, "CHANNEL OFFSET DATA ?NDATA?"); - return TCL_ERROR; - } - - if( blobHandleFromObj(interp, objv[1], &pBlob) ) return TCL_ERROR; - if( TCL_OK!=Tcl_GetIntFromObj(interp, objv[2], &iOffset) ){ - return TCL_ERROR; - } - - zBuf = Tcl_GetByteArrayFromObj(objv[3], &nBuf); - if( objc==5 && Tcl_GetIntFromObj(interp, objv[4], &nBuf) ){ - return TCL_ERROR; - } - rc = sqlite3_blob_write(pBlob, zBuf, nBuf, iOffset); - if( rc!=SQLITE_OK ){ - Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE); - } - - return (rc==SQLITE_OK ? TCL_OK : TCL_ERROR); -} - static int test_blob_reopen( ClientData clientData, /* Not used */ Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ @@ -6910,11 +6762,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ { "sqlite3_table_column_metadata", test_table_column_metadata, 0 }, #endif #ifndef SQLITE_OMIT_INCRBLOB - { "sqlite3_blob_read", test_blob_read, 0 }, - { "sqlite3_blob_write", test_blob_write, 0 }, { "sqlite3_blob_reopen", test_blob_reopen, 0 }, - { "sqlite3_blob_bytes", test_blob_bytes, 0 }, - { "sqlite3_blob_close", test_blob_close, 0 }, #endif { "pcache_stats", test_pcache_stats, 0 }, #ifdef SQLITE_ENABLE_UNLOCK_NOTIFY diff --git a/src/test_blob.c b/src/test_blob.c new file mode 100644 index 0000000000..d88c91366a --- /dev/null +++ b/src/test_blob.c @@ -0,0 +1,319 @@ +/* +** 2014 October 30 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +*/ +#include "sqliteInt.h" +#include "tcl.h" +#include +#include +#include + +/* These functions are implemented in main.c. */ +extern const char *sqlite3ErrName(int); + +/* From test1.c: */ +extern int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite3 **ppDb); +extern void *sqlite3TestTextToPtr(const char *z); + +/* +** Return a pointer to a buffer containing a text representation of the +** pointer passed as the only argument. The original pointer may be extracted +** from the text using sqlite3TestTextToPtr(). +*/ +static char *ptrToText(void *p){ + static char buf[100]; + sqlite3_snprintf(sizeof(buf)-1, buf, "%p", p); + return buf; +} + +/* +** Attempt to extract a blob handle (type sqlite3_blob*) from the Tcl +** object passed as the second argument. If successful, set *ppBlob to +** point to the blob handle and return TCL_OK. Otherwise, store an error +** message in the tcl interpreter and return TCL_ERROR. The final value +** of *ppBlob is undefined in this case. +** +** If the object contains a string that begins with "incrblob_", then it +** is assumed to be the name of a Tcl channel opened using the [db incrblob] +** command (see tclsqlite.c). Otherwise, it is assumed to be a pointer +** encoded using the ptrToText() routine or similar. +*/ +static int blobHandleFromObj( + Tcl_Interp *interp, + Tcl_Obj *pObj, + sqlite3_blob **ppBlob +){ + char *z; + int n; + + z = Tcl_GetStringFromObj(pObj, &n); + if( n==0 ){ + *ppBlob = 0; + }else if( n>9 && 0==memcmp("incrblob_", z, 9) ){ + int notUsed; + Tcl_Channel channel; + ClientData instanceData; + + channel = Tcl_GetChannel(interp, z, ¬Used); + if( !channel ) return TCL_ERROR; + + Tcl_Flush(channel); + Tcl_Seek(channel, 0, SEEK_SET); + + instanceData = Tcl_GetChannelInstanceData(channel); + *ppBlob = *((sqlite3_blob **)instanceData); + }else{ + *ppBlob = (sqlite3_blob*)sqlite3TestTextToPtr(z); + } + + return TCL_OK; +} + +/* +** Like Tcl_GetString(), except that if the string is 0 bytes in size, a +** NULL Pointer is returned. +*/ +static char *blobStringFromObj(Tcl_Obj *pObj){ + int n; + char *z; + z = Tcl_GetStringFromObj(pObj, &n); + return (n ? z : 0); +} + +/* +** sqlite3_blob_open DB DATABASE TABLE COLUMN ROWID FLAGS VARNAME +** +** Tcl test harness for the sqlite3_blob_open() function. +*/ +static int test_blob_open( + ClientData clientData, /* Not used */ + Tcl_Interp *interp, /* Calling TCL interpreter */ + int objc, /* Number of arguments */ + Tcl_Obj *CONST objv[] /* Command arguments */ +){ + sqlite3 *db; + const char *zDb; + const char *zTable; + const char *zColumn; + sqlite_int64 iRowid; + int flags; + const char *zVarname; + int nVarname; + + sqlite3_blob *pBlob = (sqlite3_blob*)0xFFFFFFFF; + int rc; + + if( objc!=8 ){ + const char *zUsage = "DB DATABASE TABLE COLUMN ROWID FLAGS VARNAME"; + Tcl_WrongNumArgs(interp, 1, objv, zUsage); + return TCL_ERROR; + } + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; + zDb = Tcl_GetString(objv[2]); + zTable = blobStringFromObj(objv[3]); + zColumn = Tcl_GetString(objv[4]); + if( Tcl_GetWideIntFromObj(interp, objv[5], &iRowid) ) return TCL_ERROR; + if( Tcl_GetIntFromObj(interp, objv[6], &flags) ) return TCL_ERROR; + zVarname = Tcl_GetStringFromObj(objv[7], &nVarname); + + if( nVarname>0 ){ + rc = sqlite3_blob_open(db, zDb, zTable, zColumn, iRowid, flags, &pBlob); + Tcl_SetVar(interp, zVarname, ptrToText(pBlob), 0); + }else{ + rc = sqlite3_blob_open(db, zDb, zTable, zColumn, iRowid, flags, 0); + } + + if( rc==SQLITE_OK ){ + Tcl_ResetResult(interp); + }else{ + Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE); + return TCL_ERROR; + } + return TCL_OK; +} + + +/* +** sqlite3_blob_close HANDLE +*/ +static int test_blob_close( + ClientData clientData, /* Not used */ + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int objc, /* Number of arguments */ + Tcl_Obj *CONST objv[] /* Command arguments */ +){ + sqlite3_blob *pBlob; + int rc; + + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "HANDLE"); + return TCL_ERROR; + } + + if( blobHandleFromObj(interp, objv[1], &pBlob) ) return TCL_ERROR; + rc = sqlite3_blob_close(pBlob); + + if( rc ){ + Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE); + }else{ + Tcl_ResetResult(interp); + } + return TCL_OK; +} + +/* +** sqlite3_blob_bytes HANDLE +*/ +static int test_blob_bytes( + ClientData clientData, /* Not used */ + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int objc, /* Number of arguments */ + Tcl_Obj *CONST objv[] /* Command arguments */ +){ + sqlite3_blob *pBlob; + int nByte; + + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "HANDLE"); + return TCL_ERROR; + } + + if( blobHandleFromObj(interp, objv[1], &pBlob) ) return TCL_ERROR; + nByte = sqlite3_blob_bytes(pBlob); + Tcl_SetObjResult(interp, Tcl_NewIntObj(nByte)); + + return TCL_OK; +} + +/* +** sqlite3_blob_read CHANNEL OFFSET N +** +** This command is used to test the sqlite3_blob_read() in ways that +** the Tcl channel interface does not. The first argument should +** be the name of a valid channel created by the [incrblob] method +** of a database handle. This function calls sqlite3_blob_read() +** to read N bytes from offset OFFSET from the underlying SQLite +** blob handle. +** +** On success, a byte-array object containing the read data is +** returned. On failure, the interpreter result is set to the +** text representation of the returned error code (i.e. "SQLITE_NOMEM") +** and a Tcl exception is thrown. +*/ +static int test_blob_read( + ClientData clientData, /* Not used */ + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int objc, /* Number of arguments */ + Tcl_Obj *CONST objv[] /* Command arguments */ +){ + sqlite3_blob *pBlob; + int nByte; + int iOffset; + unsigned char *zBuf = 0; + int rc; + + if( objc!=4 ){ + Tcl_WrongNumArgs(interp, 1, objv, "CHANNEL OFFSET N"); + return TCL_ERROR; + } + + if( blobHandleFromObj(interp, objv[1], &pBlob) ) return TCL_ERROR; + if( TCL_OK!=Tcl_GetIntFromObj(interp, objv[2], &iOffset) + || TCL_OK!=Tcl_GetIntFromObj(interp, objv[3], &nByte) + ){ + return TCL_ERROR; + } + + if( nByte>0 ){ + zBuf = (unsigned char *)Tcl_Alloc(nByte); + } + rc = sqlite3_blob_read(pBlob, zBuf, nByte, iOffset); + if( rc==SQLITE_OK ){ + Tcl_SetObjResult(interp, Tcl_NewByteArrayObj(zBuf, nByte)); + }else{ + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE); + } + Tcl_Free((char *)zBuf); + + return (rc==SQLITE_OK ? TCL_OK : TCL_ERROR); +} + +/* +** sqlite3_blob_write HANDLE OFFSET DATA ?NDATA? +** +** This command is used to test the sqlite3_blob_write() in ways that +** the Tcl channel interface does not. The first argument should +** be the name of a valid channel created by the [incrblob] method +** of a database handle. This function calls sqlite3_blob_write() +** to write the DATA byte-array to the underlying SQLite blob handle. +** at offset OFFSET. +** +** On success, an empty string is returned. On failure, the interpreter +** result is set to the text representation of the returned error code +** (i.e. "SQLITE_NOMEM") and a Tcl exception is thrown. +*/ +static int test_blob_write( + ClientData clientData, /* Not used */ + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int objc, /* Number of arguments */ + Tcl_Obj *CONST objv[] /* Command arguments */ +){ + sqlite3_blob *pBlob; + int iOffset; + int rc; + + unsigned char *zBuf; + int nBuf; + + if( objc!=4 && objc!=5 ){ + Tcl_WrongNumArgs(interp, 1, objv, "HANDLE OFFSET DATA ?NDATA?"); + return TCL_ERROR; + } + + if( blobHandleFromObj(interp, objv[1], &pBlob) ) return TCL_ERROR; + if( TCL_OK!=Tcl_GetIntFromObj(interp, objv[2], &iOffset) ){ + return TCL_ERROR; + } + + zBuf = Tcl_GetByteArrayFromObj(objv[3], &nBuf); + if( objc==5 && Tcl_GetIntFromObj(interp, objv[4], &nBuf) ){ + return TCL_ERROR; + } + rc = sqlite3_blob_write(pBlob, zBuf, nBuf, iOffset); + if( rc!=SQLITE_OK ){ + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE); + } + + return (rc==SQLITE_OK ? TCL_OK : TCL_ERROR); +} + + +/* +** Register commands with the TCL interpreter. +*/ +int Sqlitetest_blob_Init(Tcl_Interp *interp){ + static struct { + char *zName; + Tcl_ObjCmdProc *xProc; + } aObjCmd[] = { + { "sqlite3_blob_open", test_blob_open }, + { "sqlite3_blob_close", test_blob_close }, + { "sqlite3_blob_bytes", test_blob_bytes }, + { "sqlite3_blob_read", test_blob_read }, + { "sqlite3_blob_write", test_blob_write }, + }; + int i; + for(i=0; ipKeyInfo, aTempRec, sizeof(aTempRec), &pFree - ); + ); if( pIdxKey==0 ) goto no_mem; assert( pIn3->flags & MEM_Blob ); - assert( (pIn3->flags & MEM_Zero)==0 ); /* zeroblobs already expanded */ + /* assert( (pIn3->flags & MEM_Zero)==0 ); // zeroblobs already expanded */ + ExpandBlob(pIn3); sqlite3VdbeRecordUnpack(pC->pKeyInfo, pIn3->n, pIn3->z, pIdxKey); } pIdxKey->default_rc = 0; diff --git a/test/e_blobopen.test b/test/e_blobopen.test new file mode 100644 index 0000000000..01f62cdd78 --- /dev/null +++ b/test/e_blobopen.test @@ -0,0 +1,549 @@ +# 2014 October 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix e_blobopen + +forcedelete test.db2 + +do_execsql_test 1.0 { + ATTACH 'test.db2' AS aux; + + CREATE TABLE main.t1(a INTEGER PRIMARY KEY, b TEXT, c BLOB); + CREATE TEMP TABLE t1(a INTEGER PRIMARY KEY, b TEXT, c BLOB); + CREATE TABLE aux.t1(a INTEGER PRIMARY KEY, b TEXT, c BLOB); + + CREATE TABLE main.x1(a INTEGER PRIMARY KEY, b TEXT, c BLOB); + CREATE TEMP TABLE x2(a INTEGER PRIMARY KEY, b TEXT, c BLOB); + CREATE TABLE aux.x3(a INTEGER PRIMARY KEY, b TEXT, c BLOB); + + INSERT INTO main.t1 VALUES(1, 'main one', X'0101'); + INSERT INTO main.t1 VALUES(2, 'main two', X'0102'); + INSERT INTO main.t1 VALUES(3, 'main three', X'0103'); + INSERT INTO main.t1 VALUES(4, 'main four', X'0104'); + INSERT INTO main.t1 VALUES(5, 'main five', X'0105'); + + INSERT INTO main.x1 VALUES(1, 'x main one', X'000101'); + INSERT INTO main.x1 VALUES(2, 'x main two', X'000102'); + INSERT INTO main.x1 VALUES(3, 'x main three', X'000103'); + INSERT INTO main.x1 VALUES(4, 'x main four', X'000104'); + INSERT INTO main.x1 VALUES(5, 'x main five', X'000105'); + + INSERT INTO temp.t1 VALUES(1, 'temp one', X'0201'); + INSERT INTO temp.t1 VALUES(2, 'temp two', X'0202'); + INSERT INTO temp.t1 VALUES(3, 'temp three', X'0203'); + INSERT INTO temp.t1 VALUES(4, 'temp four', X'0204'); + INSERT INTO temp.t1 VALUES(5, 'temp five', X'0205'); + + INSERT INTO temp.x2 VALUES(1, 'x temp one', X'000201'); + INSERT INTO temp.x2 VALUES(2, 'x temp two', X'000202'); + INSERT INTO temp.x2 VALUES(3, 'x temp three', X'000203'); + INSERT INTO temp.x2 VALUES(4, 'x temp four', X'000204'); + INSERT INTO temp.x2 VALUES(5, 'x temp five', X'000205'); + + INSERT INTO aux.t1 VALUES(1, 'aux one', X'0301'); + INSERT INTO aux.t1 VALUES(2, 'aux two', X'0302'); + INSERT INTO aux.t1 VALUES(3, 'aux three', X'0303'); + INSERT INTO aux.t1 VALUES(4, 'aux four', X'0304'); + INSERT INTO aux.t1 VALUES(5, 'aux five', X'0305'); + + INSERT INTO aux.x3 VALUES(1, 'x aux one', X'000301'); + INSERT INTO aux.x3 VALUES(2, 'x aux two', X'000302'); + INSERT INTO aux.x3 VALUES(3, 'x aux three', X'000303'); + INSERT INTO aux.x3 VALUES(4, 'x aux four', X'000304'); + INSERT INTO aux.x3 VALUES(5, 'x aux five', X'000305'); +} + +#------------------------------------------------------------------------- +# EVIDENCE-OF: R-37639-55938 This interfaces opens a handle to the BLOB +# located in row iRow, column zColumn, table zTable in database zDb; in +# other words, the same BLOB that would be selected by: SELECT zColumn +# FROM zDb.zTable WHERE rowid = iRow; +# +proc read_blob {zDb zTab zCol iRow} { + sqlite3_blob_open db $zDb $zTab $zCol $iRow 0 B + set nByte [sqlite3_blob_bytes $B] + set data [sqlite3_blob_read $B 0 $nByte] + sqlite3_blob_close $B + return $data +} + +do_test 1.1.1 { read_blob main t1 b 1 } "main one" +do_test 1.1.2 { read_blob main t1 c 1 } "\01\01" +do_test 1.1.3 { read_blob temp t1 b 1 } "temp one" +do_test 1.1.4 { read_blob temp t1 c 1 } "\02\01" +do_test 1.1.6 { read_blob aux t1 b 1 } "aux one" +do_test 1.1.7 { read_blob aux t1 c 1 } "\03\01" + +do_test 1.2.1 { read_blob main t1 b 4 } "main four" +do_test 1.2.2 { read_blob main t1 c 4 } "\01\04" +do_test 1.2.3 { read_blob temp t1 b 4 } "temp four" +do_test 1.2.4 { read_blob temp t1 c 4 } "\02\04" +do_test 1.2.6 { read_blob aux t1 b 4 } "aux four" +do_test 1.2.7 { read_blob aux t1 c 4 } "\03\04" + +do_test 1.3.1 { read_blob main x1 b 2 } "x main two" +do_test 1.3.2 { read_blob main x1 c 2 } "\00\01\02" +do_test 1.3.3 { read_blob temp x2 b 2 } "x temp two" +do_test 1.3.4 { read_blob temp x2 c 2 } "\00\02\02" +do_test 1.3.6 { read_blob aux x3 b 2 } "x aux two" +do_test 1.3.7 { read_blob aux x3 c 2 } "\00\03\02" + +#------------------------------------------------------------------------- +# EVIDENCE-OF: R-27234-05761 Parameter zDb is not the filename that +# contains the database, but rather the symbolic name of the database. +# For attached databases, this is the name that appears after the AS +# keyword in the ATTACH statement. For the main database file, the +# database name is "main". For TEMP tables, the database name is "temp". +# +# The test cases immediately above demonstrate that the database name +# for the main db, for TEMP tables and for those in attached databases +# is correct. The following tests check that filenames cannot be +# used as well. +# +do_test 2.1 { + list [catch { sqlite3_blob_open db "test.db" t1 b 1 0 B } msg] $msg +} {1 SQLITE_ERROR} +do_test 2.2 { + list [catch { sqlite3_blob_open db "test.db2" t1 b 1 0 B } msg] $msg +} {1 SQLITE_ERROR} + +#------------------------------------------------------------------------- +# EVIDENCE-OF: R-50854-53979 If the flags parameter is non-zero, then +# the BLOB is opened for read and write access. +# +# EVIDENCE-OF: R-03922-41160 If the flags parameter is zero, the BLOB is +# opened for read-only access. +# +foreach {tn iRow flags} { + 1 1 0 + 2 2 1 + 3 3 -1 + 4 4 2147483647 + 5 5 -2147483648 +} { + do_test 3.$tn.1 { + sqlite3_blob_open db main x1 c $iRow $flags B + set n [sqlite3_blob_bytes $B] + sqlite3_blob_read $B 0 $n + } [binary format ccc 0 1 $iRow] + + if {$flags==0} { + # Blob was opened for read-only access - writing returns an error. + do_test 3.$tn.2 { + list [catch { sqlite3_blob_write $B 0 xxx 3 } msg] $msg + } {1 SQLITE_READONLY} + + do_execsql_test 3.$tn.3 { + SELECT c FROM x1 WHERE a=$iRow; + } [binary format ccc 0 1 $iRow] + } else { + # Blob was opened for read/write access - writing succeeds + do_test 3.$tn.4 { + list [catch { sqlite3_blob_write $B 0 xxx 3 } msg] $msg + } {0 {}} + + do_execsql_test 3.$tn.5 { + SELECT c FROM x1 WHERE a=$iRow; + } {xxx} + } + + sqlite3_blob_close $B +} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 4.0 { + CREATE TABLE t1(x, y); + INSERT INTO t1 VALUES('abcd', 152); + INSERT INTO t1 VALUES(NULL, X'00010203'); + INSERT INTO t1 VALUES('', 154.2); + + CREATE TABLE t2(x PRIMARY KEY, y) WITHOUT ROWID; + INSERT INTO t2 VALUES(1, 'blob'); + + CREATE TABLE t3(a PRIMARY KEY, b, c, d, e, f, UNIQUE(e, f)); + INSERT INTO t3 VALUES('aaaa', 'bbbb', 'cccc', 'dddd', 'eeee', 'ffff'); + CREATE INDEX t3b ON t3(b); + + CREATE TABLE p1(x PRIMARY KEY); + INSERT INTO p1 VALUES('abc'); + + CREATE TABLE c1(a INTEGER PRIMARY KEY, b REFERENCES p1); + INSERT INTO c1 VALUES(45, 'abc'); +} + +proc test_blob_open {tn zDb zTab zCol iRow flags errcode errmsg} { + global B + set B "0x1234" + + if {$errcode=="SQLITE_OK"} { + set expected "0 {}" + } else { + set expected "1 $errcode" + } + + set ::res [list [ + catch { sqlite3_blob_open db $zDb $zTab $zCol $iRow $flags B } msg + ] $msg] + do_test 4.$tn.1 { set ::res } $expected + + # EVIDENCE-OF: R-08940-21305 Unless it returns SQLITE_MISUSE, this + # function sets the database connection error code and message + # accessible via sqlite3_errcode() and sqlite3_errmsg() and related + # functions. + # + # This proc (test_blob_open) is used below to test various error and + # non-error conditions. But never SQLITE_MISUSE conditions. So these + # test cases are considered as partly verifying the requirement above. + # See below for a test of the SQLITE_MISUSE case. + # + do_test 4.$tn.2 { + sqlite3_errcode db + } $errcode + do_test 4.$tn.3 { + sqlite3_errmsg db + } $errmsg + + # EVIDENCE-OF: R-31086-35521 On success, SQLITE_OK is returned and the + # new BLOB handle is stored in *ppBlob. Otherwise an error code is + # returned and, unless the error code is SQLITE_MISUSE, *ppBlob is set + # to NULL. + # + do_test 4.$tn.4 { + expr {$B == "0"} + } [expr {$errcode != "SQLITE_OK"}] + + # EVIDENCE-OF: R-63421-15521 This means that, provided the API is not + # misused, it is always safe to call sqlite3_blob_close() on *ppBlob + # after this function it returns. + do_test 4.$tn.5 { + sqlite3_blob_close $B + } {} +} + +# EVIDENCE-OF: R-31204-44780 Database zDb does not exist +test_blob_open 1 nosuchdb t1 x 1 0 SQLITE_ERROR "no such table: nosuchdb.t1" + +# EVIDENCE-OF: R-28676-08005 Table zTable does not exist within database zDb +test_blob_open 2 main tt1 x 1 0 SQLITE_ERROR "no such table: main.tt1" + +# EVIDENCE-OF: R-40134-30296 Table zTable is a WITHOUT ROWID table +test_blob_open 3 main t2 y 1 0 SQLITE_ERROR \ + "cannot open table without rowid: t2" + +# EVIDENCE-OF: R-56376-21261 Column zColumn does not exist +test_blob_open 4 main t1 z 2 0 SQLITE_ERROR "no such column: \"z\"" + +# EVIDENCE-OF: R-28258-23166 Row iRow is not present in the table +test_blob_open 5 main t1 y 6 0 SQLITE_ERROR "no such rowid: 6" + +# EVIDENCE-OF: R-11683-62380 The specified column of row iRow contains a +# value that is not a TEXT or BLOB value +test_blob_open 6 main t1 x 2 0 SQLITE_ERROR "cannot open value of type null" +test_blob_open 7 main t1 y 1 0 SQLITE_ERROR "cannot open value of type integer" +test_blob_open 8 main t1 y 3 0 SQLITE_ERROR "cannot open value of type real" + +# EVIDENCE-OF: R-34146-30782 Column zColumn is part of an index, PRIMARY +# KEY or UNIQUE constraint and the blob is being opened for read/write +# access +# +# Test cases 8.1.* show that such columns can be opened for read-access. +# Tests 8.2.* show that read-write access is different. Columns "c" and "c" +# are not part of an index, PK or UNIQUE constraint, so they work in both +# cases. +# +test_blob_open 8.1.1 main t3 a 1 0 SQLITE_OK "not an error" +test_blob_open 8.1.2 main t3 b 1 0 SQLITE_OK "not an error" +test_blob_open 8.1.3 main t3 c 1 0 SQLITE_OK "not an error" +test_blob_open 8.1.4 main t3 d 1 0 SQLITE_OK "not an error" +test_blob_open 8.1.5 main t3 e 1 0 SQLITE_OK "not an error" +test_blob_open 8.1.6 main t3 f 1 0 SQLITE_OK "not an error" + +set cannot "cannot open indexed column for writing" +test_blob_open 8.2.1 main t3 a 1 8 SQLITE_ERROR $cannot +test_blob_open 8.2.2 main t3 b 1 8 SQLITE_ERROR $cannot +test_blob_open 8.2.3 main t3 c 1 8 SQLITE_OK "not an error" +test_blob_open 8.2.4 main t3 d 1 8 SQLITE_OK "not an error" +test_blob_open 8.2.5 main t3 e 1 8 SQLITE_ERROR $cannot +test_blob_open 8.2.6 main t3 f 1 8 SQLITE_ERROR $cannot + +# EVIDENCE-OF: R-50117-55204 Foreign key constraints are enabled, column +# zColumn is part of a child key definition and the blob is being opened +# for read/write access +# +# 9.1: FK disabled, read-only access. +# 9.2: FK disabled, read-only access. +# 9.3: FK enabled, read/write access. +# 9.4: FK enabled, read/write access. +# +test_blob_open 9.1 main c1 b 45 0 SQLITE_OK "not an error" +test_blob_open 9.2 main c1 b 45 1 SQLITE_OK "not an error" +execsql { PRAGMA foreign_keys = ON } +test_blob_open 9.3 main c1 b 45 0 SQLITE_OK "not an error" +test_blob_open 9.4 main c1 b 45 1 SQLITE_ERROR \ + "cannot open foreign key column for writing" + +#------------------------------------------------------------------------- +# EVIDENCE-OF: R-08940-21305 Unless it returns SQLITE_MISUSE, this +# function sets the database connection error code and message +# accessible via sqlite3_errcode() and sqlite3_errmsg() and related +# functions. +# +# This requirement is partially verified by the many uses of test +# command [test_blob_open] above. All that is left is to verify the +# SQLITE_MISUSE case. +# +# SQLITE_MISUSE is only returned if SQLITE_ENABLE_API_ARMOR is defined +# during compilation. +# +ifcapable api_armor { + sqlite3_blob_open db main t1 x 1 0 B + + do_test 10.1.1 { + list [catch {sqlite3_blob_open $B main t1 x 1 0 B2} msg] $msg + } {1 SQLITE_MISUSE} + do_test 10.1.2 { + list [sqlite3_errcode db] [sqlite3_errmsg db] + } {SQLITE_OK {not an error}} + sqlite3_blob_close $B + + do_test 10.2.1 { + list [catch {sqlite3_blob_open db main {} x 1 0 B} msg] $msg + } {1 SQLITE_MISUSE} + do_test 10.2.2 { + list [sqlite3_errcode db] [sqlite3_errmsg db] + } {SQLITE_OK {not an error}} +} + +#------------------------------------------------------------------------- +# EVIDENCE-OF: R-50542-62589 If the row that a BLOB handle points to is +# modified by an UPDATE, DELETE, or by ON CONFLICT side-effects then the +# BLOB handle is marked as "expired". This is true if any column of the +# row is changed, even a column other than the one the BLOB handle is +# open on. +# +# EVIDENCE-OF: R-48367-20048 Calls to sqlite3_blob_read() and +# sqlite3_blob_write() for an expired BLOB handle fail with a return +# code of SQLITE_ABORT. +# +# 11.2: read-only handle, DELETE. +# 11.3: read-only handle, UPDATE. +# 11.4: read-only handle, REPLACE. +# 11.5: read/write handle, DELETE. +# 11.6: read/write handle, UPDATE. +# 11.7: read/write handle, REPLACE. +# +do_execsql_test 11.1 { + CREATE TABLE b1(a INTEGER PRIMARY KEY, b, c UNIQUE); + INSERT INTO b1 VALUES(1, '1234567890', 1); + INSERT INTO b1 VALUES(2, '1234567890', 2); + INSERT INTO b1 VALUES(3, '1234567890', 3); + INSERT INTO b1 VALUES(4, '1234567890', 4); + INSERT INTO b1 VALUES(5, '1234567890', 5); + INSERT INTO b1 VALUES(6, '1234567890', 6); + + CREATE TABLE b2(a INTEGER PRIMARY KEY, b, c UNIQUE); + INSERT INTO b2 VALUES(1, '1234567890', 1); + INSERT INTO b2 VALUES(2, '1234567890', 2); + INSERT INTO b2 VALUES(3, '1234567890', 3); + INSERT INTO b2 VALUES(4, '1234567890', 4); + INSERT INTO b2 VALUES(5, '1234567890', 5); + INSERT INTO b2 VALUES(6, '1234567890', 6); +} + +do_test 11.2.1 { + sqlite3_blob_open db main b1 b 2 0 B + sqlite3_blob_read $B 0 10 +} {1234567890} +do_test 11.2.2 { + # Deleting a different row does not invalidate the blob handle. + execsql { DELETE FROM b1 WHERE a = 1 } + sqlite3_blob_read $B 0 10 +} {1234567890} +do_test 11.2.3 { + execsql { DELETE FROM b1 WHERE a = 2 } + list [catch { sqlite3_blob_read $B 0 10 } msg] $msg +} {1 SQLITE_ABORT} +do_test 11.2.4 { + sqlite3_blob_close $B +} {} + +do_test 11.3.1 { + sqlite3_blob_open db main b1 b 3 0 B + sqlite3_blob_read $B 0 10 +} {1234567890} +do_test 11.3.2 { + # Updating a different row + execsql { UPDATE b1 SET c = 42 WHERE a=4 } + sqlite3_blob_read $B 0 10 +} {1234567890} +do_test 11.3.3 { + execsql { UPDATE b1 SET c = 43 WHERE a=3 } + list [catch { sqlite3_blob_read $B 0 10 } msg] $msg +} {1 SQLITE_ABORT} +do_test 11.3.4 { + sqlite3_blob_close $B +} {} + +do_test 11.4.1 { + sqlite3_blob_open db main b1 b 6 0 B + sqlite3_blob_read $B 0 10 +} {1234567890} +do_test 11.4.2 { + # Replace a different row + execsql { INSERT OR REPLACE INTO b1 VALUES(10, 'abcdefghij', 5) } + sqlite3_blob_read $B 0 10 +} {1234567890} +do_test 11.4.3 { + execsql { INSERT OR REPLACE INTO b1 VALUES(11, 'abcdefghij', 6) } + list [catch { sqlite3_blob_read $B 0 10 } msg] $msg +} {1 SQLITE_ABORT} +do_test 11.4.4 { + sqlite3_blob_close $B +} {} + +do_test 11.4.1 { + sqlite3_blob_open db main b2 b 2 1 B + sqlite3_blob_write $B 0 "abcdefghij" +} {} +do_test 11.4.2 { + # Deleting a different row does not invalidate the blob handle. + execsql { DELETE FROM b2 WHERE a = 1 } + sqlite3_blob_write $B 0 "ABCDEFGHIJ" +} {} +do_test 11.4.3 { + execsql { DELETE FROM b2 WHERE a = 2 } + list [catch { sqlite3_blob_write $B 0 "0987654321" } msg] $msg +} {1 SQLITE_ABORT} +do_test 11.4.4 { + sqlite3_blob_close $B +} {} + +do_test 11.5.1 { + sqlite3_blob_open db main b2 b 3 1 B + sqlite3_blob_write $B 0 "abcdefghij" +} {} +do_test 11.5.2 { + # Updating a different row + execsql { UPDATE b2 SET c = 42 WHERE a=4 } + sqlite3_blob_write $B 0 "ABCDEFGHIJ" +} {} +do_test 11.5.3 { + execsql { UPDATE b2 SET c = 43 WHERE a=3 } + list [catch { sqlite3_blob_write $B 0 "0987654321" } msg] $msg +} {1 SQLITE_ABORT} +do_test 11.5.4 { + sqlite3_blob_close $B +} {} + +do_test 11.6.1 { + sqlite3_blob_open db main b2 b 6 1 B + sqlite3_blob_write $B 0 "abcdefghij" +} {} +do_test 11.6.2 { + # Replace a different row + execsql { INSERT OR REPLACE INTO b2 VALUES(10, 'abcdefghij', 5) } + sqlite3_blob_write $B 0 "ABCDEFGHIJ" +} {} +do_test 11.6.3 { + execsql { INSERT OR REPLACE INTO b2 VALUES(11, 'abcdefghij', 6) } + list [catch { sqlite3_blob_write $B 0 "0987654321" } msg] $msg +} {1 SQLITE_ABORT} +do_test 11.6.4 { + sqlite3_blob_close $B +} {} + +#------------------------------------------------------------------------- +# EVIDENCE-OF: R-45408-40694 Changes written into a BLOB prior to the +# BLOB expiring are not rolled back by the expiration of the BLOB. Such +# changes will eventually commit if the transaction continues to +# completion. +# +do_execsql_test 12.1 { + CREATE TABLE b3(x INTEGER PRIMARY KEY, y TEXT, z INTEGER); + INSERT INTO b3 VALUES(22, '..........', NULL); +} +do_test 12.2 { + sqlite3_blob_open db main b3 y 22 1 B + sqlite3_blob_write $B 0 "xxxxx" 5 +} {} +do_execsql_test 12.3 { + UPDATE b3 SET z = 'not null'; +} +do_test 12.4 { + list [catch {sqlite3_blob_write $B 5 "xxxxx" 5} msg] $msg +} {1 SQLITE_ABORT} +do_execsql_test 12.5 { + SELECT * FROM b3; +} {22 xxxxx..... {not null}} +do_test 12.5 { + sqlite3_blob_close $B +} {} +do_execsql_test 12.6 { + SELECT * FROM b3; +} {22 xxxxx..... {not null}} + +#------------------------------------------------------------------------- +# EVIDENCE-OF: R-58813-55036 The sqlite3_bind_zeroblob() and +# sqlite3_result_zeroblob() interfaces and the built-in zeroblob SQL +# function may be used to create a zero-filled blob to read or write +# using the incremental-blob interface. +# +do_execsql_test 13.1 { + CREATE TABLE c2(i INTEGER PRIMARY KEY, j); + INSERT INTO c2 VALUES(10, zeroblob(24)); +} + +do_test 13.2 { + set stmt [sqlite3_prepare_v2 db "INSERT INTO c2 VALUES(11, ?)" -1] + sqlite3_bind_zeroblob $stmt 1 45 + sqlite3_step $stmt + sqlite3_finalize $stmt +} {SQLITE_OK} + +# The blobs can be read: +# +do_test 13.3.1 { + sqlite3_blob_open db main c2 j 10 1 B + sqlite3_blob_open db main c2 j 11 1 B2 + list [sqlite3_blob_bytes $B] [sqlite3_blob_bytes $B2] +} {24 45} +do_test 13.3.2 { + sqlite3_blob_read $B 0 24 +} [string repeat [binary format c 0] 24] +do_test 13.3.3 { + sqlite3_blob_read $B2 0 45 +} [string repeat [binary format c 0] 45] + +# And also written: +# +do_test 13.4.1 { + sqlite3_blob_write $B 0 [string repeat [binary format c 1] 24] +} {} +do_test 13.4.2 { + sqlite3_blob_write $B2 0 [string repeat [binary format c 1] 45] +} {} +do_test 13.5 { + sqlite3_blob_close $B + sqlite3_blob_close $B2 + execsql { SELECT j FROM c2 } +} [list \ + [string repeat [binary format c 1] 24] \ + [string repeat [binary format c 1] 45] \ +] + + +finish_test + diff --git a/test/fkey7.test b/test/fkey7.test index c2682edbe5..6c646a9a7f 100644 --- a/test/fkey7.test +++ b/test/fkey7.test @@ -50,5 +50,22 @@ do_tblsread_test 1.3 { UPDATE par SET a=? WHERE b=? } {c1 c2 par} do_tblsread_test 1.4 { UPDATE par SET c=? WHERE b=? } {c3 par} do_tblsread_test 1.5 { UPDATE par SET a=?,b=?,c=? WHERE b=? } {c1 c2 c3 par s1} +ifcapable incrblob { + do_execsql_test 2.0 { + CREATE TABLE pX(x PRIMARY KEY); + CREATE TABLE cX(a INTEGER PRIMARY KEY, b REFERENCES pX); + } + + do_catchsql_test 2.1 { + INSERT INTO cX VALUES(11, zeroblob(40)); + } {1 {FOREIGN KEY constraint failed}} + + do_test 2.2 { + set stmt [sqlite3_prepare_v2 db "INSERT INTO cX VALUES(11, ?)" -1] + sqlite3_bind_zeroblob $stmt 1 45 + sqlite3_step $stmt + sqlite3_finalize $stmt + } {SQLITE_CONSTRAINT} +} finish_test diff --git a/test/without_rowid5.test b/test/without_rowid5.test index 45e047befe..d163d9c1bc 100644 --- a/test/without_rowid5.test +++ b/test/without_rowid5.test @@ -185,8 +185,7 @@ do_execsql_test without_rowid5-5.9 { # EVIDENCE-OF: R-12643-30541 The incremental blob I/O mechanism does not # work for WITHOUT ROWID tables. # -# EVIDENCE-OF: R-25760-33257 The sqlite3_blob_open() interface will fail -# for a WITHOUT ROWID table. +# EVIDENCE-OF: R-40134-30296 Table zTable is a WITHOUT ROWID table # do_execsql_test without_rowid5-6.1 { CREATE TABLE b1(a INTEGER PRIMARY KEY, b BLOB) WITHOUT ROWID; From 4ace5362c250de7d712ef107275427aaa8c2c304 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 10 Nov 2014 14:42:28 +0000 Subject: [PATCH 093/133] Shorten over-length source code lines in shell.c. FossilOrigin-Name: 7f3819f6422badd344c1264b0cd2f2c7afe077df --- manifest | 16 ++++---- manifest.uuid | 2 +- src/shell.c | 100 +++++++++++++++++++++++++++++++++----------------- 3 files changed, 76 insertions(+), 42 deletions(-) diff --git a/manifest b/manifest index e08707974e..5e62c4081e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\snew\stest\sfile\se_blobopen.test,\scontaining\stests\sfor\ssqlite3_blob_open(). -D 2014-11-07T14:41:11.404 +C Shorten\sover-length\ssource\scode\slines\sin\sshell.c. +D 2014-11-10T14:42:28.114 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -228,7 +228,7 @@ F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b -F src/shell.c 0500307f63e3675dc4ab21112c6086b4d146fc08 +F src/shell.c bc28d5992109717c87804e2eb1a08a7c8cc7a2fd F src/sqlite.h.in 5abfd7d6c9bd2054105820de5ce12b1e491b1dc9 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d @@ -452,7 +452,7 @@ F test/descidx3.test 09ddbe3f5295f482d2f8b687cf6db8bad7acd9a2 F test/diskfull.test 106391384780753ea6896b7b4f005d10e9866b6e F test/distinct.test 086e70c765f172e8974e9f83b9ac5ca03c154e77 F test/distinctagg.test 1a6ef9c87a58669438fc771450d7a72577417376 -F test/e_blobopen.test 234f960d90235a9b51ec3ca1e062e8541dd558d8 w test/e_blob.test +F test/e_blobopen.test 234f960d90235a9b51ec3ca1e062e8541dd558d8 F test/e_changes.test fd66105385153dbf21fdb35eb8ef6c3e1eade579 F test/e_createtable.test c7e67b49e6cf92473c8fb30ab26143e9e2128cf7 F test/e_delete.test d5186e2f5478b659f16a2c8b66c09892823e542a @@ -1213,7 +1213,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P b45bc80bb16f07192d84fd14433bb724a84d4146 -R 0ead7b89f3ed1286937ae097bba351c9 -U dan -Z d785e78de9d268050b7beac3067db32e +P ecbccd0e594d22b3ae7fabc8037951dc49570bc3 +R 1a35da25c621486ba7906429ccbf946c +U drh +Z 427c2830707548878070627d5502fc48 diff --git a/manifest.uuid b/manifest.uuid index 6227627e36..541a9c512c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ecbccd0e594d22b3ae7fabc8037951dc49570bc3 \ No newline at end of file +7f3819f6422badd344c1264b0cd2f2c7afe077df \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index a33e65b1f8..bbd37c7a04 100644 --- a/src/shell.c +++ b/src/shell.c @@ -172,7 +172,8 @@ static HANDLE hProcess; static FILETIME ftKernelBegin; static FILETIME ftUserBegin; static sqlite3_int64 ftWallBegin; -typedef BOOL (WINAPI *GETPROCTIMES)(HANDLE, LPFILETIME, LPFILETIME, LPFILETIME, LPFILETIME); +typedef BOOL (WINAPI *GETPROCTIMES)(HANDLE, LPFILETIME, LPFILETIME, + LPFILETIME, LPFILETIME); static GETPROCTIMES getProcessTimesAddr = NULL; /* @@ -183,15 +184,16 @@ static int hasTimer(void){ if( getProcessTimesAddr ){ return 1; } else { - /* GetProcessTimes() isn't supported in WIN95 and some other Windows versions. - ** See if the version we are running on has it, and if it does, save off - ** a pointer to it and the current process handle. + /* GetProcessTimes() isn't supported in WIN95 and some other Windows + ** versions. See if the version we are running on has it, and if it + ** does, save off a pointer to it and the current process handle. */ hProcess = GetCurrentProcess(); if( hProcess ){ HINSTANCE hinstLib = LoadLibrary(TEXT("Kernel32.dll")); if( NULL != hinstLib ){ - getProcessTimesAddr = (GETPROCTIMES) GetProcAddress(hinstLib, "GetProcessTimes"); + getProcessTimesAddr = + (GETPROCTIMES) GetProcAddress(hinstLib, "GetProcessTimes"); if( NULL != getProcessTimesAddr ){ return 1; } @@ -208,7 +210,8 @@ static int hasTimer(void){ static void beginTimer(void){ if( enableTimer && getProcessTimesAddr ){ FILETIME ftCreation, ftExit; - getProcessTimesAddr(hProcess, &ftCreation, &ftExit, &ftKernelBegin, &ftUserBegin); + getProcessTimesAddr(hProcess,&ftCreation,&ftExit, + &ftKernelBegin,&ftUserBegin); ftWallBegin = timeOfDay(); } } @@ -227,7 +230,7 @@ static void endTimer(void){ if( enableTimer && getProcessTimesAddr){ FILETIME ftCreation, ftExit, ftKernelEnd, ftUserEnd; sqlite3_int64 ftWallEnd = timeOfDay(); - getProcessTimesAddr(hProcess, &ftCreation, &ftExit, &ftKernelEnd, &ftUserEnd); + getProcessTimesAddr(hProcess,&ftCreation,&ftExit,&ftKernelEnd,&ftUserEnd); printf("Run Time: real %.3f user %f sys %f\n", (ftWallEnd - ftWallBegin)*0.001, timeDiff(&ftUserBegin, &ftUserEnd), @@ -726,7 +729,13 @@ static void interrupt_handler(int NotUsed){ ** This is the callback routine that the shell ** invokes for each row of a query result. */ -static int shell_callback(void *pArg, int nArg, char **azArg, char **azCol, int *aiType){ +static int shell_callback( + void *pArg, + int nArg, /* Number of result columns */ + char **azArg, /* Text of each result column */ + char **azCol, /* Column names */ + int *aiType /* Column types */ +){ int i; ShellState *p = (ShellState*)pArg; @@ -1105,57 +1114,77 @@ static int display_stats( iHiwtr = iCur = -1; sqlite3_status(SQLITE_STATUS_MEMORY_USED, &iCur, &iHiwtr, bReset); - fprintf(pArg->out, "Memory Used: %d (max %d) bytes\n", iCur, iHiwtr); + fprintf(pArg->out, + "Memory Used: %d (max %d) bytes\n", + iCur, iHiwtr); iHiwtr = iCur = -1; sqlite3_status(SQLITE_STATUS_MALLOC_COUNT, &iCur, &iHiwtr, bReset); - fprintf(pArg->out, "Number of Outstanding Allocations: %d (max %d)\n", iCur, iHiwtr); + fprintf(pArg->out, "Number of Outstanding Allocations: %d (max %d)\n", + iCur, iHiwtr); if( pArg->shellFlgs & SHFLG_Pagecache ){ iHiwtr = iCur = -1; sqlite3_status(SQLITE_STATUS_PAGECACHE_USED, &iCur, &iHiwtr, bReset); - fprintf(pArg->out, "Number of Pcache Pages Used: %d (max %d) pages\n", iCur, iHiwtr); + fprintf(pArg->out, + "Number of Pcache Pages Used: %d (max %d) pages\n", + iCur, iHiwtr); } iHiwtr = iCur = -1; sqlite3_status(SQLITE_STATUS_PAGECACHE_OVERFLOW, &iCur, &iHiwtr, bReset); - fprintf(pArg->out, "Number of Pcache Overflow Bytes: %d (max %d) bytes\n", iCur, iHiwtr); + fprintf(pArg->out, + "Number of Pcache Overflow Bytes: %d (max %d) bytes\n", + iCur, iHiwtr); if( pArg->shellFlgs & SHFLG_Scratch ){ iHiwtr = iCur = -1; sqlite3_status(SQLITE_STATUS_SCRATCH_USED, &iCur, &iHiwtr, bReset); - fprintf(pArg->out, "Number of Scratch Allocations Used: %d (max %d)\n", iCur, iHiwtr); + fprintf(pArg->out, "Number of Scratch Allocations Used: %d (max %d)\n", + iCur, iHiwtr); } iHiwtr = iCur = -1; sqlite3_status(SQLITE_STATUS_SCRATCH_OVERFLOW, &iCur, &iHiwtr, bReset); - fprintf(pArg->out, "Number of Scratch Overflow Bytes: %d (max %d) bytes\n", iCur, iHiwtr); + fprintf(pArg->out, + "Number of Scratch Overflow Bytes: %d (max %d) bytes\n", + iCur, iHiwtr); iHiwtr = iCur = -1; sqlite3_status(SQLITE_STATUS_MALLOC_SIZE, &iCur, &iHiwtr, bReset); - fprintf(pArg->out, "Largest Allocation: %d bytes\n", iHiwtr); + fprintf(pArg->out, "Largest Allocation: %d bytes\n", + iHiwtr); iHiwtr = iCur = -1; sqlite3_status(SQLITE_STATUS_PAGECACHE_SIZE, &iCur, &iHiwtr, bReset); - fprintf(pArg->out, "Largest Pcache Allocation: %d bytes\n", iHiwtr); + fprintf(pArg->out, "Largest Pcache Allocation: %d bytes\n", + iHiwtr); iHiwtr = iCur = -1; sqlite3_status(SQLITE_STATUS_SCRATCH_SIZE, &iCur, &iHiwtr, bReset); - fprintf(pArg->out, "Largest Scratch Allocation: %d bytes\n", iHiwtr); + fprintf(pArg->out, "Largest Scratch Allocation: %d bytes\n", + iHiwtr); #ifdef YYTRACKMAXSTACKDEPTH iHiwtr = iCur = -1; sqlite3_status(SQLITE_STATUS_PARSER_STACK, &iCur, &iHiwtr, bReset); - fprintf(pArg->out, "Deepest Parser Stack: %d (max %d)\n", iCur, iHiwtr); + fprintf(pArg->out, "Deepest Parser Stack: %d (max %d)\n", + iCur, iHiwtr); #endif } if( pArg && pArg->out && db ){ if( pArg->shellFlgs & SHFLG_Lookaside ){ iHiwtr = iCur = -1; - sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_USED, &iCur, &iHiwtr, bReset); - fprintf(pArg->out, "Lookaside Slots Used: %d (max %d)\n", iCur, iHiwtr); - sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_HIT, &iCur, &iHiwtr, bReset); + sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_USED, + &iCur, &iHiwtr, bReset); + fprintf(pArg->out, "Lookaside Slots Used: %d (max %d)\n", + iCur, iHiwtr); + sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_HIT, + &iCur, &iHiwtr, bReset); fprintf(pArg->out, "Successful lookaside attempts: %d\n", iHiwtr); - sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE, &iCur, &iHiwtr, bReset); + sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE, + &iCur, &iHiwtr, bReset); fprintf(pArg->out, "Lookaside failures due to size: %d\n", iHiwtr); - sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL, &iCur, &iHiwtr, bReset); + sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL, + &iCur, &iHiwtr, bReset); fprintf(pArg->out, "Lookaside failures due to OOM: %d\n", iHiwtr); } iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_USED, &iCur, &iHiwtr, bReset); - fprintf(pArg->out, "Pager Heap Usage: %d bytes\n", iCur); iHiwtr = iCur = -1; + fprintf(pArg->out, "Pager Heap Usage: %d bytes\n",iCur); + iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_HIT, &iCur, &iHiwtr, 1); fprintf(pArg->out, "Page cache hits: %d\n", iCur); iHiwtr = iCur = -1; @@ -1166,18 +1195,19 @@ static int display_stats( fprintf(pArg->out, "Page cache writes: %d\n", iCur); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_SCHEMA_USED, &iCur, &iHiwtr, bReset); - fprintf(pArg->out, "Schema Heap Usage: %d bytes\n", iCur); + fprintf(pArg->out, "Schema Heap Usage: %d bytes\n",iCur); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_STMT_USED, &iCur, &iHiwtr, bReset); - fprintf(pArg->out, "Statement Heap/Lookaside Usage: %d bytes\n", iCur); + fprintf(pArg->out, "Statement Heap/Lookaside Usage: %d bytes\n",iCur); } if( pArg && pArg->out && db && pArg->pStmt ){ - iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_FULLSCAN_STEP, bReset); + iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_FULLSCAN_STEP, + bReset); fprintf(pArg->out, "Fullscan Steps: %d\n", iCur); iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_SORT, bReset); fprintf(pArg->out, "Sort Operations: %d\n", iCur); - iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_AUTOINDEX, bReset); + iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_AUTOINDEX,bReset); fprintf(pArg->out, "Autoindex Inserts: %d\n", iCur); iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_VM_STEP, bReset); fprintf(pArg->out, "Virtual Machine Steps: %d\n", iCur); @@ -1221,7 +1251,8 @@ static void display_scanstats( sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_EXPLAIN, (void*)&zExplain); fprintf(pArg->out, "Loop %2d: %s\n", n, zExplain); rEstLoop *= rEst; - fprintf(pArg->out, " nLoop=%-8lld nRow=%-8lld estRow=%-8lld estRow/Loop=%-8g\n", + fprintf(pArg->out, + " nLoop=%-8lld nRow=%-8lld estRow=%-8lld estRow/Loop=%-8g\n", nLoop, nVisit, (sqlite3_int64)(rEstLoop+0.5), rEst ); } @@ -1271,7 +1302,8 @@ static void explain_data_prepare(ShellState *p, sqlite3_stmt *pSql){ const char *azNext[] = { "Next", "Prev", "VPrev", "VNext", "SorterNext", "NextIfOpen", "PrevIfOpen", 0 }; - const char *azYield[] = { "Yield", "SeekLT", "SeekGT", "RowSetRead", "Rewind", 0 }; + const char *azYield[] = { "Yield", "SeekLT", "SeekGT", "RowSetRead", + "Rewind", 0 }; const char *azGoto[] = { "Goto", 0 }; /* Try to figure out if this is really an EXPLAIN statement. If this @@ -1384,7 +1416,8 @@ static int shell_exec( /* Show the EXPLAIN QUERY PLAN if .eqp is on */ if( pArg && pArg->autoEQP ){ sqlite3_stmt *pExplain; - char *zEQP = sqlite3_mprintf("EXPLAIN QUERY PLAN %s", sqlite3_sql(pStmt)); + char *zEQP = sqlite3_mprintf("EXPLAIN QUERY PLAN %s", + sqlite3_sql(pStmt)); rc = sqlite3_prepare_v2(db, zEQP, -1, &pExplain, 0); if( rc==SQLITE_OK ){ while( sqlite3_step(pExplain)==SQLITE_ROW ){ @@ -3335,7 +3368,7 @@ static int do_meta_command(char *zLine, ShellState *p){ for(i=0; iout, "%s%-*s", zSp, maxlen, azResult[j] ? azResult[j] : ""); + fprintf(p->out, "%s%-*s", zSp, maxlen, azResult[j] ? azResult[j]:""); } fprintf(p->out, "\n"); } @@ -3805,7 +3838,8 @@ static char *find_home_dir(void){ static char *home_dir = NULL; if( home_dir ) return home_dir; -#if !defined(_WIN32) && !defined(WIN32) && !defined(_WIN32_WCE) && !defined(__RTP__) && !defined(_WRS_KERNEL) +#if !defined(_WIN32) && !defined(WIN32) && !defined(_WIN32_WCE) \ + && !defined(__RTP__) && !defined(_WRS_KERNEL) { struct passwd *pwent; uid_t uid = getuid(); From 1728bcb07f9e6784bdf5c223ffe2c035011f9123 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 10 Nov 2014 16:49:56 +0000 Subject: [PATCH 094/133] Add the eval() SQL function extension in ext/misc/eval.c. FossilOrigin-Name: 27cf665b957f2c0ced403e3032099e80c295598f --- Makefile.in | 1 + Makefile.msc | 1 + ext/misc/eval.c | 119 ++++++++++++++++++++++++++++++++++++++++++++++++ main.mk | 1 + manifest | 20 ++++---- manifest.uuid | 2 +- src/test1.c | 2 + test/misc8.test | 41 +++++++++++++++++ 8 files changed, 177 insertions(+), 10 deletions(-) create mode 100644 ext/misc/eval.c create mode 100644 test/misc8.test diff --git a/Makefile.in b/Makefile.in index a2213e89e5..646cb39756 100644 --- a/Makefile.in +++ b/Makefile.in @@ -394,6 +394,7 @@ TESTSRC = \ TESTSRC += \ $(TOP)/ext/misc/amatch.c \ $(TOP)/ext/misc/closure.c \ + $(TOP)/ext/misc/eval.c \ $(TOP)/ext/misc/fileio.c \ $(TOP)/ext/misc/fuzzer.c \ $(TOP)/ext/misc/ieee754.c \ diff --git a/Makefile.msc b/Makefile.msc index 4173eaae27..4aaa2894a7 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -863,6 +863,7 @@ TESTSRC = \ TESTEXT = \ $(TOP)\ext\misc\amatch.c \ $(TOP)\ext\misc\closure.c \ + $(TOP)\ext\misc\eval.c \ $(TOP)\ext\misc\fileio.c \ $(TOP)\ext\misc\fuzzer.c \ $(TOP)\ext\misc\ieee754.c \ diff --git a/ext/misc/eval.c b/ext/misc/eval.c new file mode 100644 index 0000000000..a5e297ad38 --- /dev/null +++ b/ext/misc/eval.c @@ -0,0 +1,119 @@ +/* +** 2014-11-10 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This SQLite extension implements SQL function eval() which runs +** SQL statements recursively. +*/ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include + +/* +** Structure used to accumulate the output +*/ +struct EvalResult { + char *z; /* Accumulated output */ + const char *zSep; /* Separator */ + int szSep; /* Size of the separator string */ + int nAlloc; /* Number of bytes allocated for z[] */ + int nUsed; /* Number of bytes of z[] actually used */ +}; + +/* +** Callback from sqlite_exec() for the eval() function. +*/ +static int callback(void *pCtx, int argc, char **argv, char **colnames){ + struct EvalResult *p = (struct EvalResult*)pCtx; + int i; + for(i=0; inUsed+p->szSep+1 > p->nAlloc ){ + char *zNew; + p->nAlloc = p->nAlloc*2 + sz + p->szSep + 1; + zNew = sqlite3_realloc(p->z, p->nAlloc); + if( zNew==0 ){ + sqlite3_free(p->z); + memset(p, 0, sizeof(*p)); + return 1; + } + p->z = zNew; + } + if( p->nUsed>0 ){ + memcpy(&p->z[p->nUsed], p->zSep, p->szSep); + p->nUsed += p->szSep; + } + memcpy(&p->z[p->nUsed], z, sz); + p->nUsed += sz; + } + return 0; +} + +/* +** Implementation of the eval(X) and eval(X,Y) SQL functions. +** +** Evaluate the SQL text in X. Return the results, using string +** Y as the separator. If Y is omitted, use a single space character. +*/ +static void sqlEvalFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zSql; + sqlite3 *db; + char *zErr = 0; + int rc; + struct EvalResult x; + + memset(&x, 0, sizeof(x)); + x.zSep = " "; + zSql = (const char*)sqlite3_value_text(argv[0]); + if( zSql==0 ) return; + if( argc>1 ){ + x.zSep = (const char*)sqlite3_value_text(argv[1]); + if( x.zSep==0 ) return; + } + x.szSep = (int)strlen(x.zSep); + db = sqlite3_context_db_handle(context); + rc = sqlite3_exec(db, zSql, callback, &x, &zErr); + if( rc!=SQLITE_OK ){ + sqlite3_result_error(context, zErr, -1); + sqlite3_free(zErr); + }else if( x.zSep==0 ){ + sqlite3_result_error_nomem(context); + sqlite3_free(x.z); + }else{ + sqlite3_result_text(context, x.z, x.nUsed, sqlite3_free); + } +} + + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_eval_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Unused parameter */ + rc = sqlite3_create_function(db, "eval", 1, SQLITE_UTF8, 0, + sqlEvalFunc, 0, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "eval", 2, SQLITE_UTF8, 0, + sqlEvalFunc, 0, 0); + } + return rc; +} diff --git a/main.mk b/main.mk index c8bf72b48b..cac996864d 100644 --- a/main.mk +++ b/main.mk @@ -281,6 +281,7 @@ TESTSRC = \ TESTSRC += \ $(TOP)/ext/misc/amatch.c \ $(TOP)/ext/misc/closure.c \ + $(TOP)/ext/misc/eval.c \ $(TOP)/ext/misc/fileio.c \ $(TOP)/ext/misc/fuzzer.c \ $(TOP)/ext/misc/ieee754.c \ diff --git a/manifest b/manifest index 5e62c4081e..7f7fe158b9 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Shorten\sover-length\ssource\scode\slines\sin\sshell.c. -D 2014-11-10T14:42:28.114 +C Add\sthe\seval()\sSQL\sfunction\sextension\sin\sext/misc/eval.c. +D 2014-11-10T16:49:56.620 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f -F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a +F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 -F Makefile.msc e31dee24038965fb6269d6d61073fd6b7e331dec +F Makefile.msc 788f1288633a0c3c3cbbe0f3e4827d033f7ba530 F Makefile.vxworks 034289efa9d591b04b1a73598623119c306cbba0 F README.md 64f270c43c38c46de749e419c22f0ae2f4499fe8 F VERSION d846487aff892625eb8e75960234e7285f0462fe @@ -109,6 +109,7 @@ F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37 F ext/misc/amatch.c 678056a4bfcd83c4e82dea81d37543cd1d6dbee1 F ext/misc/closure.c 636024302cde41b2bf0c542f81c40c624cfb7012 F ext/misc/compress.c 76e45655f4046e756064ab10c62e18f2eb846b9f +F ext/misc/eval.c 04e630bde869aa1fec6b993d40591f963be2f868 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f F ext/misc/fuzzer.c 136533c53cfce0957f0b48fa11dba27e21c5c01d F ext/misc/ieee754.c b0362167289170627659e84173f5d2e8fee8566e @@ -151,7 +152,7 @@ F ext/userauth/userauth.c 5fa3bdb492f481bbc1709fc83c91ebd13460c69e F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk 3fececc835c5c23637c3e0af970e27a81a9ba476 +F main.mk 084976077a4aa3bd985154b5423e7aed88e4a2e9 F mkopcodec.awk c2ff431854d702cdd2d779c9c0d1f58fa16fa4ea F mkopcodeh.awk c6b3fa301db6ef7ac916b14c60868aeaec1337b5 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 @@ -237,7 +238,7 @@ F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 81712116e826b0089bb221b018929536b2b5406f F src/table.c f142bba7903e93ca8d113a5b8877a108ad1a27dc F src/tclsqlite.c 0a874655dd39a9875e39c5d3c464db662171d228 -F src/test1.c cce6ad0effe8c66a62d2634d9714ffdc7372ef11 +F src/test1.c 6b0469b8e06c77b1de1d3e4a3834cf26edea9cc7 F src/test2.c 98049e51a17dc62606a99a9eb95ee477f9996712 F src/test3.c 1c0e5d6f080b8e33c1ce8b3078e7013fdbcd560c F src/test4.c 9b32d22f5f150abe23c1830e2057c4037c45b3df @@ -729,6 +730,7 @@ F test/misc4.test 9c078510fbfff05a9869a0b6d8b86a623ad2c4f6 F test/misc5.test 528468b26d03303b1f047146e5eefc941b9069f5 F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test edd0b63e2ee29a256900b0514f6fff27e19e9bb2 +F test/misc8.test e538d122d5faacadbaa6fc02403f1e1427df684f F test/misuse.test 3c34719944ba045cc6c188a4852ba04680728912 F test/mmap1.test 1bfd611b9841eafb44f7d83c0788e146d84a33c9 F test/mmap2.test 9d6dd9ddb4ad2379f29cc78f38ce1e63ed418022 @@ -1213,7 +1215,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ecbccd0e594d22b3ae7fabc8037951dc49570bc3 -R 1a35da25c621486ba7906429ccbf946c +P 7f3819f6422badd344c1264b0cd2f2c7afe077df +R a579eb1c0cce07efa12ad49b9c702f07 U drh -Z 427c2830707548878070627d5502fc48 +Z 4009c695bea2e084a946ee86095e7182 diff --git a/manifest.uuid b/manifest.uuid index 541a9c512c..f90ad54213 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7f3819f6422badd344c1264b0cd2f2c7afe077df \ No newline at end of file +27cf665b957f2c0ced403e3032099e80c295598f \ No newline at end of file diff --git a/src/test1.c b/src/test1.c index e7a5929323..ca3b54a513 100644 --- a/src/test1.c +++ b/src/test1.c @@ -6260,6 +6260,7 @@ static int tclLoadStaticExtensionCmd( ){ extern int sqlite3_amatch_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_closure_init(sqlite3*,char**,const sqlite3_api_routines*); + extern int sqlite3_eval_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_fileio_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_fuzzer_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_ieee_init(sqlite3*,char**,const sqlite3_api_routines*); @@ -6275,6 +6276,7 @@ static int tclLoadStaticExtensionCmd( } aExtension[] = { { "amatch", sqlite3_amatch_init }, { "closure", sqlite3_closure_init }, + { "eval", sqlite3_eval_init }, { "fileio", sqlite3_fileio_init }, { "fuzzer", sqlite3_fuzzer_init }, { "ieee754", sqlite3_ieee_init }, diff --git a/test/misc8.test b/test/misc8.test new file mode 100644 index 0000000000..06ee314b60 --- /dev/null +++ b/test/misc8.test @@ -0,0 +1,41 @@ +# 2014-11-10 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. +# The focus of this script is testing the "eval.c" loadable extension. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +load_static_extension db eval +do_execsql_test misc8-1.0 { + CREATE TABLE t1(a,b,c); + INSERT INTO t1 VALUES(1,2,3),(4,5,6); + SELECT quote(eval('SELECT * FROM t1 ORDER BY a','-abc-')); +} {'1-abc-2-abc-3-abc-4-abc-5-abc-6'} +do_execsql_test misc8-1.1 { + SELECT quote(eval('SELECT * FROM t1 ORDER BY a')); +} {{'1 2 3 4 5 6'}} +do_catchsql_test misc8-1.2 { + SELECT quote(eval('SELECT d FROM t1 ORDER BY a')); +} {1 {no such column: d}} +do_execsql_test misc8-1.3 { + INSERT INTO t1 VALUES(7,null,9); + SELECT eval('SELECT * FROM t1 ORDER BY a',','); +} {1,2,3,4,5,6,7,,9} +do_catchsql_test misc8-1.4 { + BEGIN; + INSERT INTO t1 VALUES(10,11,12); + SELECT coalesce(b, eval('ROLLBACK')) FROM t1 ORDER BY a; +} {1 {abort due to ROLLBACK}} + + +finish_test From 923c4b35be963675b00b33a1af34dda0e6688dfa Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 10 Nov 2014 17:53:03 +0000 Subject: [PATCH 095/133] Add test file e_blobwrite.test, containing tests for the sqlite3_blob_write() interface. FossilOrigin-Name: 1df77e5f1bd82de4dc92fe28359c3e56ab3f9ed4 --- manifest | 17 ++-- manifest.uuid | 2 +- src/sqlite.h.in | 25 +++--- src/vdbeblob.c | 3 +- test/e_blobwrite.test | 204 ++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 229 insertions(+), 22 deletions(-) create mode 100644 test/e_blobwrite.test diff --git a/manifest b/manifest index 7f7fe158b9..d14dec8c51 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\seval()\sSQL\sfunction\sextension\sin\sext/misc/eval.c. -D 2014-11-10T16:49:56.620 +C Add\stest\sfile\se_blobwrite.test,\scontaining\stests\sfor\sthe\ssqlite3_blob_write()\sinterface. +D 2014-11-10T17:53:03.345 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -230,7 +230,7 @@ F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c bc28d5992109717c87804e2eb1a08a7c8cc7a2fd -F src/sqlite.h.in 5abfd7d6c9bd2054105820de5ce12b1e491b1dc9 +F src/sqlite.h.in 5531c4c69ee0a351c2aa5ad9f3f0f2424a57a9f4 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 71b0bf1a7fc55b5cb374f7579fd140e730a6e0f4 @@ -296,7 +296,7 @@ F src/vdbe.h 6fc69d9c5e146302c56e163cb4b31d1ee64a18c3 F src/vdbeInt.h 9bb69ff2447c34b6ccc58b34ec35b615f86ead78 F src/vdbeapi.c 07acb615d1e4170e71fc1b0d087f3c53a1ad8e83 F src/vdbeaux.c 9b0a251b6dfab349dd6c6efb40062eb7386b26f5 -F src/vdbeblob.c 8b5442ff0954c44b45cbabbe2e94091a2e16fdef +F src/vdbeblob.c 4af4bfb71f6df7778397b4a0ebc1879793276778 F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f F src/vdbesort.c 87f3923483113d1c95d84640becb4e4946f27d9a F src/vdbetrace.c 7e4222955e07dd707a2f360c0eb73452be1cb010 @@ -454,6 +454,7 @@ F test/diskfull.test 106391384780753ea6896b7b4f005d10e9866b6e F test/distinct.test 086e70c765f172e8974e9f83b9ac5ca03c154e77 F test/distinctagg.test 1a6ef9c87a58669438fc771450d7a72577417376 F test/e_blobopen.test 234f960d90235a9b51ec3ca1e062e8541dd558d8 +F test/e_blobwrite.test 615b405f29feb2cfb5a1f03dab7933258294fa26 F test/e_changes.test fd66105385153dbf21fdb35eb8ef6c3e1eade579 F test/e_createtable.test c7e67b49e6cf92473c8fb30ab26143e9e2128cf7 F test/e_delete.test d5186e2f5478b659f16a2c8b66c09892823e542a @@ -1215,7 +1216,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 7f3819f6422badd344c1264b0cd2f2c7afe077df -R a579eb1c0cce07efa12ad49b9c702f07 -U drh -Z 4009c695bea2e084a946ee86095e7182 +P 27cf665b957f2c0ced403e3032099e80c295598f +R 37c32da8fb4d2f1d3e3ebc03749cb035 +U dan +Z 2886cb98deb64368f4cf93641ea335c8 diff --git a/manifest.uuid b/manifest.uuid index f90ad54213..719ea3ef1b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -27cf665b957f2c0ced403e3032099e80c295598f \ No newline at end of file +1df77e5f1bd82de4dc92fe28359c3e56ab3f9ed4 \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index bed64bfae2..b5585defd6 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -5821,21 +5821,27 @@ int sqlite3_blob_read(sqlite3_blob *, void *Z, int N, int iOffset); /* ** CAPI3REF: Write Data Into A BLOB Incrementally ** -** ^This function is used to write data into an open [BLOB handle] from a -** caller-supplied buffer. ^N bytes of data are copied from the buffer Z -** into the open BLOB, starting at offset iOffset. +** ^(This function is used to write data into an open [BLOB handle] from a +** caller-supplied buffer. N bytes of data are copied from the buffer Z +** into the open BLOB, starting at offset iOffset.)^ +** +** ^(On success, sqlite3_blob_write() returns SQLITE_OK. +** Otherwise, an [error code] or an [extended error code] is returned.)^ +** ^Unless SQLITE_MISUSE is returned, this function sets the +** [database connection] error code and message accessible via +** [sqlite3_errcode()] and [sqlite3_errmsg()] and related functions. ** ** ^If the [BLOB handle] passed as the first argument was not opened for ** writing (the flags parameter to [sqlite3_blob_open()] was zero), ** this function returns [SQLITE_READONLY]. ** -** ^This function may only modify the contents of the BLOB; it is +** This function may only modify the contents of the BLOB; it is ** not possible to increase the size of a BLOB using this API. ** ^If offset iOffset is less than N bytes from the end of the BLOB, -** [SQLITE_ERROR] is returned and no data is written. ^If N is -** less than zero [SQLITE_ERROR] is returned and no data is written. -** The size of the BLOB (and hence the maximum value of N+iOffset) -** can be determined using the [sqlite3_blob_bytes()] interface. +** [SQLITE_ERROR] is returned and no data is written. The size of the +** BLOB (and hence the maximum value of N+iOffset) can be determined +** using the [sqlite3_blob_bytes()] interface. ^If N or iOffset are less +** than zero [SQLITE_ERROR] is returned and no data is written. ** ** ^An attempt to write to an expired [BLOB handle] fails with an ** error code of [SQLITE_ABORT]. ^Writes to the BLOB that occurred @@ -5844,9 +5850,6 @@ int sqlite3_blob_read(sqlite3_blob *, void *Z, int N, int iOffset); ** have been overwritten by the statement that expired the BLOB handle ** or by other independent statements. ** -** ^(On success, sqlite3_blob_write() returns SQLITE_OK. -** Otherwise, an [error code] or an [extended error code] is returned.)^ -** ** This routine only works on a [BLOB handle] which has been created ** by a prior successful call to [sqlite3_blob_open()] and which has not ** been closed by [sqlite3_blob_close()]. Passing any other pointer in diff --git a/src/vdbeblob.c b/src/vdbeblob.c index 0cf2b0652b..cf1eb59054 100644 --- a/src/vdbeblob.c +++ b/src/vdbeblob.c @@ -376,7 +376,6 @@ static int blobReadWrite( if( n<0 || iOffset<0 || (iOffset+n)>p->nByte ){ /* Request is out of range. Return a transient error. */ rc = SQLITE_ERROR; - sqlite3Error(db, SQLITE_ERROR); }else if( v==0 ){ /* If there is no statement handle, then the blob-handle has ** already been invalidated. Return SQLITE_ABORT in this case. @@ -394,10 +393,10 @@ static int blobReadWrite( sqlite3VdbeFinalize(v); p->pStmt = 0; }else{ - db->errCode = rc; v->rc = rc; } } + sqlite3Error(db, rc); rc = sqlite3ApiExit(db, rc); sqlite3_mutex_leave(db->mutex); return rc; diff --git a/test/e_blobwrite.test b/test/e_blobwrite.test new file mode 100644 index 0000000000..a0d33336de --- /dev/null +++ b/test/e_blobwrite.test @@ -0,0 +1,204 @@ +# 2014 October 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix e_blobwrite + +#-------------------------------------------------------------------------- +# EVIDENCE-OF: R-62898-22698 This function is used to write data into an +# open BLOB handle from a caller-supplied buffer. N bytes of data are +# copied from the buffer Z into the open BLOB, starting at offset +# iOffset. +# +set dots [string repeat . 40] +do_execsql_test 1.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, t TEXT); + INSERT INTO t1 VALUES(-1, $dots); + INSERT INTO t1 VALUES(-2, $dots); + INSERT INTO t1 VALUES(-3, $dots); + INSERT INTO t1 VALUES(-4, $dots); + INSERT INTO t1 VALUES(-5, $dots); + INSERT INTO t1 VALUES(-6, $dots); +} + +proc blob_write_test {tn id iOffset blob nData final} { + sqlite3_blob_open db main t1 t $id 1 B + + # EVIDENCE-OF: R-45864-01884 On success, sqlite3_blob_write() returns + # SQLITE_OK. Otherwise, an error code or an extended error code is + # returned. + # + # This block tests the SQLITE_OK case in the requirement above (the + # Tcl sqlite3_blob_write() wrapper uses an empty string in place of + # "SQLITE_OK"). The error cases are tested by the "blob_write_error_test" + # tests below. + # + set res [sqlite3_blob_write $B $iOffset $blob $nData] + uplevel [list do_test $tn.1 [list set {} $res] {}] + + sqlite3_blob_close $B + uplevel [list do_execsql_test $tn.3 "SELECT t FROM t1 WHERE a=$id" $final] +} + +set blob "0123456789012345678901234567890123456789" +blob_write_test 1.1 -1 0 $blob 10 { 0123456789.............................. } +blob_write_test 1.2 -2 8 $blob 10 { ........0123456789...................... } +blob_write_test 1.3 -3 8 $blob 1 { ........0............................... } +blob_write_test 1.4 -4 18 $blob 22 { ..................0123456789012345678901 } +blob_write_test 1.5 -5 18 $blob 0 { ........................................ } +blob_write_test 1.6 -6 0 $blob 40 { 0123456789012345678901234567890123456789 } + + +proc blob_write_error_test {tn B iOffset blob nData errcode errmsg} { + + # In cases where the underlying sqlite3_blob_write() function returns + # SQLITE_OK, the Tcl wrapper returns an empty string. If the underlying + # function returns an error, the Tcl wrapper throws an exception with + # the error code as the Tcl exception message. + # + if {$errcode=="SQLITE_OK"} { + set ret "" + set isError 0 + } else { + set ret $errcode + set isError 1 + } + + set cmd [list sqlite3_blob_write $B $iOffset $blob $nData] + uplevel [list do_test $tn.1 [subst -nocommands { + list [catch {$cmd} msg] [set msg] + }] [list $isError $ret]] + + # EVIDENCE-OF: R-34782-18311 Unless SQLITE_MISUSE is returned, this + # function sets the database connection error code and message + # accessible via sqlite3_errcode() and sqlite3_errmsg() and related + # functions. + # + if {$errcode == "SQLITE_MISUSE"} { error "test proc misuse!" } + uplevel [list do_test $tn.2 [list sqlite3_errcode db] $errcode] + uplevel [list do_test $tn.3 [list sqlite3_errmsg db] $errmsg] +} + +do_execsql_test 2.0 { + CREATE TABLE t2(a TEXT, b INTEGER PRIMARY KEY); + INSERT INTO t2 VALUES($dots, 43); + INSERT INTO t2 VALUES($dots, 44); + INSERT INTO t2 VALUES($dots, 45); +} + +# EVIDENCE-OF: R-63341-57517 If the BLOB handle passed as the first +# argument was not opened for writing (the flags parameter to +# sqlite3_blob_open() was zero), this function returns SQLITE_READONLY. +# +sqlite3_blob_open db main t2 a 43 0 B +blob_write_error_test 2.1 $B 0 $blob 10 \ + SQLITE_READONLY {attempt to write a readonly database} +sqlite3_blob_close $B + +# EVIDENCE-OF: R-29804-27366 If offset iOffset is less than N bytes from +# the end of the BLOB, SQLITE_ERROR is returned and no data is written. +# +sqlite3_blob_open db main t2 a 44 3 B +blob_write_error_test 2.2.1 $B 31 $blob 10 \ + SQLITE_ERROR {SQL logic error or missing database} + +# Make a successful write to the blob handle. This shows that the +# sqlite3_errcode() and sqlite3_errmsg() values are set even if the +# blob_write() call succeeds (see requirement in the [blob_write_error_test] +# proc). +blob_write_error_test 2.2.1 $B 30 $blob 10 SQLITE_OK {not an error} + +# EVIDENCE-OF: R-58570-38916 If N or iOffset are less than zero +# SQLITE_ERROR is returned and no data is written. +# +blob_write_error_test 2.2.2 $B 31 $blob -1 \ + SQLITE_ERROR {SQL logic error or missing database} +blob_write_error_test 2.2.3 $B 20 $blob 10 SQLITE_OK {not an error} +blob_write_error_test 2.2.4 $B -1 $blob 10 \ + SQLITE_ERROR {SQL logic error or missing database} +sqlite3_blob_close $B + +# EVIDENCE-OF: R-20958-54138 An attempt to write to an expired BLOB +# handle fails with an error code of SQLITE_ABORT. +# +do_test 2.3 { + sqlite3_blob_open db main t2 a 43 0 B + execsql { DELETE FROM t2 WHERE b=43 } +} {} +blob_write_error_test 2.3.1 $B 5 $blob 5 \ + SQLITE_ABORT {callback requested query abort} +do_test 2.3.2 { + execsql { SELECT 1, 2, 3 } + sqlite3_errcode db +} {SQLITE_OK} +blob_write_error_test 2.3.3 $B 5 $blob 5 \ + SQLITE_ABORT {callback requested query abort} +sqlite3_blob_close $B + +# EVIDENCE-OF: R-08382-59936 Writes to the BLOB that occurred before the +# BLOB handle expired are not rolled back by the expiration of the +# handle, though of course those changes might have been overwritten by +# the statement that expired the BLOB handle or by other independent +# statements. +# +# 3.1.*: not rolled back, +# 3.2.*: overwritten. +# +do_execsql_test 3.0 { + CREATE TABLE t3(i INTEGER PRIMARY KEY, j TEXT, k TEXT); + INSERT INTO t3 VALUES(1, $dots, $dots); + INSERT INTO t3 VALUES(2, $dots, $dots); + SELECT * FROM t3 WHERE i=1; +} { + 1 + ........................................ + ........................................ +} +sqlite3_blob_open db main t3 j 1 1 B +blob_write_error_test 3.1.1 $B 5 $blob 10 SQLITE_OK {not an error} +do_execsql_test 3.1.2 { + UPDATE t3 SET k = 'xyz' WHERE i=1; + SELECT * FROM t3 WHERE i=1; +} { + 1 .....0123456789......................... xyz +} +blob_write_error_test 3.1.3 $B 15 $blob 10 \ + SQLITE_ABORT {callback requested query abort} +sqlite3_blob_close $B +do_execsql_test 3.1.4 { + SELECT * FROM t3 WHERE i=1; +} { + 1 .....0123456789......................... xyz +} + +sqlite3_blob_open db main t3 j 2 1 B +blob_write_error_test 3.2.1 $B 5 $blob 10 SQLITE_OK {not an error} +do_execsql_test 3.2.2 { + UPDATE t3 SET j = 'xyz' WHERE i=2; + SELECT * FROM t3 WHERE i=2; +} { + 2 xyz ........................................ +} +blob_write_error_test 3.2.3 $B 15 $blob 10 \ + SQLITE_ABORT {callback requested query abort} +sqlite3_blob_close $B +do_execsql_test 3.2.4 { + SELECT * FROM t3 WHERE i=2; +} { + 2 xyz ........................................ +} + + + +finish_test + From de58f4fe7f33438a9d1b41666bb35a578d156885 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 10 Nov 2014 19:16:59 +0000 Subject: [PATCH 096/133] New test cases for deleting content out from under a SELECT statement. FossilOrigin-Name: 8289c3e9b47f7c2a606a88839f6bf615f8904ac2 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/misc8.test | 11 +++++++++++ 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index d14dec8c51..762ded40f6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stest\sfile\se_blobwrite.test,\scontaining\stests\sfor\sthe\ssqlite3_blob_write()\sinterface. -D 2014-11-10T17:53:03.345 +C New\stest\scases\sfor\sdeleting\scontent\sout\sfrom\sunder\sa\sSELECT\sstatement. +D 2014-11-10T19:16:59.654 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -731,7 +731,7 @@ F test/misc4.test 9c078510fbfff05a9869a0b6d8b86a623ad2c4f6 F test/misc5.test 528468b26d03303b1f047146e5eefc941b9069f5 F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test edd0b63e2ee29a256900b0514f6fff27e19e9bb2 -F test/misc8.test e538d122d5faacadbaa6fc02403f1e1427df684f +F test/misc8.test 2439b3576628bacab12cd6c73abed65de0979fd7 F test/misuse.test 3c34719944ba045cc6c188a4852ba04680728912 F test/mmap1.test 1bfd611b9841eafb44f7d83c0788e146d84a33c9 F test/mmap2.test 9d6dd9ddb4ad2379f29cc78f38ce1e63ed418022 @@ -1216,7 +1216,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 27cf665b957f2c0ced403e3032099e80c295598f -R 37c32da8fb4d2f1d3e3ebc03749cb035 -U dan -Z 2886cb98deb64368f4cf93641ea335c8 +P 1df77e5f1bd82de4dc92fe28359c3e56ab3f9ed4 +R 4b8ad408c58c17649b0e1468e9b897aa +U drh +Z 1daa82bfd52bfa0c612ccd059afbb463 diff --git a/manifest.uuid b/manifest.uuid index 719ea3ef1b..7f7352c14d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1df77e5f1bd82de4dc92fe28359c3e56ab3f9ed4 \ No newline at end of file +8289c3e9b47f7c2a606a88839f6bf615f8904ac2 \ No newline at end of file diff --git a/test/misc8.test b/test/misc8.test index 06ee314b60..da8f86970e 100644 --- a/test/misc8.test +++ b/test/misc8.test @@ -36,6 +36,17 @@ do_catchsql_test misc8-1.4 { INSERT INTO t1 VALUES(10,11,12); SELECT coalesce(b, eval('ROLLBACK')) FROM t1 ORDER BY a; } {1 {abort due to ROLLBACK}} +do_catchsql_test misc8-1.5 { + INSERT INTO t1 VALUES(10,11,12); + SELECT a, coalesce(b, eval('SELECT ''bam''')), c + FROM t1 + ORDER BY rowid; +} {0 {1 2 3 4 5 6 7 bam 9 10 11 12}} +do_catchsql_test misc8-1.6 { + SELECT a, coalesce(b, eval('DELETE FROM t1; SELECT ''bam''')), c + FROM t1 + ORDER BY rowid; +} {0 {1 2 3 4 5 6 7 bam {}}} finish_test From 47b7fc784347f2817c7669b245e5c4d65e6a3747 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 11 Nov 2014 01:33:57 +0000 Subject: [PATCH 097/133] Experimental changes that permit read operations to continue after a ROLLBACK, as long as the schema is unchanged. FossilOrigin-Name: fa6e6a9ae276cad60e9a4abc1bc23cf2809ea786 --- manifest | 41 ++++++++++++++++++---------------- manifest.uuid | 2 +- src/backup.c | 2 +- src/btree.c | 46 ++++++++++++++++++++++----------------- src/btree.h | 4 ++-- src/main.c | 9 +++++--- src/vdbe.c | 11 +++++++--- src/wal.c | 3 +-- test/capi3.test | 6 ++--- test/capi3c.test | 6 ++--- test/misc8.test | 13 +++++++++-- test/rollback.test | 4 ++-- test/savepoint.test | 2 +- test/savepoint7.test | 6 +++-- test/tkt-f777251dc7a.test | 4 +++- test/trans3.test | 2 +- 16 files changed, 95 insertions(+), 66 deletions(-) diff --git a/manifest b/manifest index 762ded40f6..5ef3b08ad7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C New\stest\scases\sfor\sdeleting\scontent\sout\sfrom\sunder\sa\sSELECT\sstatement. -D 2014-11-10T19:16:59.654 +C Experimental\schanges\sthat\spermit\sread\soperations\sto\scontinue\safter\sa\nROLLBACK,\sas\slong\sas\sthe\sschema\sis\sunchanged. +D 2014-11-11T01:33:57.231 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -170,11 +170,11 @@ F src/alter.c ba266a779bc7ce10e52e59e7d3dc79fa342e8fdb F src/analyze.c afbcca663c3f3625340b8e30d440cd7a97ded6bc F src/attach.c f4e94df2d1826feda65eb0939f7f6f5f923a0ad9 F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 -F src/backup.c 7f841396adfd47507ff670a471162d2bfcda3136 +F src/backup.c 86b311af1ad35decd1f2f0cee457d344acb83c3b F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 4a126e2066076872ab6f37f9ad116eb5f651cd38 -F src/btree.h 49b408be9c1cd41249076898e0673711071205d8 +F src/btree.c eaef0003bbfe740c62189355dabc818fc3a98999 +F src/btree.h d24fc2f3bc53be220b79b95800bdb2ee207b1089 F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 67bb05b1077e0cdaccb2e36bfcbe7a5df9ed31e8 F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 @@ -195,7 +195,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 F src/loadext.c de741e66e5ddc1598d904d7289239696e40ed994 -F src/main.c f88ed28716cbbada0f3d81479e6d43823b553de6 +F src/main.c d3310d5ed56e246bf1589e47eeaca8be582bd4b8 F src/malloc.c 740db54387204c9a2eb67c6d98e68b08e9ef4eab F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c faf615aafd8be74a71494dfa027c113ea5c6615f @@ -291,7 +291,7 @@ F src/update.c 3c4ecc282accf12d39edb8d524cf089645e55a13 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 3b627daa45c7308c1e36e3dbaa3f9ce7e5c7fa73 F src/vacuum.c 9b30ec729337dd012ed88d4c292922c8ef9cf00c -F src/vdbe.c d5dab22208e36e5689e9fb553aea3613921054ec +F src/vdbe.c c71d819bb34269c3dbccd92e6bb308f0ec025b5d F src/vdbe.h 6fc69d9c5e146302c56e163cb4b31d1ee64a18c3 F src/vdbeInt.h 9bb69ff2447c34b6ccc58b34ec35b615f86ead78 F src/vdbeapi.c 07acb615d1e4170e71fc1b0d087f3c53a1ad8e83 @@ -301,7 +301,7 @@ F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f F src/vdbesort.c 87f3923483113d1c95d84640becb4e4946f27d9a F src/vdbetrace.c 7e4222955e07dd707a2f360c0eb73452be1cb010 F src/vtab.c 2a30791bbd7926b589401bd09c3abb33de563793 -F src/wal.c 825c948066c7604a07d56e67958cdab210749016 +F src/wal.c fa090966140602f03a621f87d82ee69e66ca63b5 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 F src/where.c 3862a1173ae2716bde12f1ab3fb649f1d85b05c2 @@ -381,9 +381,9 @@ F test/btreefault.test c2bcb542685eea44621275cfedbd8a13f65201e3 F test/busy.test 76b4887f8b9160ba903c1ac22e8ff406ad6ae2f0 F test/cache.test 13bc046b26210471ca6f2889aceb1ea52dc717de F test/capi2.test 011c16da245fdc0106a2785035de6b242c05e738 -F test/capi3.test 71bcf2fbd36a9732f617766dfd752552c8e491b5 +F test/capi3.test f0718f4f90d0efdc980119bfbdf1d7f1541ee5ef F test/capi3b.test efb2b9cfd127efa84433cd7a2d72ce0454ae0dc4 -F test/capi3c.test a21869e4d50d5dbb7e566e328fc0bc7c2efa6a32 +F test/capi3c.test fdc0d67a2cb8e8fc400d5b7735e330161ea057a2 F test/capi3d.test c84af0c49267f9c3fbf4c1c46aa647646023811e F test/capi3e.test 3d49c01ef2a1a55f41d73cba2b23b5059ec460fe F test/cast.test 4c275cbdc8202d6f9c54a3596701719868ac7dc3 @@ -731,7 +731,7 @@ F test/misc4.test 9c078510fbfff05a9869a0b6d8b86a623ad2c4f6 F test/misc5.test 528468b26d03303b1f047146e5eefc941b9069f5 F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test edd0b63e2ee29a256900b0514f6fff27e19e9bb2 -F test/misc8.test 2439b3576628bacab12cd6c73abed65de0979fd7 +F test/misc8.test e838ec20c9c988bc94812fdb89af26409c20931b F test/misuse.test 3c34719944ba045cc6c188a4852ba04680728912 F test/mmap1.test 1bfd611b9841eafb44f7d83c0788e146d84a33c9 F test/mmap2.test 9d6dd9ddb4ad2379f29cc78f38ce1e63ed418022 @@ -794,18 +794,18 @@ F test/reindex.test 44edd3966b474468b823d481eafef0c305022254 F test/releasetest.mk 2eced2f9ae701fd0a29e714a241760503ccba25a F test/releasetest.tcl a4279c890698584feb2ffc86735857a4e4474180 F test/resolver01.test 33abf37ff8335e6bf98f2b45a0af3e06996ccd9a -F test/rollback.test e9504a009a202c3ed711da2e6879ff60c5a4669c +F test/rollback.test 458fe73eb3ffdfdf9f6ba3e9b7350a6220414dea F test/rowhash.test 0bc1d31415e4575d10cacf31e1a66b5cc0f8be81 F test/rowid.test 742b5741584a8a44fd83e856cc2896688401d645 F test/rtree.test 0c8d9dd458d6824e59683c19ab2ffa9ef946f798 F test/run-wordcount.sh 891e89c4c2d16e629cd45951d4ed899ad12afc09 -F test/savepoint.test 51d3900dc071a7c2ad4248578a5925631b476313 +F test/savepoint.test c671fdbd34cd3bfe1518a777526ada595180cf8d F test/savepoint2.test 9b8543940572a2f01a18298c3135ad0c9f4f67d7 F test/savepoint3.test e328085853b14898d78ceea00dfe7db18bb6a9ec F test/savepoint4.test c8f8159ade6d2acd9128be61e1230f1c1edc6cc0 F test/savepoint5.test 0735db177e0ebbaedc39812c8d065075d563c4fd F test/savepoint6.test f41279c5e137139fa5c21485773332c7adb98cd7 -F test/savepoint7.test fbf319a7b2dda089ec5be30a424a0e95f121d423 +F test/savepoint7.test 1c8f26b1e2a4221b0214e222ce12a97a59918eb2 F test/scanstatus.test a6dd739bc4d9638e8f5c2493b518057f2b681655 F test/schema.test 8f7999be894260f151adf15c2c7540f1c6d6a481 F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5 @@ -954,7 +954,7 @@ F test/tkt-d635236375.test 9d37e988b47d87505bc9445be0ca447002df5d09 F test/tkt-d82e3f3721.test bcc0dfba658d15bab30fd4a9320c9e35d214ce30 F test/tkt-f3e5abed55.test d5a0126118142d13e27f6ce9f4c47096e9321c00 F test/tkt-f67b41381a.test a23bc124c981662db712167bacd0ed8ad11abac9 -F test/tkt-f777251dc7a.test af6531446c64bfd268416f07b4df7be7f9c749d2 +F test/tkt-f777251dc7a.test 6295d235a03c82160549da4841a83dc8e758932f F test/tkt-f7b4edec.test d998a08ff2b18b7f62edce8e3044317c45efe6c7 F test/tkt-f973c7ac31.test 28ef85c7f015477916795246d8286aeda39d4ead F test/tkt-fa7bf5ec.test 9102dfea58aa371d78969da735f9392c57e2e035 @@ -1049,7 +1049,7 @@ F test/trace.test 73a5508100f7fccfbc3f8018d5f6963ed478eea0 F test/trace2.test 93b47ca6996c66b47f57224cfb146f34e07df382 F test/trans.test 6e1b4c6a42dba31bd65f8fa5e61a2708e08ddde6 F test/trans2.test 62bd045bfc7a1c14c5ba83ba64d21ade31583f76 -F test/trans3.test 373ac5183cc56be69f48ae44090e7f672939f732 +F test/trans3.test 91a100e5412b488e22a655fe423a14c26403ab94 F test/transitive1.test 03f532954f46cdf5608f7766bff0b0c52bf2a7cd F test/trigger1.test dc47573ac79ffe0ee3eecaa517d70d8dacbccd03 F test/trigger2.test 5cd7d69a7ba1143ee045e4ae2963ff32ae4c87a6 @@ -1216,7 +1216,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 1df77e5f1bd82de4dc92fe28359c3e56ab3f9ed4 -R 4b8ad408c58c17649b0e1468e9b897aa +P 8289c3e9b47f7c2a606a88839f6bf615f8904ac2 +R 703cbe5a59d81119b89af85a339f9302 +T *branch * read-after-rollback +T *sym-read-after-rollback * +T -sym-trunk * U drh -Z 1daa82bfd52bfa0c612ccd059afbb463 +Z dd5cfd6556a10d97eb1c5e11a770b9f2 diff --git a/manifest.uuid b/manifest.uuid index 7f7352c14d..0b7de2e6e0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8289c3e9b47f7c2a606a88839f6bf615f8904ac2 \ No newline at end of file +fa6e6a9ae276cad60e9a4abc1bc23cf2809ea786 \ No newline at end of file diff --git a/src/backup.c b/src/backup.c index da4303e5fd..57f7f447ea 100644 --- a/src/backup.c +++ b/src/backup.c @@ -607,7 +607,7 @@ int sqlite3_backup_finish(sqlite3_backup *p){ } /* If a transaction is still open on the Btree, roll it back. */ - sqlite3BtreeRollback(p->pDest, SQLITE_OK); + sqlite3BtreeRollback(p->pDest, SQLITE_OK, 0); /* Set the error code of the destination database handle. */ rc = (p->rc==SQLITE_DONE) ? SQLITE_OK : p->rc; diff --git a/src/btree.c b/src/btree.c index 8b4a2a4f3b..70fae7c707 100644 --- a/src/btree.c +++ b/src/btree.c @@ -2213,7 +2213,7 @@ int sqlite3BtreeClose(Btree *p){ ** The call to sqlite3BtreeRollback() drops any table-locks held by ** this handle. */ - sqlite3BtreeRollback(p, SQLITE_OK); + sqlite3BtreeRollback(p, SQLITE_OK, 0); sqlite3BtreeLeave(p); /* If there are still other outstanding references to the shared-btree @@ -3506,27 +3506,28 @@ int sqlite3BtreeCommit(Btree *p){ /* ** This routine sets the state to CURSOR_FAULT and the error -** code to errCode for every cursor on BtShared that pBtree -** references. +** code to errCode for every cursor on any BtShared that pBtree +** references. Or if the writeOnly flag is set to 1, then only +** trip write cursors and leave read cursors unchanged. ** -** Every cursor is tripped, including cursors that belong -** to other database connections that happen to be sharing -** the cache with pBtree. +** Every cursor is a candidate to be tripped, including cursors +** that belong to other database connections that happen to be +** sharing the cache with pBtree. ** -** This routine gets called when a rollback occurs. -** All cursors using the same cache must be tripped -** to prevent them from trying to use the btree after -** the rollback. The rollback may have deleted tables -** or moved root pages, so it is not sufficient to -** save the state of the cursor. The cursor must be -** invalidated. +** This routine gets called when a rollback occurs. The writeOnly +** flag is set to 1 if the transaction did not make any schema +** changes, in which case the read cursors can continue operating. +** If schema changes did occur in the transaction, then both read +** and write cursors must both be tripped. */ -void sqlite3BtreeTripAllCursors(Btree *pBtree, int errCode){ +void sqlite3BtreeTripAllCursors(Btree *pBtree, int errCode, int writeOnly){ BtCursor *p; + assert( (writeOnly==0 || writeOnly==1) && BTCF_WriteFlag==1 ); if( pBtree==0 ) return; sqlite3BtreeEnter(pBtree); for(p=pBtree->pBt->pCursor; p; p=p->pNext){ int i; + if( writeOnly && (p->curFlags & BTCF_WriteFlag)==0 ) continue; sqlite3BtreeClearCursor(p); p->eState = CURSOR_FAULT; p->skipNext = errCode; @@ -3539,27 +3540,32 @@ void sqlite3BtreeTripAllCursors(Btree *pBtree, int errCode){ } /* -** Rollback the transaction in progress. All cursors will be -** invalided by this operation. Any attempt to use a cursor -** that was open at the beginning of this operation will result -** in an error. +** Rollback the transaction in progress. +** +** If tripCode is not SQLITE_OK then cursors will be invalidated (tripped). +** Only write cursors are tripped if writeOnly is true but all cursors are +** tripped if writeOnly is false. Any attempt to use +** a tripped cursor will result in an error. ** ** This will release the write lock on the database file. If there ** are no active cursors, it also releases the read lock. */ -int sqlite3BtreeRollback(Btree *p, int tripCode){ +int sqlite3BtreeRollback(Btree *p, int tripCode, int writeOnly){ int rc; BtShared *pBt = p->pBt; MemPage *pPage1; + assert( writeOnly==1 || writeOnly==0 ); + assert( tripCode==SQLITE_ABORT_ROLLBACK || tripCode==SQLITE_OK ); sqlite3BtreeEnter(p); if( tripCode==SQLITE_OK ){ rc = tripCode = saveAllCursors(pBt, 0, 0); + if( rc ) writeOnly = 0; }else{ rc = SQLITE_OK; } if( tripCode ){ - sqlite3BtreeTripAllCursors(p, tripCode); + sqlite3BtreeTripAllCursors(p, tripCode, writeOnly); } btreeIntegrity(p); diff --git a/src/btree.h b/src/btree.h index 63337b1946..a6093a5edd 100644 --- a/src/btree.h +++ b/src/btree.h @@ -83,7 +83,7 @@ int sqlite3BtreeBeginTrans(Btree*,int); int sqlite3BtreeCommitPhaseOne(Btree*, const char *zMaster); int sqlite3BtreeCommitPhaseTwo(Btree*, int); int sqlite3BtreeCommit(Btree*); -int sqlite3BtreeRollback(Btree*,int); +int sqlite3BtreeRollback(Btree*,int,int); int sqlite3BtreeBeginStmt(Btree*,int); int sqlite3BtreeCreateTable(Btree*, int*, int flags); int sqlite3BtreeIsInTrans(Btree*); @@ -116,7 +116,7 @@ int sqlite3BtreeIncrVacuum(Btree *); int sqlite3BtreeDropTable(Btree*, int, int*); int sqlite3BtreeClearTable(Btree*, int, int*); int sqlite3BtreeClearTableOfCursor(BtCursor*); -void sqlite3BtreeTripAllCursors(Btree*, int); +void sqlite3BtreeTripAllCursors(Btree*, int, int); void sqlite3BtreeGetMeta(Btree *pBtree, int idx, u32 *pValue); int sqlite3BtreeUpdateMeta(Btree*, int idx, u32 value); diff --git a/src/main.c b/src/main.c index ae3753c139..f223b71f74 100644 --- a/src/main.c +++ b/src/main.c @@ -1111,13 +1111,15 @@ void sqlite3LeaveMutexAndCloseZombie(sqlite3 *db){ /* ** Rollback all database files. If tripCode is not SQLITE_OK, then -** any open cursors are invalidated ("tripped" - as in "tripping a circuit +** any write cursors are invalidated ("tripped" - as in "tripping a circuit ** breaker") and made to return tripCode if there are any further -** attempts to use that cursor. +** attempts to use that cursor. Read cursors remain open and valid +** but are "saved" in case the table pages are moved around. */ void sqlite3RollbackAll(sqlite3 *db, int tripCode){ int i; int inTrans = 0; + int schemaChange; assert( sqlite3_mutex_held(db->mutex) ); sqlite3BeginBenignMalloc(); @@ -1128,6 +1130,7 @@ void sqlite3RollbackAll(sqlite3 *db, int tripCode){ ** the database rollback and schema reset, which can cause false ** corruption reports in some cases. */ sqlite3BtreeEnterAll(db); + schemaChange = (db->flags & SQLITE_InternChanges)!=0 && db->init.busy==0; for(i=0; inDb; i++){ Btree *p = db->aDb[i].pBt; @@ -1135,7 +1138,7 @@ void sqlite3RollbackAll(sqlite3 *db, int tripCode){ if( sqlite3BtreeIsInTrans(p) ){ inTrans = 1; } - sqlite3BtreeRollback(p, tripCode); + sqlite3BtreeRollback(p, tripCode, !schemaChange); } } sqlite3VtabRollback(db); diff --git a/src/vdbe.c b/src/vdbe.c index f80e7787af..715862e73c 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2825,11 +2825,16 @@ case OP_Savepoint: { db->isTransactionSavepoint = 0; rc = p->rc; }else{ + int isSchemaChange; iSavepoint = db->nSavepoint - iSavepoint - 1; if( p1==SAVEPOINT_ROLLBACK ){ + isSchemaChange = (db->flags & SQLITE_InternChanges)!=0; for(ii=0; iinDb; ii++){ - sqlite3BtreeTripAllCursors(db->aDb[ii].pBt, SQLITE_ABORT); + sqlite3BtreeTripAllCursors(db->aDb[ii].pBt, SQLITE_ABORT, + isSchemaChange==0); } + }else{ + isSchemaChange = 0; } for(ii=0; iinDb; ii++){ rc = sqlite3BtreeSavepoint(db->aDb[ii].pBt, p1, iSavepoint); @@ -2837,7 +2842,7 @@ case OP_Savepoint: { goto abort_due_to_error; } } - if( p1==SAVEPOINT_ROLLBACK && (db->flags&SQLITE_InternChanges)!=0 ){ + if( isSchemaChange ){ sqlite3ExpirePreparedStatements(db); sqlite3ResetAllSchemasOfConnection(db); db->flags = (db->flags | SQLITE_InternChanges); @@ -3234,7 +3239,7 @@ case OP_OpenWrite: { || p->readOnly==0 ); if( p->expired ){ - rc = SQLITE_ABORT; + rc = SQLITE_ABORT_ROLLBACK; break; } diff --git a/src/wal.c b/src/wal.c index d2ed293a4d..6fed8f3c6c 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2506,7 +2506,7 @@ int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx){ memcpy(&pWal->hdr, (void *)walIndexHdr(pWal), sizeof(WalIndexHdr)); for(iFrame=pWal->hdr.mxFrame+1; - ALWAYS(rc==SQLITE_OK) && iFrame<=iMax; + rc==SQLITE_OK && iFrame<=iMax; iFrame++ ){ /* This call cannot fail. Unless the page for which the page number @@ -2525,7 +2525,6 @@ int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx){ } if( iMax!=pWal->hdr.mxFrame ) walCleanupHash(pWal); } - assert( rc==SQLITE_OK ); return rc; } diff --git a/test/capi3.test b/test/capi3.test index cbaa5c5e5b..9f3d6f6916 100644 --- a/test/capi3.test +++ b/test/capi3.test @@ -912,7 +912,7 @@ do_test capi3-11.9.3 { } 1 do_test capi3-11.10 { sqlite3_step $STMT -} {SQLITE_ERROR} +} {SQLITE_ROW} ifcapable !autoreset { # If SQLITE_OMIT_AUTORESET is defined, then the statement must be # reset() before it can be passed to step() again. @@ -921,11 +921,11 @@ ifcapable !autoreset { } do_test capi3-11.11 { sqlite3_step $STMT -} {SQLITE_ROW} +} {SQLITE_DONE} do_test capi3-11.12 { sqlite3_step $STMT sqlite3_step $STMT -} {SQLITE_DONE} +} {SQLITE_ROW} do_test capi3-11.13 { sqlite3_finalize $STMT } {SQLITE_OK} diff --git a/test/capi3c.test b/test/capi3c.test index 6388717e00..6ab3bc24f6 100644 --- a/test/capi3c.test +++ b/test/capi3c.test @@ -863,7 +863,7 @@ do_test capi3c-11.9.3 { } 1 do_test capi3c-11.10 { sqlite3_step $STMT -} {SQLITE_ABORT} +} {SQLITE_ROW} ifcapable !autoreset { # If SQLITE_OMIT_AUTORESET is defined, then the statement must be # reset() before it can be passed to step() again. @@ -872,11 +872,11 @@ ifcapable !autoreset { } do_test capi3c-11.11 { sqlite3_step $STMT -} {SQLITE_ROW} +} {SQLITE_DONE} do_test capi3c-11.12 { sqlite3_step $STMT sqlite3_step $STMT -} {SQLITE_DONE} +} {SQLITE_ROW} do_test capi3c-11.13 { sqlite3_finalize $STMT } {SQLITE_OK} diff --git a/test/misc8.test b/test/misc8.test index da8f86970e..8c0c126a61 100644 --- a/test/misc8.test +++ b/test/misc8.test @@ -34,8 +34,9 @@ do_execsql_test misc8-1.3 { do_catchsql_test misc8-1.4 { BEGIN; INSERT INTO t1 VALUES(10,11,12); - SELECT coalesce(b, eval('ROLLBACK')) FROM t1 ORDER BY a; -} {1 {abort due to ROLLBACK}} + SELECT a, coalesce(b, eval('ROLLBACK; SELECT ''bam'';')), c + FROM t1 ORDER BY a; +} {0 {1 2 3 4 5 6 7 bam 9}} do_catchsql_test misc8-1.5 { INSERT INTO t1 VALUES(10,11,12); SELECT a, coalesce(b, eval('SELECT ''bam''')), c @@ -47,6 +48,14 @@ do_catchsql_test misc8-1.6 { FROM t1 ORDER BY rowid; } {0 {1 2 3 4 5 6 7 bam {}}} +do_catchsql_test misc8-1.7 { + INSERT INTO t1 VALUES(1,2,3),(4,5,6),(7,null,9); + BEGIN; + CREATE TABLE t2(x); + SELECT a, coalesce(b, eval('ROLLBACK; SELECT ''bam''')), c + FROM t1 + ORDER BY rowid; +} {1 {abort due to ROLLBACK}} finish_test diff --git a/test/rollback.test b/test/rollback.test index c339c5d7d6..7abafece61 100644 --- a/test/rollback.test +++ b/test/rollback.test @@ -60,11 +60,11 @@ ifcapable conflict { # do_test rollback-1.5 { sqlite3_step $STMT - } {SQLITE_ERROR} + } {SQLITE_ROW} # Restart the SELECT statement # - do_test rollback-1.6 { sqlite3_reset $STMT } {SQLITE_ABORT} + do_test rollback-1.6 { sqlite3_reset $STMT } {SQLITE_OK} } else { do_test rollback-1.6 { sqlite3_reset $STMT } {SQLITE_OK} } diff --git a/test/savepoint.test b/test/savepoint.test index 9362c8fe19..8055e61d9e 100644 --- a/test/savepoint.test +++ b/test/savepoint.test @@ -315,7 +315,7 @@ ifcapable incrblob { do_test savepoint-5.3.2.3 { set rc [catch {seek $fd 0; read $fd} res] set rc - } {1} + } {0} do_test savepoint-5.3.3 { catchsql {RELEASE def} } {0 {}} diff --git a/test/savepoint7.test b/test/savepoint7.test index bc99187d27..908ec571f5 100644 --- a/test/savepoint7.test +++ b/test/savepoint7.test @@ -30,6 +30,7 @@ do_test savepoint7-1.1 { db eval {SELECT * FROM t1} { db eval { SAVEPOINT x2; + CREATE TABLE IF NOT EXISTS t3(xyz); INSERT INTO t2 VALUES($a,$b,$c); RELEASE x2; } @@ -46,7 +47,7 @@ do_test savepoint7-1.2 { RELEASE x2; } } - db eval {SELECT * FROM t2} + db eval {SELECT * FROM t2;} } {1 2 3 4 5 6 7 8 9} do_test savepoint7-1.3 { @@ -65,7 +66,7 @@ do_test savepoint7-1.3 { # queries in outer contexts. # do_test savepoint7-2.1 { - db eval {DELETE FROM t2; SAVEPOINT x1;} + db eval {DELETE FROM t2; SAVEPOINT x1; CREATE TABLE t4(abc);} set rc [catch { db eval {SELECT * FROM t1} { db eval { @@ -85,6 +86,7 @@ do_test savepoint7-2.2 { db eval {SELECT * FROM t1} { db eval { SAVEPOINT x2; + CREATE TABLE t5(pqr); INSERT INTO t2 VALUES($a,$b,$c); ROLLBACK TO x2; } diff --git a/test/tkt-f777251dc7a.test b/test/tkt-f777251dc7a.test index af6f71ad96..f814d246bf 100644 --- a/test/tkt-f777251dc7a.test +++ b/test/tkt-f777251dc7a.test @@ -38,8 +38,10 @@ proc force_rollback {} { db function force_rollback force_rollback do_test tkt-f7772-1.2 { +breakpoint catchsql { BEGIN IMMEDIATE; + CREATE TABLE xyzzy(abc); SELECT x, force_rollback(), EXISTS(SELECT 1 FROM t3 WHERE w=x) FROM t2; } } {1 {abort due to ROLLBACK}} @@ -67,7 +69,7 @@ do_test tkt-f7772-2.2 { catchsql { SELECT x, force_rollback(), EXISTS(SELECT 1 FROM t3 WHERE w=x) FROM t2 } -} {1 {callback requested query abort}} +} {1 {abort due to ROLLBACK}} do_test tkt-f7772-2.3 { sqlite3_get_autocommit db } {1} diff --git a/test/trans3.test b/test/trans3.test index d5b316bec8..e828442415 100644 --- a/test/trans3.test +++ b/test/trans3.test @@ -52,7 +52,7 @@ do_test trans3-1.4 { db eval {SELECT * FROM t1} } {1 2 3 4} do_test trans3-1.5 { - db eval BEGIN + db eval {BEGIN; CREATE TABLE xyzzy(abc);} db eval {INSERT INTO t1 VALUES(5);} set ::ecode {} set x [catch { From 43f4066e102d096958e3d3211a4e44b98d6d83df Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 11 Nov 2014 12:20:35 +0000 Subject: [PATCH 098/133] Add new test file e_blobclose.test, containing tests for sqlite3_blob_close(). FossilOrigin-Name: 5a1eac2419b1462e6f21595a3fff26d9cc49d203 --- manifest | 15 ++-- manifest.uuid | 2 +- src/sqlite.h.in | 30 ++++---- test/e_blobclose.test | 171 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 194 insertions(+), 24 deletions(-) create mode 100644 test/e_blobclose.test diff --git a/manifest b/manifest index 762ded40f6..089246851e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C New\stest\scases\sfor\sdeleting\scontent\sout\sfrom\sunder\sa\sSELECT\sstatement. -D 2014-11-10T19:16:59.654 +C Add\snew\stest\sfile\se_blobclose.test,\scontaining\stests\sfor\ssqlite3_blob_close(). +D 2014-11-11T12:20:35.141 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -230,7 +230,7 @@ F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c bc28d5992109717c87804e2eb1a08a7c8cc7a2fd -F src/sqlite.h.in 5531c4c69ee0a351c2aa5ad9f3f0f2424a57a9f4 +F src/sqlite.h.in 0e6612f84936cca29166f2c66068e0227a13fdf6 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 71b0bf1a7fc55b5cb374f7579fd140e730a6e0f4 @@ -453,6 +453,7 @@ F test/descidx3.test 09ddbe3f5295f482d2f8b687cf6db8bad7acd9a2 F test/diskfull.test 106391384780753ea6896b7b4f005d10e9866b6e F test/distinct.test 086e70c765f172e8974e9f83b9ac5ca03c154e77 F test/distinctagg.test 1a6ef9c87a58669438fc771450d7a72577417376 +F test/e_blobclose.test df756753f571bc30e42e3a6cba2807576e49e716 F test/e_blobopen.test 234f960d90235a9b51ec3ca1e062e8541dd558d8 F test/e_blobwrite.test 615b405f29feb2cfb5a1f03dab7933258294fa26 F test/e_changes.test fd66105385153dbf21fdb35eb8ef6c3e1eade579 @@ -1216,7 +1217,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 1df77e5f1bd82de4dc92fe28359c3e56ab3f9ed4 -R 4b8ad408c58c17649b0e1468e9b897aa -U drh -Z 1daa82bfd52bfa0c612ccd059afbb463 +P 8289c3e9b47f7c2a606a88839f6bf615f8904ac2 +R 5495040e5aa9cb71b6dcdaffde6bd935 +U dan +Z 0fca1fa0cec9033f254d39e73d650ca5 diff --git a/manifest.uuid b/manifest.uuid index 7f7352c14d..dbade0f26a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8289c3e9b47f7c2a606a88839f6bf615f8904ac2 \ No newline at end of file +5a1eac2419b1462e6f21595a3fff26d9cc49d203 \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index b5585defd6..2812bf511d 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -5754,24 +5754,22 @@ SQLITE_EXPERIMENTAL int sqlite3_blob_reopen(sqlite3_blob *, sqlite3_int64); /* ** CAPI3REF: Close A BLOB Handle ** -** ^Closes an open [BLOB handle]. +** ^This function closes an open [BLOB handle]. ^(The BLOB handle is closed +** unconditionally. Even if this routine returns an error code, the +** handle is still closed.)^ ** -** ^Closing a BLOB shall cause the current transaction to commit -** if there are no other BLOBs, no pending prepared statements, and the -** database connection is in [autocommit mode]. -** ^If any writes were made to the BLOB, they might be held in cache -** until the close operation if they will fit. +** ^If the blob handle being closed was opened for read-write access, and if +** the database is in auto-commit mode and there are no other open read-write +** blob handles or active write statements, the current transaction is +** committed. ^If an error occurs while committing the transaction, an error +** code is returned and the transaction rolled back. ** -** ^(Closing the BLOB often forces the changes -** out to disk and so if any I/O errors occur, they will likely occur -** at the time when the BLOB is closed. Any errors that occur during -** closing are reported as a non-zero return value.)^ -** -** ^(The BLOB is closed unconditionally. Even if this routine returns -** an error code, the BLOB is still closed.)^ -** -** ^Calling this routine with a null pointer (such as would be returned -** by a failed call to [sqlite3_blob_open()]) is a harmless no-op. +** Calling this function with an argument that is not a NULL pointer or an +** open blob handle results in undefined behaviour. ^Calling this routine +** with a null pointer (such as would be returned by a failed call to +** [sqlite3_blob_open()]) is a harmless no-op. ^Otherwise, if this function +** is passed a valid open blob handle, the values returned by the +** sqlite3_errcode() and sqlite3_errmsg() functions are set before returning. */ int sqlite3_blob_close(sqlite3_blob *); diff --git a/test/e_blobclose.test b/test/e_blobclose.test new file mode 100644 index 0000000000..a5d432d3b5 --- /dev/null +++ b/test/e_blobclose.test @@ -0,0 +1,171 @@ +# 2014 October 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix e_blobclose + +set dots [string repeat . 40] +do_execsql_test 1.0 { + CREATE TABLE x1(a INTEGER PRIMARY KEY, b DOTS); + INSERT INTO x1 VALUES(-1, $dots); + INSERT INTO x1 VALUES(-10, $dots); + INSERT INTO x1 VALUES(-100, $dots); + INSERT INTO x1 VALUES(-1000, $dots); + INSERT INTO x1 VALUES(-10000, $dots); +} + +# EVIDENCE-OF: R-03145-46390 This function closes an open BLOB handle. +# +# It's not clear how to test that a blob handle really is closed. +# Attempting to use a closed blob handle will likely crash the process. +# Assume here that if the SHARED lock on the db file is released, +# the blob handle has been closed. +# +do_execsql_test 1.1 { PRAGMA lock_status } {main unlocked temp closed} +sqlite3_blob_open db main x1 b -1 0 B +do_execsql_test 1.2 { PRAGMA lock_status } {main shared temp closed} +sqlite3_blob_close $B +do_execsql_test 1.3 { PRAGMA lock_status } {main unlocked temp closed} + + +# EVIDENCE-OF: R-34027-00617 If the blob handle being closed was opened +# for read-write access, and if the database is in auto-commit mode and +# there are no other open read-write blob handles or active write +# statements, the current transaction is committed. +# +# 2.1.*: Transaction is not committed if there are other open +# read-write blob handles. +# +# 2.2.*: Transaction is not committed if not in auto-commit mode. +# +# 2.3.*: Active write statements. +# +do_test 2.1.1 { + sqlite3_blob_open db main x1 b -100 1 B1 + sqlite3_blob_open db main x1 b -1000 1 B2 + sqlite3_blob_open db main x1 b -10000 1 B3 + sqlite3_blob_open db main x1 b -10000 0 B4 ;# B4 is read-only! + execsql { PRAGMA lock_status } +} {main reserved temp closed} +do_test 2.1.2 { + sqlite3_blob_close $B1 + execsql { PRAGMA lock_status } +} {main reserved temp closed} +do_test 2.1.3 { + sqlite3_blob_close $B2 + execsql { PRAGMA lock_status } +} {main reserved temp closed} +do_test 2.1.4 { + sqlite3_blob_close $B3 + execsql { PRAGMA lock_status } +} {main shared temp closed} +do_test 2.1.5 { + sqlite3_blob_close $B4 + execsql { PRAGMA lock_status } +} {main unlocked temp closed} + +do_test 2.2.1 { + sqlite3_blob_open db main x1 b -100 1 B1 + execsql { PRAGMA lock_status } +} {main reserved temp closed} +do_test 2.2.2 { + execsql { BEGIN } + sqlite3_blob_close $B1 + execsql { PRAGMA lock_status } +} {main reserved temp closed} +do_test 2.2.3 { + execsql { COMMIT } + execsql { PRAGMA lock_status } +} {main unlocked temp closed} + +proc val {} { + sqlite3_blob_close $::B + db eval { PRAGMA lock_status } +} +db func val val +do_test 2.3.1 { + sqlite3_blob_open db main x1 b -100 1 B + execsql { PRAGMA lock_status } +} {main reserved temp closed} +do_test 2.3.2 { + execsql { INSERT INTO x1 VALUES(15, val()) } + execsql { PRAGMA lock_status } +} {main unlocked temp closed} +do_test 2.3.3 { + execsql { SELECT * FROM x1 WHERE a = 15 } +} {15 {main reserved temp closed}} + +# A reader does not inhibit commit. +do_test 2.3.4 { + sqlite3_blob_open db main x1 b -100 1 B + execsql { PRAGMA lock_status } +} {main reserved temp closed} +do_test 2.3.5 { + execsql { SELECT a, val() FROM x1 LIMIT 1 } +} {-10000 {main shared temp closed}} + + +do_test 3.1 { + sqlite3_blob_open db main x1 b -10 1 B + execsql { + INSERT INTO x1 VALUES(1, 'abc'); + SELECT * FROM x1 WHERE a=1; + } +} {1 abc} +do_test 3.2 { + sqlite3_blob_write $B 0 "abcdefghij" 10 + execsql { SELECT * FROM x1 WHERE a=-10 } +} {-10 abcdefghij..............................} + +do_test 3.3 { + sqlite3 db2 test.db + execsql { BEGIN ; SELECT * FROM x1 } db2 + sqlite3_blob_close $B +} {SQLITE_BUSY} + +# EVIDENCE-OF: R-41959-38737 Otherwise, if this function is passed a +# valid open blob handle, the values returned by the sqlite3_errcode() +# and sqlite3_errmsg() functions are set before returning. +# +do_test 3.4 { + list [sqlite3_errcode db] [sqlite3_errmsg db] +} {SQLITE_BUSY {database is locked}} + +# EVIDENCE-OF: R-37801-37633 The BLOB handle is closed unconditionally. +# Even if this routine returns an error code, the handle is still +# closed. +# +# Test that the lock has been released. Assume this means the handle +# is closed, even though blob_close() returned SQLITE_BUSY. +# +do_execsql_test 3.4 { PRAGMA lock_status } {main unlocked temp closed} + +# EVIDENCE-OF: R-35111-05628 If an error occurs while committing the +# transaction, an error code is returned and the transaction rolled +# back. +# +# Row 1 is removed (it was inserted this transaction) and row -10 +# is restored to its original state. Transaction has been rolled back. +# +do_execsql_test 3.5 { + SELECT * FROM x1 WHERE a IN (1, -10); +} {-10 ........................................} + +# EVIDENCE-OF: R-25894-51060 Calling this routine with a null pointer +# (such as would be returned by a failed call to sqlite3_blob_open()) is +# a harmless no-op. +# +do_test 4.0 { sqlite3_blob_close 0 } {} + +finish_test + From 2d8e3caa2e54c4856db48998ba71a2c56b47e956 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 11 Nov 2014 16:11:04 +0000 Subject: [PATCH 099/133] Add tests for sqlite3_blob_bytes(). FossilOrigin-Name: a066a3832a7c6de65c3016e77e49ac00e09db749 --- manifest | 14 ++++---- manifest.uuid | 2 +- test/e_blobbytes.test | 76 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 84 insertions(+), 8 deletions(-) create mode 100644 test/e_blobbytes.test diff --git a/manifest b/manifest index 51c9f2ee01..c994394f6f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Permit\sread\soperations\sto\scontinue\safter\sa\sROLLBACK\sas\slong\sas\sthe\sschema\ndoes\snot\schange. -D 2014-11-11T14:59:31.172 +C Add\stests\sfor\ssqlite3_blob_bytes(). +D 2014-11-11T16:11:04.911 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -453,6 +453,7 @@ F test/descidx3.test 09ddbe3f5295f482d2f8b687cf6db8bad7acd9a2 F test/diskfull.test 106391384780753ea6896b7b4f005d10e9866b6e F test/distinct.test 086e70c765f172e8974e9f83b9ac5ca03c154e77 F test/distinctagg.test 1a6ef9c87a58669438fc771450d7a72577417376 +F test/e_blobbytes.test 9bea1d3e2b20f3010b04abba58f6ba172301f49f F test/e_blobclose.test df756753f571bc30e42e3a6cba2807576e49e716 F test/e_blobopen.test 234f960d90235a9b51ec3ca1e062e8541dd558d8 F test/e_blobwrite.test 615b405f29feb2cfb5a1f03dab7933258294fa26 @@ -1217,8 +1218,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 5a1eac2419b1462e6f21595a3fff26d9cc49d203 fa6e6a9ae276cad60e9a4abc1bc23cf2809ea786 -R 90c0902351a625ca12308516be116ef0 -T +closed fa6e6a9ae276cad60e9a4abc1bc23cf2809ea786 -U drh -Z 3e9637efba5697b65ccb54a130a31d2a +P b5df5ac0529f7b0d1e880a7f4a307e7d77b7fa6c +R be9d88d05dd4414f44a05008ae6eba4c +U dan +Z 2fc1cd52a3184d003db0d33d71426d71 diff --git a/manifest.uuid b/manifest.uuid index 699edcd220..0982b7b00f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b5df5ac0529f7b0d1e880a7f4a307e7d77b7fa6c \ No newline at end of file +a066a3832a7c6de65c3016e77e49ac00e09db749 \ No newline at end of file diff --git a/test/e_blobbytes.test b/test/e_blobbytes.test new file mode 100644 index 0000000000..a6283ab852 --- /dev/null +++ b/test/e_blobbytes.test @@ -0,0 +1,76 @@ +# 2014 October 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix e_blobbytes + +do_execsql_test 1.0 { + CREATE TABLE q1(r INTEGER PRIMARY KEY, s TEXT); + WITH d(a, b) AS ( + SELECT 0, '' + UNION ALL + SELECT a+1, b||'.' FROM d WHERE a<10000 + ) + INSERT INTO q1 SELECT * FROM d; +} + + +# EVIDENCE-OF: R-07796-55423 Returns the size in bytes of the BLOB +# accessible via the successfully opened BLOB handle in its only +# argument. +# +proc check_blob_size {tn rowid bytes} { + uplevel [list do_test $tn [subst -nocommands { + sqlite3_blob_open db main q1 s $rowid 0 B + set res [sqlite3_blob_bytes [set B]] + sqlite3_blob_close [set B] + set res + }] $bytes] +} +check_blob_size 1.1 43 43 +check_blob_size 1.2 391 391 +check_blob_size 1.3 6349 6349 +check_blob_size 1.4 2621 2621 +check_blob_size 1.5 7771 7771 +check_blob_size 1.6 7949 7949 +check_blob_size 1.7 4374 4374 +check_blob_size 1.8 2578 2578 +check_blob_size 1.9 7004 7004 +check_blob_size 1.10 2180 2180 +check_blob_size 1.11 3796 3796 +check_blob_size 1.12 7101 7101 +check_blob_size 1.13 7449 7449 +check_blob_size 1.14 7224 7224 +check_blob_size 1.15 3038 3038 +check_blob_size 1.16 1083 1083 +check_blob_size 1.17 5157 5157 +check_blob_size 1.18 6686 6686 +check_blob_size 1.19 6592 6592 +check_blob_size 1.20 0 0 + + +# EVIDENCE-OF: R-53088-19343 The incremental blob I/O routines can only +# read or overwriting existing blob content; they cannot change the size +# of a blob. +# +# Also demonstrated in other e_blobXXX.test files. +# +do_test 2.1 { + sqlite3_blob_open db main q1 s 86 1 B + list [catch { sqlite3_blob_write $B 86 "1" 1 } msg] $msg +} {1 SQLITE_ERROR} +sqlite3_blob_close $B + +finish_test + + From bfa395d085fbadc6436bb4baf40e96c384c64463 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Tue, 11 Nov 2014 19:07:56 +0000 Subject: [PATCH 100/133] Remove some calls to the 'breakpoint' test command. FossilOrigin-Name: 1412fcc480799ecbd68d44dd18d5bad40e20ccf1 --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- test/capi3d.test | 1 - test/corruptH.test | 1 - test/sort2.test | 1 - test/tkt-f777251dc7a.test | 1 - 6 files changed, 11 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index c994394f6f..97192e581d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stests\sfor\ssqlite3_blob_bytes(). -D 2014-11-11T16:11:04.911 +C Remove\ssome\scalls\sto\sthe\s'breakpoint'\stest\scommand. +D 2014-11-11T19:07:56.372 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -384,7 +384,7 @@ F test/capi2.test 011c16da245fdc0106a2785035de6b242c05e738 F test/capi3.test f0718f4f90d0efdc980119bfbdf1d7f1541ee5ef F test/capi3b.test efb2b9cfd127efa84433cd7a2d72ce0454ae0dc4 F test/capi3c.test fdc0d67a2cb8e8fc400d5b7735e330161ea057a2 -F test/capi3d.test c84af0c49267f9c3fbf4c1c46aa647646023811e +F test/capi3d.test a82b6321c50a1cfc848e386fa2c851893606f68c F test/capi3e.test 3d49c01ef2a1a55f41d73cba2b23b5059ec460fe F test/cast.test 4c275cbdc8202d6f9c54a3596701719868ac7dc3 F test/check.test 5831ddb6f2c687782eaf2e1a07b6e17f24c4f763 @@ -423,7 +423,7 @@ F test/corruptD.test b3c205fac7952b1de645ce44bb02335cd9e3e040 F test/corruptE.test 193b4ca4e927e77c1d5f4f56203ddc998432a7ee F test/corruptF.test be9fde98e4c93648f1ba52b74e5318edc8f59fe4 F test/corruptG.test 1ab3bf97ee7bdba70e0ff3ba2320657df55d1804 -F test/corruptH.test 88ed71a086e13591c917aac6de32750e7c7281cb +F test/corruptH.test 5dd4fa98c6c1ed33b178f9e8a48c4fdd3cfc9067 F test/corruptI.test 221ad8b7f0a9ac6b80fc577e73b5ad8cdea31243 F test/cost.test 19d314526616ce4473eb4e4e450fcb94499ce318 F test/count.test 42a251178e32f617eda33f76236a7f79825a50b5 @@ -860,7 +860,7 @@ F test/skipscan6.test 5866039d03a56f5bd0b3d172a012074a1d90a15b F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24 F test/sort.test c4400e7533748f6bd7413851ff148645e82b9e2d -F test/sort2.test 269f4f50c6e468cc32b302ae7ff0add8338ec6de +F test/sort2.test 84a92eebf697feee9c6697746918c7d67373eea1 F test/sort3.test 6178ade30810ac9166fcdf14b7065e49c0f534e2 F test/sort4.test 6c37d85f7cd28d50cce222fcab84ccd771e105cb F test/sort5.test a448240a42b49239edc00f85d6d7ac7a1b261e1f @@ -956,7 +956,7 @@ F test/tkt-d635236375.test 9d37e988b47d87505bc9445be0ca447002df5d09 F test/tkt-d82e3f3721.test bcc0dfba658d15bab30fd4a9320c9e35d214ce30 F test/tkt-f3e5abed55.test d5a0126118142d13e27f6ce9f4c47096e9321c00 F test/tkt-f67b41381a.test a23bc124c981662db712167bacd0ed8ad11abac9 -F test/tkt-f777251dc7a.test 6295d235a03c82160549da4841a83dc8e758932f +F test/tkt-f777251dc7a.test d1a8fc3eefb7a9e64d19ff24d5c8c94c34a632fb F test/tkt-f7b4edec.test d998a08ff2b18b7f62edce8e3044317c45efe6c7 F test/tkt-f973c7ac31.test 28ef85c7f015477916795246d8286aeda39d4ead F test/tkt-fa7bf5ec.test 9102dfea58aa371d78969da735f9392c57e2e035 @@ -1218,7 +1218,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P b5df5ac0529f7b0d1e880a7f4a307e7d77b7fa6c -R be9d88d05dd4414f44a05008ae6eba4c -U dan -Z 2fc1cd52a3184d003db0d33d71426d71 +P a066a3832a7c6de65c3016e77e49ac00e09db749 +R e8fc41a7f82a02ab8618a341f7ef7735 +U mistachkin +Z 90417f4c358533080e79b8017b4e7453 diff --git a/manifest.uuid b/manifest.uuid index 0982b7b00f..f90bf2d8c8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a066a3832a7c6de65c3016e77e49ac00e09db749 \ No newline at end of file +1412fcc480799ecbd68d44dd18d5bad40e20ccf1 \ No newline at end of file diff --git a/test/capi3d.test b/test/capi3d.test index fb8abe86d2..1459c5abfe 100644 --- a/test/capi3d.test +++ b/test/capi3d.test @@ -155,7 +155,6 @@ do_execsql_test capi3d-4.1 { } do_test capi3d-4.2.1 { - breakpoint set ::s1 [sqlite3_prepare_v2 db "ROLLBACK" -1 notused] sqlite3_step $::s1 } {SQLITE_DONE} diff --git a/test/corruptH.test b/test/corruptH.test index ee2bb1ee48..5c83cb3b90 100644 --- a/test/corruptH.test +++ b/test/corruptH.test @@ -64,7 +64,6 @@ do_test 1.2 { } {} do_test 1.3 { -breakpoint db eval { PRAGMA secure_delete=1 } list [catch { db eval { SELECT * FROM t1 WHERE a IN (1, 2) } { diff --git a/test/sort2.test b/test/sort2.test index 29001f0099..a4c55c9f24 100644 --- a/test/sort2.test +++ b/test/sort2.test @@ -62,7 +62,6 @@ foreach {tn script} { do_execsql_test $tn.2.4 { PRAGMA integrity_check } {ok} - breakpoint do_execsql_test $tn.3 { PRAGMA cache_size = 5; WITH r(x,y) AS ( diff --git a/test/tkt-f777251dc7a.test b/test/tkt-f777251dc7a.test index f814d246bf..b91e438da5 100644 --- a/test/tkt-f777251dc7a.test +++ b/test/tkt-f777251dc7a.test @@ -38,7 +38,6 @@ proc force_rollback {} { db function force_rollback force_rollback do_test tkt-f7772-1.2 { -breakpoint catchsql { BEGIN IMMEDIATE; CREATE TABLE xyzzy(abc); From 85fabf1444ff49f7f1e2060d0b93d9df6f8f9f6f Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 11 Nov 2014 22:55:26 +0000 Subject: [PATCH 101/133] This is a cherry-pick of version [b5df5ac052]. FossilOrigin-Name: d4b2d5d066891e06f2bf4337902b44b000fa9fd2 --- manifest | 40 ++++++++++++++++------------------ manifest.uuid | 2 +- src/backup.c | 2 +- src/btree.c | 46 ++++++++++++++++++++++----------------- src/btree.h | 4 ++-- src/main.c | 9 +++++--- src/vdbe.c | 11 +++++++--- src/wal.c | 3 +-- test/capi3.test | 6 ++--- test/capi3c.test | 6 ++--- test/rollback.test | 4 ++-- test/savepoint.test | 2 +- test/savepoint7.test | 6 +++-- test/tkt-f777251dc7a.test | 4 +++- test/trans3.test | 2 +- 15 files changed, 81 insertions(+), 66 deletions(-) diff --git a/manifest b/manifest index 8688f8549f..0e3e6a9ff4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Version\s3.8.7.1 -D 2014-10-29T13:59:56.070 +C This\sis\sa\scherry-pick\sof\sversion\s[b5df5ac052]. +D 2014-11-11T22:55:26.891 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -169,11 +169,11 @@ F src/alter.c ba266a779bc7ce10e52e59e7d3dc79fa342e8fdb F src/analyze.c 8c322e1ecc08909526dbd5ab4421889d05f2263d F src/attach.c f4e94df2d1826feda65eb0939f7f6f5f923a0ad9 F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 -F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e +F src/backup.c 8cdfeb0c8a6d8bdad3faefae418eb3dc767051b6 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 1b1123cba0c65caa0baa51e71b8c089e3167c3ed -F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 +F src/btree.c 944b84f72fd9048acde31400d89439976bf766b0 +F src/btree.h 97c4d0ef860f151e096e97eef78b87f7b4a8d899 F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 9dc2bd94347b878c89627000c92b0c8d97ec2919 F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 @@ -194,7 +194,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 F src/loadext.c de741e66e5ddc1598d904d7289239696e40ed994 -F src/main.c bbe872b0ac0007bed0ebe1936fc493b039ad4f51 +F src/main.c 1bdabb62205af168498a17460bdb7533b2a4a915 F src/malloc.c 3c3ac67969612493d435e14b6832793209afd2ec F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c faf615aafd8be74a71494dfa027c113ea5c6615f @@ -289,7 +289,7 @@ F src/update.c 3c4ecc282accf12d39edb8d524cf089645e55a13 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 4006c01772bd8d8ac4306d523bbcee41d3e392d8 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c 5a1afb571853ddb911d698ac996bc4fd8ddf1eed +F src/vdbe.c 4c77cdf16be330bd5227691919332b42d557e211 F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 F src/vdbeInt.h e2a060a55ee18a6ab973353a5e2ec7ee569bf787 F src/vdbeapi.c 37a6c6ae284a97bcace365f2f0a225680c0499d9 @@ -299,7 +299,7 @@ F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f F src/vdbesort.c 975aeffa99acb0991b2f288d30294756bff41438 F src/vdbetrace.c 7e4222955e07dd707a2f360c0eb73452be1cb010 F src/vtab.c cb0c194303fea276b48d7d4b6d970b5a96bde8de -F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 +F src/wal.c 73051f1222321712fa4280c495780ba81d302dad F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 F src/where.c 2947912f1f3d6a7766fe087fd532a5d688d745b1 @@ -378,9 +378,9 @@ F test/btreefault.test c2bcb542685eea44621275cfedbd8a13f65201e3 F test/busy.test 76b4887f8b9160ba903c1ac22e8ff406ad6ae2f0 F test/cache.test 13bc046b26210471ca6f2889aceb1ea52dc717de F test/capi2.test 011c16da245fdc0106a2785035de6b242c05e738 -F test/capi3.test 71bcf2fbd36a9732f617766dfd752552c8e491b5 +F test/capi3.test f0718f4f90d0efdc980119bfbdf1d7f1541ee5ef F test/capi3b.test efb2b9cfd127efa84433cd7a2d72ce0454ae0dc4 -F test/capi3c.test a21869e4d50d5dbb7e566e328fc0bc7c2efa6a32 +F test/capi3c.test fdc0d67a2cb8e8fc400d5b7735e330161ea057a2 F test/capi3d.test c84af0c49267f9c3fbf4c1c46aa647646023811e F test/capi3e.test 3d49c01ef2a1a55f41d73cba2b23b5059ec460fe F test/cast.test 4c275cbdc8202d6f9c54a3596701719868ac7dc3 @@ -785,18 +785,18 @@ F test/reindex.test 44edd3966b474468b823d481eafef0c305022254 F test/releasetest.mk 2eced2f9ae701fd0a29e714a241760503ccba25a F test/releasetest.tcl a4279c890698584feb2ffc86735857a4e4474180 F test/resolver01.test 33abf37ff8335e6bf98f2b45a0af3e06996ccd9a -F test/rollback.test e9504a009a202c3ed711da2e6879ff60c5a4669c +F test/rollback.test 458fe73eb3ffdfdf9f6ba3e9b7350a6220414dea F test/rowhash.test 0bc1d31415e4575d10cacf31e1a66b5cc0f8be81 F test/rowid.test 742b5741584a8a44fd83e856cc2896688401d645 F test/rtree.test 0c8d9dd458d6824e59683c19ab2ffa9ef946f798 F test/run-wordcount.sh 891e89c4c2d16e629cd45951d4ed899ad12afc09 -F test/savepoint.test 51d3900dc071a7c2ad4248578a5925631b476313 +F test/savepoint.test c671fdbd34cd3bfe1518a777526ada595180cf8d F test/savepoint2.test 9b8543940572a2f01a18298c3135ad0c9f4f67d7 F test/savepoint3.test e328085853b14898d78ceea00dfe7db18bb6a9ec F test/savepoint4.test c8f8159ade6d2acd9128be61e1230f1c1edc6cc0 F test/savepoint5.test 0735db177e0ebbaedc39812c8d065075d563c4fd F test/savepoint6.test f41279c5e137139fa5c21485773332c7adb98cd7 -F test/savepoint7.test fbf319a7b2dda089ec5be30a424a0e95f121d423 +F test/savepoint7.test 1c8f26b1e2a4221b0214e222ce12a97a59918eb2 F test/schema.test 8f7999be894260f151adf15c2c7540f1c6d6a481 F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5 F test/schema3.test 1bc1008e1f8cb5654b248c55f27249366eb7ed38 @@ -943,7 +943,7 @@ F test/tkt-d635236375.test 9d37e988b47d87505bc9445be0ca447002df5d09 F test/tkt-d82e3f3721.test bcc0dfba658d15bab30fd4a9320c9e35d214ce30 F test/tkt-f3e5abed55.test d5a0126118142d13e27f6ce9f4c47096e9321c00 F test/tkt-f67b41381a.test a23bc124c981662db712167bacd0ed8ad11abac9 -F test/tkt-f777251dc7a.test af6531446c64bfd268416f07b4df7be7f9c749d2 +F test/tkt-f777251dc7a.test 6295d235a03c82160549da4841a83dc8e758932f F test/tkt-f7b4edec.test d998a08ff2b18b7f62edce8e3044317c45efe6c7 F test/tkt-f973c7ac31.test 28ef85c7f015477916795246d8286aeda39d4ead F test/tkt-fa7bf5ec.test 9102dfea58aa371d78969da735f9392c57e2e035 @@ -1038,7 +1038,7 @@ F test/trace.test 73a5508100f7fccfbc3f8018d5f6963ed478eea0 F test/trace2.test 93b47ca6996c66b47f57224cfb146f34e07df382 F test/trans.test 6e1b4c6a42dba31bd65f8fa5e61a2708e08ddde6 F test/trans2.test 62bd045bfc7a1c14c5ba83ba64d21ade31583f76 -F test/trans3.test 373ac5183cc56be69f48ae44090e7f672939f732 +F test/trans3.test 91a100e5412b488e22a655fe423a14c26403ab94 F test/transitive1.test 03f532954f46cdf5608f7766bff0b0c52bf2a7cd F test/trigger1.test dc47573ac79ffe0ee3eecaa517d70d8dacbccd03 F test/trigger2.test 5cd7d69a7ba1143ee045e4ae2963ff32ae4c87a6 @@ -1204,10 +1204,8 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 83afe23e553e802c0947c80d0ffdd120423e7c52 -R 5381895dceb9bc7ad4d425f399561cb7 -T +bgcolor * #d0c0ff -T +sym-release * -T +sym-version-3.8.7.1 * +P 3b7b72c4685aa5cf5e675c2c47ebec10d9704221 +Q +b5df5ac0529f7b0d1e880a7f4a307e7d77b7fa6c +R 27a95de33658c36b59b93f043d249813 U drh -Z e5ce942eb65e60b92fcaefcb5b84039b +Z ea313ee60fabf91291f912ad6dcdc285 diff --git a/manifest.uuid b/manifest.uuid index 1787d659c6..983840bd0e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3b7b72c4685aa5cf5e675c2c47ebec10d9704221 \ No newline at end of file +d4b2d5d066891e06f2bf4337902b44b000fa9fd2 \ No newline at end of file diff --git a/src/backup.c b/src/backup.c index 92c6334bde..9fcb669106 100644 --- a/src/backup.c +++ b/src/backup.c @@ -597,7 +597,7 @@ int sqlite3_backup_finish(sqlite3_backup *p){ } /* If a transaction is still open on the Btree, roll it back. */ - sqlite3BtreeRollback(p->pDest, SQLITE_OK); + sqlite3BtreeRollback(p->pDest, SQLITE_OK, 0); /* Set the error code of the destination database handle. */ rc = (p->rc==SQLITE_DONE) ? SQLITE_OK : p->rc; diff --git a/src/btree.c b/src/btree.c index 758dfe6335..9d7a05157d 100644 --- a/src/btree.c +++ b/src/btree.c @@ -2171,7 +2171,7 @@ int sqlite3BtreeClose(Btree *p){ ** The call to sqlite3BtreeRollback() drops any table-locks held by ** this handle. */ - sqlite3BtreeRollback(p, SQLITE_OK); + sqlite3BtreeRollback(p, SQLITE_OK, 0); sqlite3BtreeLeave(p); /* If there are still other outstanding references to the shared-btree @@ -3464,27 +3464,28 @@ int sqlite3BtreeCommit(Btree *p){ /* ** This routine sets the state to CURSOR_FAULT and the error -** code to errCode for every cursor on BtShared that pBtree -** references. +** code to errCode for every cursor on any BtShared that pBtree +** references. Or if the writeOnly flag is set to 1, then only +** trip write cursors and leave read cursors unchanged. ** -** Every cursor is tripped, including cursors that belong -** to other database connections that happen to be sharing -** the cache with pBtree. +** Every cursor is a candidate to be tripped, including cursors +** that belong to other database connections that happen to be +** sharing the cache with pBtree. ** -** This routine gets called when a rollback occurs. -** All cursors using the same cache must be tripped -** to prevent them from trying to use the btree after -** the rollback. The rollback may have deleted tables -** or moved root pages, so it is not sufficient to -** save the state of the cursor. The cursor must be -** invalidated. +** This routine gets called when a rollback occurs. The writeOnly +** flag is set to 1 if the transaction did not make any schema +** changes, in which case the read cursors can continue operating. +** If schema changes did occur in the transaction, then both read +** and write cursors must both be tripped. */ -void sqlite3BtreeTripAllCursors(Btree *pBtree, int errCode){ +void sqlite3BtreeTripAllCursors(Btree *pBtree, int errCode, int writeOnly){ BtCursor *p; + assert( (writeOnly==0 || writeOnly==1) && BTCF_WriteFlag==1 ); if( pBtree==0 ) return; sqlite3BtreeEnter(pBtree); for(p=pBtree->pBt->pCursor; p; p=p->pNext){ int i; + if( writeOnly && (p->curFlags & BTCF_WriteFlag)==0 ) continue; sqlite3BtreeClearCursor(p); p->eState = CURSOR_FAULT; p->skipNext = errCode; @@ -3497,27 +3498,32 @@ void sqlite3BtreeTripAllCursors(Btree *pBtree, int errCode){ } /* -** Rollback the transaction in progress. All cursors will be -** invalided by this operation. Any attempt to use a cursor -** that was open at the beginning of this operation will result -** in an error. +** Rollback the transaction in progress. +** +** If tripCode is not SQLITE_OK then cursors will be invalidated (tripped). +** Only write cursors are tripped if writeOnly is true but all cursors are +** tripped if writeOnly is false. Any attempt to use +** a tripped cursor will result in an error. ** ** This will release the write lock on the database file. If there ** are no active cursors, it also releases the read lock. */ -int sqlite3BtreeRollback(Btree *p, int tripCode){ +int sqlite3BtreeRollback(Btree *p, int tripCode, int writeOnly){ int rc; BtShared *pBt = p->pBt; MemPage *pPage1; + assert( writeOnly==1 || writeOnly==0 ); + assert( tripCode==SQLITE_ABORT_ROLLBACK || tripCode==SQLITE_OK ); sqlite3BtreeEnter(p); if( tripCode==SQLITE_OK ){ rc = tripCode = saveAllCursors(pBt, 0, 0); + if( rc ) writeOnly = 0; }else{ rc = SQLITE_OK; } if( tripCode ){ - sqlite3BtreeTripAllCursors(p, tripCode); + sqlite3BtreeTripAllCursors(p, tripCode, writeOnly); } btreeIntegrity(p); diff --git a/src/btree.h b/src/btree.h index 38abdca1a2..f724269a3e 100644 --- a/src/btree.h +++ b/src/btree.h @@ -83,7 +83,7 @@ int sqlite3BtreeBeginTrans(Btree*,int); int sqlite3BtreeCommitPhaseOne(Btree*, const char *zMaster); int sqlite3BtreeCommitPhaseTwo(Btree*, int); int sqlite3BtreeCommit(Btree*); -int sqlite3BtreeRollback(Btree*,int); +int sqlite3BtreeRollback(Btree*,int,int); int sqlite3BtreeBeginStmt(Btree*,int); int sqlite3BtreeCreateTable(Btree*, int*, int flags); int sqlite3BtreeIsInTrans(Btree*); @@ -116,7 +116,7 @@ int sqlite3BtreeIncrVacuum(Btree *); int sqlite3BtreeDropTable(Btree*, int, int*); int sqlite3BtreeClearTable(Btree*, int, int*); int sqlite3BtreeClearTableOfCursor(BtCursor*); -void sqlite3BtreeTripAllCursors(Btree*, int); +void sqlite3BtreeTripAllCursors(Btree*, int, int); void sqlite3BtreeGetMeta(Btree *pBtree, int idx, u32 *pValue); int sqlite3BtreeUpdateMeta(Btree*, int idx, u32 value); diff --git a/src/main.c b/src/main.c index ea03f2639f..46b3ccdf58 100644 --- a/src/main.c +++ b/src/main.c @@ -1016,13 +1016,15 @@ void sqlite3LeaveMutexAndCloseZombie(sqlite3 *db){ /* ** Rollback all database files. If tripCode is not SQLITE_OK, then -** any open cursors are invalidated ("tripped" - as in "tripping a circuit +** any write cursors are invalidated ("tripped" - as in "tripping a circuit ** breaker") and made to return tripCode if there are any further -** attempts to use that cursor. +** attempts to use that cursor. Read cursors remain open and valid +** but are "saved" in case the table pages are moved around. */ void sqlite3RollbackAll(sqlite3 *db, int tripCode){ int i; int inTrans = 0; + int schemaChange; assert( sqlite3_mutex_held(db->mutex) ); sqlite3BeginBenignMalloc(); @@ -1033,6 +1035,7 @@ void sqlite3RollbackAll(sqlite3 *db, int tripCode){ ** the database rollback and schema reset, which can cause false ** corruption reports in some cases. */ sqlite3BtreeEnterAll(db); + schemaChange = (db->flags & SQLITE_InternChanges)!=0 && db->init.busy==0; for(i=0; inDb; i++){ Btree *p = db->aDb[i].pBt; @@ -1040,7 +1043,7 @@ void sqlite3RollbackAll(sqlite3 *db, int tripCode){ if( sqlite3BtreeIsInTrans(p) ){ inTrans = 1; } - sqlite3BtreeRollback(p, tripCode); + sqlite3BtreeRollback(p, tripCode, !schemaChange); } } sqlite3VtabRollback(db); diff --git a/src/vdbe.c b/src/vdbe.c index 88fadb023e..6e152cd27e 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2822,11 +2822,16 @@ case OP_Savepoint: { db->isTransactionSavepoint = 0; rc = p->rc; }else{ + int isSchemaChange; iSavepoint = db->nSavepoint - iSavepoint - 1; if( p1==SAVEPOINT_ROLLBACK ){ + isSchemaChange = (db->flags & SQLITE_InternChanges)!=0; for(ii=0; iinDb; ii++){ - sqlite3BtreeTripAllCursors(db->aDb[ii].pBt, SQLITE_ABORT); + sqlite3BtreeTripAllCursors(db->aDb[ii].pBt, SQLITE_ABORT, + isSchemaChange==0); } + }else{ + isSchemaChange = 0; } for(ii=0; iinDb; ii++){ rc = sqlite3BtreeSavepoint(db->aDb[ii].pBt, p1, iSavepoint); @@ -2834,7 +2839,7 @@ case OP_Savepoint: { goto abort_due_to_error; } } - if( p1==SAVEPOINT_ROLLBACK && (db->flags&SQLITE_InternChanges)!=0 ){ + if( isSchemaChange ){ sqlite3ExpirePreparedStatements(db); sqlite3ResetAllSchemasOfConnection(db); db->flags = (db->flags | SQLITE_InternChanges); @@ -3231,7 +3236,7 @@ case OP_OpenWrite: { || p->readOnly==0 ); if( p->expired ){ - rc = SQLITE_ABORT; + rc = SQLITE_ABORT_ROLLBACK; break; } diff --git a/src/wal.c b/src/wal.c index c0861d5be7..24540a2cd5 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2506,7 +2506,7 @@ int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx){ memcpy(&pWal->hdr, (void *)walIndexHdr(pWal), sizeof(WalIndexHdr)); for(iFrame=pWal->hdr.mxFrame+1; - ALWAYS(rc==SQLITE_OK) && iFrame<=iMax; + rc==SQLITE_OK && iFrame<=iMax; iFrame++ ){ /* This call cannot fail. Unless the page for which the page number @@ -2525,7 +2525,6 @@ int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx){ } if( iMax!=pWal->hdr.mxFrame ) walCleanupHash(pWal); } - assert( rc==SQLITE_OK ); return rc; } diff --git a/test/capi3.test b/test/capi3.test index cbaa5c5e5b..9f3d6f6916 100644 --- a/test/capi3.test +++ b/test/capi3.test @@ -912,7 +912,7 @@ do_test capi3-11.9.3 { } 1 do_test capi3-11.10 { sqlite3_step $STMT -} {SQLITE_ERROR} +} {SQLITE_ROW} ifcapable !autoreset { # If SQLITE_OMIT_AUTORESET is defined, then the statement must be # reset() before it can be passed to step() again. @@ -921,11 +921,11 @@ ifcapable !autoreset { } do_test capi3-11.11 { sqlite3_step $STMT -} {SQLITE_ROW} +} {SQLITE_DONE} do_test capi3-11.12 { sqlite3_step $STMT sqlite3_step $STMT -} {SQLITE_DONE} +} {SQLITE_ROW} do_test capi3-11.13 { sqlite3_finalize $STMT } {SQLITE_OK} diff --git a/test/capi3c.test b/test/capi3c.test index 6388717e00..6ab3bc24f6 100644 --- a/test/capi3c.test +++ b/test/capi3c.test @@ -863,7 +863,7 @@ do_test capi3c-11.9.3 { } 1 do_test capi3c-11.10 { sqlite3_step $STMT -} {SQLITE_ABORT} +} {SQLITE_ROW} ifcapable !autoreset { # If SQLITE_OMIT_AUTORESET is defined, then the statement must be # reset() before it can be passed to step() again. @@ -872,11 +872,11 @@ ifcapable !autoreset { } do_test capi3c-11.11 { sqlite3_step $STMT -} {SQLITE_ROW} +} {SQLITE_DONE} do_test capi3c-11.12 { sqlite3_step $STMT sqlite3_step $STMT -} {SQLITE_DONE} +} {SQLITE_ROW} do_test capi3c-11.13 { sqlite3_finalize $STMT } {SQLITE_OK} diff --git a/test/rollback.test b/test/rollback.test index c339c5d7d6..7abafece61 100644 --- a/test/rollback.test +++ b/test/rollback.test @@ -60,11 +60,11 @@ ifcapable conflict { # do_test rollback-1.5 { sqlite3_step $STMT - } {SQLITE_ERROR} + } {SQLITE_ROW} # Restart the SELECT statement # - do_test rollback-1.6 { sqlite3_reset $STMT } {SQLITE_ABORT} + do_test rollback-1.6 { sqlite3_reset $STMT } {SQLITE_OK} } else { do_test rollback-1.6 { sqlite3_reset $STMT } {SQLITE_OK} } diff --git a/test/savepoint.test b/test/savepoint.test index 9362c8fe19..8055e61d9e 100644 --- a/test/savepoint.test +++ b/test/savepoint.test @@ -315,7 +315,7 @@ ifcapable incrblob { do_test savepoint-5.3.2.3 { set rc [catch {seek $fd 0; read $fd} res] set rc - } {1} + } {0} do_test savepoint-5.3.3 { catchsql {RELEASE def} } {0 {}} diff --git a/test/savepoint7.test b/test/savepoint7.test index bc99187d27..908ec571f5 100644 --- a/test/savepoint7.test +++ b/test/savepoint7.test @@ -30,6 +30,7 @@ do_test savepoint7-1.1 { db eval {SELECT * FROM t1} { db eval { SAVEPOINT x2; + CREATE TABLE IF NOT EXISTS t3(xyz); INSERT INTO t2 VALUES($a,$b,$c); RELEASE x2; } @@ -46,7 +47,7 @@ do_test savepoint7-1.2 { RELEASE x2; } } - db eval {SELECT * FROM t2} + db eval {SELECT * FROM t2;} } {1 2 3 4 5 6 7 8 9} do_test savepoint7-1.3 { @@ -65,7 +66,7 @@ do_test savepoint7-1.3 { # queries in outer contexts. # do_test savepoint7-2.1 { - db eval {DELETE FROM t2; SAVEPOINT x1;} + db eval {DELETE FROM t2; SAVEPOINT x1; CREATE TABLE t4(abc);} set rc [catch { db eval {SELECT * FROM t1} { db eval { @@ -85,6 +86,7 @@ do_test savepoint7-2.2 { db eval {SELECT * FROM t1} { db eval { SAVEPOINT x2; + CREATE TABLE t5(pqr); INSERT INTO t2 VALUES($a,$b,$c); ROLLBACK TO x2; } diff --git a/test/tkt-f777251dc7a.test b/test/tkt-f777251dc7a.test index af6f71ad96..f814d246bf 100644 --- a/test/tkt-f777251dc7a.test +++ b/test/tkt-f777251dc7a.test @@ -38,8 +38,10 @@ proc force_rollback {} { db function force_rollback force_rollback do_test tkt-f7772-1.2 { +breakpoint catchsql { BEGIN IMMEDIATE; + CREATE TABLE xyzzy(abc); SELECT x, force_rollback(), EXISTS(SELECT 1 FROM t3 WHERE w=x) FROM t2; } } {1 {abort due to ROLLBACK}} @@ -67,7 +69,7 @@ do_test tkt-f7772-2.2 { catchsql { SELECT x, force_rollback(), EXISTS(SELECT 1 FROM t3 WHERE w=x) FROM t2 } -} {1 {callback requested query abort}} +} {1 {abort due to ROLLBACK}} do_test tkt-f7772-2.3 { sqlite3_get_autocommit db } {1} diff --git a/test/trans3.test b/test/trans3.test index d5b316bec8..e828442415 100644 --- a/test/trans3.test +++ b/test/trans3.test @@ -52,7 +52,7 @@ do_test trans3-1.4 { db eval {SELECT * FROM t1} } {1 2 3 4} do_test trans3-1.5 { - db eval BEGIN + db eval {BEGIN; CREATE TABLE xyzzy(abc);} db eval {INSERT INTO t1 VALUES(5);} set ::ecode {} set x [catch { From 51a205410c5039849bc3179a6a1b1c136494f4da Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 12 Nov 2014 14:07:28 +0000 Subject: [PATCH 102/133] Make sure that NULL results from OP_Column are fully and completely NULL and do not have the MEM_Ephem bit set. Fix for ticket [094d39a4c95ee4]. FossilOrigin-Name: e1017745e183f5d7429ce787ec2feef946a24f0f --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/vdbe.c | 2 +- test/table.test | 18 +++++++++++++++++- 4 files changed, 27 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 0e3e6a9ff4..e652361975 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C This\sis\sa\scherry-pick\sof\sversion\s[b5df5ac052]. -D 2014-11-11T22:55:26.891 +C Make\ssure\sthat\sNULL\sresults\sfrom\sOP_Column\sare\sfully\sand\scompletely\sNULL\sand\sdo\snot\shave\sthe\sMEM_Ephem\sbit\sset.\sFix\sfor\sticket\s[094d39a4c95ee4]. +D 2014-11-12T14:07:28.525 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/update.c 3c4ecc282accf12d39edb8d524cf089645e55a13 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 4006c01772bd8d8ac4306d523bbcee41d3e392d8 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c 4c77cdf16be330bd5227691919332b42d557e211 +F src/vdbe.c 0b4ffa1aeeb7a10fefc6497dd0159bf5d9828464 F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 F src/vdbeInt.h e2a060a55ee18a6ab973353a5e2ec7ee569bf787 F src/vdbeapi.c 37a6c6ae284a97bcace365f2f0a225680c0499d9 @@ -873,7 +873,7 @@ F test/superlock.test 1cde669f68d2dd37d6c9bd35eee1d95491ae3fc2 F test/sync.test a34cd43e98b7fb84eabbf38f7ed8f7349b3f3d85 F test/syscall.test d2fdaad713f103ac611fe7ef9b724c7b69f8149c F test/sysfault.test fa776e60bf46bdd3ae69f0b73e46ee3977a58ae6 -F test/table.test 2a1d2fa52c531de5915f28023747d9a8c27b6f31 +F test/table.test 06271d61eb13871490d38168433c1ef3dd82bb2a F test/tableapi.test 2674633fa95d80da917571ebdd759a14d9819126 F test/tableopts.test dba698ba97251017b7c80d738c198d39ab747930 F test/tclsqlite.test 37a61c2da7e3bfe3b8c1a2867199f6b860df5d43 @@ -1204,8 +1204,8 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 3b7b72c4685aa5cf5e675c2c47ebec10d9704221 -Q +b5df5ac0529f7b0d1e880a7f4a307e7d77b7fa6c -R 27a95de33658c36b59b93f043d249813 +P d4b2d5d066891e06f2bf4337902b44b000fa9fd2 +Q +42705fd7d892c4fdfb95fbbb468c99569beece25 +R 55a6f98cd186254776492ca00fba4393 U drh -Z ea313ee60fabf91291f912ad6dcdc285 +Z efc3d14840732f527ab6a674cff641f8 diff --git a/manifest.uuid b/manifest.uuid index 983840bd0e..cfac283fb9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d4b2d5d066891e06f2bf4337902b44b000fa9fd2 \ No newline at end of file +e1017745e183f5d7429ce787ec2feef946a24f0f \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 6e152cd27e..9462bd71f8 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2298,7 +2298,7 @@ case OP_Column: { pC->payloadSize = pC->szRow = avail = pReg->n; pC->aRow = (u8*)pReg->z; }else{ - MemSetTypeFlag(pDest, MEM_Null); + sqlite3VdbeMemSetNull(pDest); goto op_column_out; } }else{ diff --git a/test/table.test b/test/table.test index 656884ca73..69f105aa6c 100644 --- a/test/table.test +++ b/test/table.test @@ -11,7 +11,6 @@ # This file implements regression tests for SQLite library. The # focus of this file is testing the CREATE TABLE statement. # -# $Id: table.test,v 1.53 2009/06/05 17:09:12 drh Exp $ set testdir [file dirname $argv0] source $testdir/tester.tcl @@ -773,4 +772,21 @@ do_catchsql_test table-16.7 { INSERT INTO t16 DEFAULT VALUES; } {1 {unknown function: group_concat()}} +# Ticket [https://www.sqlite.org/src/info/094d39a4c95ee4abbc417f04214617675ba15c63] +# describes a assertion fault that occurs on a CREATE TABLE .. AS SELECT statement. +# the following test verifies that the problem has been fixed. +# +do_execsql_test table-17.1 { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(a TEXT); + INSERT INTO t1(a) VALUES(1),(2); + DROP TABLE IF EXISTS t2; + CREATE TABLE t2(x TEXT, y TEXT); + INSERT INTO t2(x,y) VALUES(3,4); + DROP TABLE IF EXISTS t3; + CREATE TABLE t3 AS + SELECT a AS p, coalesce(y,a) AS q FROM t1 LEFT JOIN t2 ON a=x; + SELECT p, q, '|' FROM t3 ORDER BY p; +} {1 1 | 2 2 |} + finish_test From 5a2c8c885c722e52b87040330a931d36caf3d4af Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 12 Nov 2014 14:12:28 +0000 Subject: [PATCH 103/133] Fix the %c format character in sqlite3VXPrintf() so that it correctly handles precisions larger than 70. FossilOrigin-Name: 839a6df9f98b90fb593534a62145d9c913540bae --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/printf.c | 33 +++++++++++++++++++-------------- src/sqliteInt.h | 2 +- test/printf2.test | 20 ++++++++++++++++++++ 5 files changed, 50 insertions(+), 25 deletions(-) diff --git a/manifest b/manifest index e652361975..a2c5fb1bb8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\ssure\sthat\sNULL\sresults\sfrom\sOP_Column\sare\sfully\sand\scompletely\sNULL\sand\sdo\snot\shave\sthe\sMEM_Ephem\sbit\sset.\sFix\sfor\sticket\s[094d39a4c95ee4]. -D 2014-11-12T14:07:28.525 +C Fix\sthe\s%c\sformat\scharacter\sin\ssqlite3VXPrintf()\sso\sthat\sit\scorrectly\shandles\sprecisions\slarger\sthan\s70. +D 2014-11-12T14:12:28.051 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -223,7 +223,7 @@ F src/pcache.h 9b559127b83f84ff76d735c8262f04853be0c59a F src/pcache1.c e412cb585f777c840ddce0500eddc5c6043c2bb5 F src/pragma.c 3f3e959390a10c0131676f0e307acce372777e0f F src/prepare.c 6ef0cf2f9274982988ed6b7cab1be23147e94196 -F src/printf.c 090fac0f779c93c8a95089a125339686648835e4 +F src/printf.c d83b573624f3f6bc12b800af7fd55ce90be70659 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c a3466128b52a86c466e47ac1a19e2174f7b5cf89 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e @@ -232,7 +232,7 @@ F src/shell.c 18ee8bbe9502d8848072dc2eddd1ea09254ba494 F src/sqlite.h.in 4a5e5158c189d2bcd45c7c4607c2c0eb6d25c153 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d -F src/sqliteInt.h f7812f74f2d0c6041ef6b91a99c5a45f775dd408 +F src/sqliteInt.h c97db3c4d20b34c050a801c93451ef18e4f22de1 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 961d5926e5a8fda611d385ec22c226b8635cd1cb F src/table.c 2e99ef7ef16187e17033d9398dc962ce22dab5cb @@ -768,7 +768,7 @@ F test/permutations.test cef25f5e8499a15846eccd06785f17f4180407ab F test/pragma.test 19d0241a007bcdd77fc2606ec60fc60357e7fc8b F test/pragma2.test aea7b3d82c76034a2df2b38a13745172ddc0bc13 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 -F test/printf2.test bed79b4c3e5da08ba88ad637c0bf62586843cfb1 +F test/printf2.test b4acd4bf8734243257f01ddefa17c4fb090acc8a F test/progress.test a282973d1d17f08071bc58a77d6b80f2a81c354d F test/ptrchng.test ef1aa72d6cf35a2bbd0869a649b744e9d84977fc F test/queryonly.test 5f653159e0f552f0552d43259890c1089391dcca @@ -1204,8 +1204,8 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P d4b2d5d066891e06f2bf4337902b44b000fa9fd2 -Q +42705fd7d892c4fdfb95fbbb468c99569beece25 -R 55a6f98cd186254776492ca00fba4393 +P e1017745e183f5d7429ce787ec2feef946a24f0f +Q +08a27440f19b7fc884464832e6105af1bf008172 +R 3a2fc5ef3a825f22ce8bbd1b4d69c828 U drh -Z efc3d14840732f527ab6a674cff641f8 +Z 3e707181121665e3dbca925048596208 diff --git a/manifest.uuid b/manifest.uuid index cfac283fb9..55a1630790 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e1017745e183f5d7429ce787ec2feef946a24f0f \ No newline at end of file +839a6df9f98b90fb593534a62145d9c913540bae \ No newline at end of file diff --git a/src/printf.c b/src/printf.c index 1df287fbb6..387b9e90c1 100644 --- a/src/printf.c +++ b/src/printf.c @@ -212,7 +212,7 @@ void sqlite3VXPrintf( const et_info *infop; /* Pointer to the appropriate info structure */ char *zOut; /* Rendering buffer */ int nOut; /* Size of the rendering buffer */ - char *zExtra; /* Malloced memory used by some conversion */ + char *zExtra = 0; /* Malloced memory used by some conversion */ #ifndef SQLITE_OMIT_FLOATING_POINT int exp, e2; /* exponent of real numbers */ int nsd; /* Number of significant digits returned */ @@ -329,7 +329,6 @@ void sqlite3VXPrintf( break; } } - zExtra = 0; /* ** At this point, variables are initialized as follows: @@ -620,13 +619,16 @@ void sqlite3VXPrintf( }else{ c = va_arg(ap,int); } - buf[0] = (char)c; - if( precision>=0 ){ - for(idx=1; idx1 ){ + width -= precision-1; + if( width>1 && !flag_leftjustify ){ + sqlite3AppendChar(pAccum, width-1, ' '); + width = 0; + } + sqlite3AppendChar(pAccum, precision-1, c); } + length = 1; + buf[0] = c; bufpt = buf; break; case etSTRING: @@ -727,11 +729,14 @@ void sqlite3VXPrintf( ** the output. */ width -= length; - if( width>0 && !flag_leftjustify ) sqlite3AppendSpace(pAccum, width); + if( width>0 && !flag_leftjustify ) sqlite3AppendChar(pAccum, width, ' '); sqlite3StrAccumAppend(pAccum, bufpt, length); - if( width>0 && flag_leftjustify ) sqlite3AppendSpace(pAccum, width); + if( width>0 && flag_leftjustify ) sqlite3AppendChar(pAccum, width, ' '); - if( zExtra ) sqlite3_free(zExtra); + if( zExtra ){ + sqlite3_free(zExtra); + zExtra = 0; + } }/* End for loop over the format string */ } /* End of function */ @@ -784,11 +789,11 @@ static int sqlite3StrAccumEnlarge(StrAccum *p, int N){ } /* -** Append N space characters to the given string buffer. +** Append N copies of character c to the given string buffer. */ -void sqlite3AppendSpace(StrAccum *p, int N){ +void sqlite3AppendChar(StrAccum *p, int N, char c){ if( p->nChar+N >= p->nAlloc && (N = sqlite3StrAccumEnlarge(p, N))<=0 ) return; - while( (N--)>0 ) p->zText[p->nChar++] = ' '; + while( (N--)>0 ) p->zText[p->nChar++] = c; } /* diff --git a/src/sqliteInt.h b/src/sqliteInt.h index cba89b03e7..40b79a79b8 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3539,7 +3539,7 @@ int sqlite3OpenTempDatabase(Parse *); void sqlite3StrAccumInit(StrAccum*, char*, int, int); void sqlite3StrAccumAppend(StrAccum*,const char*,int); void sqlite3StrAccumAppendAll(StrAccum*,const char*); -void sqlite3AppendSpace(StrAccum*,int); +void sqlite3AppendChar(StrAccum*,int,char); char *sqlite3StrAccumFinish(StrAccum*); void sqlite3StrAccumReset(StrAccum*); void sqlite3SelectDestInit(SelectDest*,int,int); diff --git a/test/printf2.test b/test/printf2.test index 4cb1783bfb..21deeb779d 100644 --- a/test/printf2.test +++ b/test/printf2.test @@ -95,5 +95,25 @@ do_execsql_test printf2-2.3 { SELECT printf('%s=(%d/%g/%s)',a) FROM t1 ORDER BY a; } {-1=(0/0/) 1=(0/0/) 1.5=(0/0/) abc=(0/0/)} +# The precision of the %c conversion causes the character to repeat. +# +do_execsql_test printf2-3.1 { + SELECT printf('|%110.100c|','*'); +} {{| ****************************************************************************************************|}} +do_execsql_test printf2-3.2 { + SELECT printf('|%-110.100c|','*'); +} {{|**************************************************************************************************** |}} +do_execsql_test printf2-3.3 { + SELECT printf('|%9.8c|%-9.8c|','*','*'); +} {{| ********|******** |}} +do_execsql_test printf2-3.4 { + SELECT printf('|%8.8c|%-8.8c|','*','*'); +} {|********|********|} +do_execsql_test printf2-3.5 { + SELECT printf('|%7.8c|%-7.8c|','*','*'); +} {|********|********|} + + + finish_test From 80231042527411987a907328722e169e4e1cb47e Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 12 Nov 2014 14:56:02 +0000 Subject: [PATCH 104/133] When a transaction or savepoint rollback occurs, save the positions of all open read-cursors so that they can be restored following the rollback operation. FossilOrigin-Name: dd03a2802f3f276525f3cef9a93f825dd8606626 --- manifest | 19 ++++++------ manifest.uuid | 2 +- src/btree.c | 63 ++++++++++++++++++++++++++------------ src/btree.h | 2 +- src/vdbe.c | 3 +- test/rollback2.test | 74 +++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 132 insertions(+), 31 deletions(-) create mode 100644 test/rollback2.test diff --git a/manifest b/manifest index 97192e581d..11b9906fa0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\ssome\scalls\sto\sthe\s'breakpoint'\stest\scommand. -D 2014-11-11T19:07:56.372 +C When\sa\stransaction\sor\ssavepoint\srollback\soccurs,\ssave\sthe\spositions\sof\sall\sopen\sread-cursors\sso\sthat\sthey\scan\sbe\srestored\sfollowing\sthe\srollback\soperation. +D 2014-11-12T14:56:02.923 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -173,8 +173,8 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 86b311af1ad35decd1f2f0cee457d344acb83c3b F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c eaef0003bbfe740c62189355dabc818fc3a98999 -F src/btree.h d24fc2f3bc53be220b79b95800bdb2ee207b1089 +F src/btree.c d5d991b518fa5bebc64037dfeb98a48051d864d7 +F src/btree.h e31a3a3ebdedb1caf9bda3ad5dbab3db9b780f6e F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 67bb05b1077e0cdaccb2e36bfcbe7a5df9ed31e8 F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 @@ -291,7 +291,7 @@ F src/update.c 3c4ecc282accf12d39edb8d524cf089645e55a13 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 3b627daa45c7308c1e36e3dbaa3f9ce7e5c7fa73 F src/vacuum.c 9b30ec729337dd012ed88d4c292922c8ef9cf00c -F src/vdbe.c c71d819bb34269c3dbccd92e6bb308f0ec025b5d +F src/vdbe.c 5e47308836e9bb5fdb4835fdf88eeab071848d3f F src/vdbe.h 6fc69d9c5e146302c56e163cb4b31d1ee64a18c3 F src/vdbeInt.h 9bb69ff2447c34b6ccc58b34ec35b615f86ead78 F src/vdbeapi.c 07acb615d1e4170e71fc1b0d087f3c53a1ad8e83 @@ -797,6 +797,7 @@ F test/releasetest.mk 2eced2f9ae701fd0a29e714a241760503ccba25a F test/releasetest.tcl a4279c890698584feb2ffc86735857a4e4474180 F test/resolver01.test 33abf37ff8335e6bf98f2b45a0af3e06996ccd9a F test/rollback.test 458fe73eb3ffdfdf9f6ba3e9b7350a6220414dea +F test/rollback2.test 552abaab8e721b6060a727d639896427059e51ec F test/rowhash.test 0bc1d31415e4575d10cacf31e1a66b5cc0f8be81 F test/rowid.test 742b5741584a8a44fd83e856cc2896688401d645 F test/rtree.test 0c8d9dd458d6824e59683c19ab2ffa9ef946f798 @@ -1218,7 +1219,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P a066a3832a7c6de65c3016e77e49ac00e09db749 -R e8fc41a7f82a02ab8618a341f7ef7735 -U mistachkin -Z 90417f4c358533080e79b8017b4e7453 +P 1412fcc480799ecbd68d44dd18d5bad40e20ccf1 +R 668a86f911283b47b6c156fecf772681 +U dan +Z cf8e25329461439611751d7913ef3ef9 diff --git a/manifest.uuid b/manifest.uuid index f90bf2d8c8..3369a7ad97 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1412fcc480799ecbd68d44dd18d5bad40e20ccf1 \ No newline at end of file +dd03a2802f3f276525f3cef9a93f825dd8606626 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 70fae7c707..f153a0cc45 100644 --- a/src/btree.c +++ b/src/btree.c @@ -3514,29 +3514,52 @@ int sqlite3BtreeCommit(Btree *p){ ** that belong to other database connections that happen to be ** sharing the cache with pBtree. ** -** This routine gets called when a rollback occurs. The writeOnly -** flag is set to 1 if the transaction did not make any schema -** changes, in which case the read cursors can continue operating. -** If schema changes did occur in the transaction, then both read -** and write cursors must both be tripped. +** This routine gets called when a rollback occurs. If the writeOnly +** flag is true, then only write-cursors need be tripped - read-only +** cursors save their current positions so that they may continue +** following the rollback. Or, if writeOnly is false, all cursors are +** tripped. In general, writeOnly is false if the transaction being +** rolled back modified the database schema. In this case b-tree root +** pages may be moved or deleted from the database altogether, making +** it unsafe for read cursors to continue. +** +** If the writeOnly flag is true and an error is encountered while +** saving the current position of a read-only cursor, all cursors, +** including all read-cursors are tripped. +** +** SQLITE_OK is returned if successful, or if an error occurs while +** saving a cursor position, an SQLite error code. */ -void sqlite3BtreeTripAllCursors(Btree *pBtree, int errCode, int writeOnly){ +int sqlite3BtreeTripAllCursors(Btree *pBtree, int errCode, int writeOnly){ BtCursor *p; + int rc = SQLITE_OK; + assert( (writeOnly==0 || writeOnly==1) && BTCF_WriteFlag==1 ); - if( pBtree==0 ) return; - sqlite3BtreeEnter(pBtree); - for(p=pBtree->pBt->pCursor; p; p=p->pNext){ - int i; - if( writeOnly && (p->curFlags & BTCF_WriteFlag)==0 ) continue; - sqlite3BtreeClearCursor(p); - p->eState = CURSOR_FAULT; - p->skipNext = errCode; - for(i=0; i<=p->iPage; i++){ - releasePage(p->apPage[i]); - p->apPage[i] = 0; + if( pBtree ){ + sqlite3BtreeEnter(pBtree); + for(p=pBtree->pBt->pCursor; p; p=p->pNext){ + int i; + if( writeOnly && (p->curFlags & BTCF_WriteFlag)==0 ){ + if( p->eState==CURSOR_VALID ){ + int rc = saveCursorPosition(p); + if( rc!=SQLITE_OK ){ + (void)sqlite3BtreeTripAllCursors(pBtree, rc, 0); + break; + } + } + }else{ + sqlite3BtreeClearCursor(p); + p->eState = CURSOR_FAULT; + p->skipNext = errCode; + } + for(i=0; i<=p->iPage; i++){ + releasePage(p->apPage[i]); + p->apPage[i] = 0; + } } + sqlite3BtreeLeave(pBtree); } - sqlite3BtreeLeave(pBtree); + return rc; } /* @@ -3565,7 +3588,9 @@ int sqlite3BtreeRollback(Btree *p, int tripCode, int writeOnly){ rc = SQLITE_OK; } if( tripCode ){ - sqlite3BtreeTripAllCursors(p, tripCode, writeOnly); + int rc2 = sqlite3BtreeTripAllCursors(p, tripCode, writeOnly); + assert( rc==SQLITE_OK || (writeOnly==0 && rc2==SQLITE_OK) ); + if( rc2!=SQLITE_OK ) rc = rc2; } btreeIntegrity(p); diff --git a/src/btree.h b/src/btree.h index a6093a5edd..281e57dafd 100644 --- a/src/btree.h +++ b/src/btree.h @@ -116,7 +116,7 @@ int sqlite3BtreeIncrVacuum(Btree *); int sqlite3BtreeDropTable(Btree*, int, int*); int sqlite3BtreeClearTable(Btree*, int, int*); int sqlite3BtreeClearTableOfCursor(BtCursor*); -void sqlite3BtreeTripAllCursors(Btree*, int, int); +int sqlite3BtreeTripAllCursors(Btree*, int, int); void sqlite3BtreeGetMeta(Btree *pBtree, int idx, u32 *pValue); int sqlite3BtreeUpdateMeta(Btree*, int idx, u32 value); diff --git a/src/vdbe.c b/src/vdbe.c index 715862e73c..e4cbeb6565 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2830,8 +2830,9 @@ case OP_Savepoint: { if( p1==SAVEPOINT_ROLLBACK ){ isSchemaChange = (db->flags & SQLITE_InternChanges)!=0; for(ii=0; iinDb; ii++){ - sqlite3BtreeTripAllCursors(db->aDb[ii].pBt, SQLITE_ABORT, + rc = sqlite3BtreeTripAllCursors(db->aDb[ii].pBt, SQLITE_ABORT, isSchemaChange==0); + if( rc!=SQLITE_OK ) goto abort_due_to_error; } }else{ isSchemaChange = 0; diff --git a/test/rollback2.test b/test/rollback2.test new file mode 100644 index 0000000000..9637f0c0e5 --- /dev/null +++ b/test/rollback2.test @@ -0,0 +1,74 @@ +# 2014 November 12 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set ::testprefix rollback2 + +proc int2hex {i} { format %.2X $i } +db func int2hex int2hex + +do_execsql_test 1.0 { + SELECT int2hex(0), int2hex(100), int2hex(255) +} {00 64 FF} +do_execsql_test 1.1 { + CREATE TABLE t1(i, h); + CREATE INDEX i1 ON t1(h); + WITH data(a, b) AS ( + SELECT 1, int2hex(1) + UNION ALL + SELECT a+1, int2hex(a+1) FROM data WHERE a<40 + ) + INSERT INTO t1 SELECT * FROM data; +} {} + + +proc do_rollback_test {tn args} { + set A(-setup) "" + set A(-select) "" + set A(-result) "" + set A(-rollback) ROLLBACK + + array set O $args + foreach k [array names O] { + if {[info exists A($k)]==0} { error "unknown option: $k" } + set A($k) $O($k) + } + + for {set iRollback 0} 1 {incr iRollback} { + catch { db eval ROLLBACK } + set res [list] + db eval $A(-setup) + + set i 0 + db eval $A(-select) x { + if {$i==$iRollback} { db eval $A(-rollback) } + foreach k $x(*) { lappend res $x($k) } + incr i + } + + do_test $tn.$iRollback [list set {} $res] [list {*}$A(-result)] + if {$i < $iRollback} break + } +} + +do_rollback_test 2 -setup { + BEGIN; + DELETE FROM t1 WHERE (i%2)==1; +} -select { + SELECT i FROM t1 WHERE (i%2)==0 +} -result { + 2 4 6 8 10 12 14 16 18 20 22 24 26 28 30 32 34 36 38 40 +} + +finish_test + From d7b06909ad144b993f59e3a67d214b5ad27149fe Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 12 Nov 2014 17:45:37 +0000 Subject: [PATCH 105/133] Add further tests for rollback operations in the presence of ongoing selects. FossilOrigin-Name: eaf3aae014f59c8d37aa20aa31d54cf13f9e86fc --- manifest | 13 +++--- manifest.uuid | 2 +- test/rollback2.test | 87 ++++++++++++++++++++++++++++++++++++++++- test/rollbackfault.test | 84 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 177 insertions(+), 9 deletions(-) create mode 100644 test/rollbackfault.test diff --git a/manifest b/manifest index 11b9906fa0..dddb788b2d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\sa\stransaction\sor\ssavepoint\srollback\soccurs,\ssave\sthe\spositions\sof\sall\sopen\sread-cursors\sso\sthat\sthey\scan\sbe\srestored\sfollowing\sthe\srollback\soperation. -D 2014-11-12T14:56:02.923 +C Add\sfurther\stests\sfor\srollback\soperations\sin\sthe\spresence\sof\songoing\sselects. +D 2014-11-12T17:45:37.113 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -797,7 +797,8 @@ F test/releasetest.mk 2eced2f9ae701fd0a29e714a241760503ccba25a F test/releasetest.tcl a4279c890698584feb2ffc86735857a4e4474180 F test/resolver01.test 33abf37ff8335e6bf98f2b45a0af3e06996ccd9a F test/rollback.test 458fe73eb3ffdfdf9f6ba3e9b7350a6220414dea -F test/rollback2.test 552abaab8e721b6060a727d639896427059e51ec +F test/rollback2.test fc14cf6d1a2b250d2735ef16124b971bce152f14 +F test/rollbackfault.test 6a004f71087cc399296cffbb5429ea6da655ae65 F test/rowhash.test 0bc1d31415e4575d10cacf31e1a66b5cc0f8be81 F test/rowid.test 742b5741584a8a44fd83e856cc2896688401d645 F test/rtree.test 0c8d9dd458d6824e59683c19ab2ffa9ef946f798 @@ -1219,7 +1220,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 1412fcc480799ecbd68d44dd18d5bad40e20ccf1 -R 668a86f911283b47b6c156fecf772681 +P dd03a2802f3f276525f3cef9a93f825dd8606626 +R c0a8c133338658022754212d7071964f U dan -Z cf8e25329461439611751d7913ef3ef9 +Z c8dc7391cf2e61af731936b53ad2fd87 diff --git a/manifest.uuid b/manifest.uuid index 3369a7ad97..634cdcad8d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dd03a2802f3f276525f3cef9a93f825dd8606626 \ No newline at end of file +eaf3aae014f59c8d37aa20aa31d54cf13f9e86fc \ No newline at end of file diff --git a/test/rollback2.test b/test/rollback2.test index 9637f0c0e5..4d42dda5d6 100644 --- a/test/rollback2.test +++ b/test/rollback2.test @@ -9,6 +9,9 @@ # #*********************************************************************** # +# This file containst tests to verify that ROLLBACK or ROLLBACK TO +# operations interact correctly with ongoing SELECT statements. +# set testdir [file dirname $argv0] source $testdir/tester.tcl @@ -16,7 +19,6 @@ set ::testprefix rollback2 proc int2hex {i} { format %.2X $i } db func int2hex int2hex - do_execsql_test 1.0 { SELECT int2hex(0), int2hex(100), int2hex(255) } {00 64 FF} @@ -32,6 +34,17 @@ do_execsql_test 1.1 { } {} +# do_rollback_test ID SWITCHES +# +# where SWITCHES are: +# +# -setup SQL script to open transaction and begin writing. +# -select SELECT to execute after -setup script +# -result Expected result of -select statement +# -rollback Use this SQL command ("ROLLBACK" or "ROLLBACK TO ...") to +# rollback the transaction in the middle of the -select statment +# execution. +# proc do_rollback_test {tn args} { set A(-setup) "" set A(-select) "" @@ -61,7 +74,7 @@ proc do_rollback_test {tn args} { } } -do_rollback_test 2 -setup { +do_rollback_test 2.1 -setup { BEGIN; DELETE FROM t1 WHERE (i%2)==1; } -select { @@ -70,5 +83,75 @@ do_rollback_test 2 -setup { 2 4 6 8 10 12 14 16 18 20 22 24 26 28 30 32 34 36 38 40 } +do_rollback_test 2.2 -setup { + BEGIN; + DELETE FROM t1 WHERE (i%4)==1; + SAVEPOINT one; + DELETE FROM t1 WHERE (i%2)==1; +} -rollback { + ROLLBACK TO one; +} -select { + SELECT i FROM t1 WHERE (i%2)==0 +} -result { + 2 4 6 8 10 12 14 16 18 20 22 24 26 28 30 32 34 36 38 40 +} + +#-------------------------------------------------------------------- +# Try with some index scans +# +do_eqp_test 3.1 { + SELECT i FROM t1 WHERE (i%2)==0 ORDER BY h DESC; +} {0 0 0 {SCAN TABLE t1 USING INDEX i1}} +do_rollback_test 3.2 -setup { + BEGIN; + DELETE FROM t1 WHERE (i%2)==1; +} -select { + SELECT i FROM t1 WHERE (i%2)==0 ORDER BY h DESC; +} -result { + 40 38 36 34 32 30 28 26 24 22 20 18 16 14 12 10 8 6 4 2 +} +do_rollback_test 3.3 -setup { + BEGIN; + DELETE FROM t1 WHERE (i%4)==1; + SAVEPOINT one; + DELETE FROM t1 WHERE (i%2)==1; +} -rollback { + ROLLBACK TO one; +} -select { + SELECT i FROM t1 WHERE (i%2)==0 ORDER BY h DESC; +} -result { + 40 38 36 34 32 30 28 26 24 22 20 18 16 14 12 10 8 6 4 2 +} + +#-------------------------------------------------------------------- +# Now with some index scans that feature overflow keys. +# +set leader [string repeat "abcdefghij" 70] +do_execsql_test 4.1 { UPDATE t1 SET h = $leader || h; } + +do_eqp_test 4.2 { + SELECT i FROM t1 WHERE (i%2)==0 ORDER BY h ASC; +} {0 0 0 {SCAN TABLE t1 USING INDEX i1}} +do_rollback_test 4.3 -setup { + BEGIN; + DELETE FROM t1 WHERE (i%2)==1; +} -select { + SELECT i FROM t1 WHERE (i%2)==0 ORDER BY h ASC; +} -result { + 2 4 6 8 10 12 14 16 18 20 22 24 26 28 30 32 34 36 38 40 +} +do_rollback_test 4.4 -setup { + BEGIN; + DELETE FROM t1 WHERE (i%4)==1; + SAVEPOINT one; + DELETE FROM t1 WHERE (i%2)==1; +} -rollback { + ROLLBACK TO one; +} -select { + SELECT i FROM t1 WHERE (i%2)==0 ORDER BY h ASC; +} -result { + 2 4 6 8 10 12 14 16 18 20 22 24 26 28 30 32 34 36 38 40 +} + finish_test diff --git a/test/rollbackfault.test b/test/rollbackfault.test new file mode 100644 index 0000000000..f248d0758d --- /dev/null +++ b/test/rollbackfault.test @@ -0,0 +1,84 @@ +# 2014-11-12 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Test that errors encountered during a ROLLBACK operation correctly +# affect ongoing SQL statements. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/malloc_common.tcl +set testprefix rollbackfault + + +proc int2hex {i} { format %.2X $i } +db func int2hex int2hex +do_execsql_test 1.0 { + SELECT int2hex(0), int2hex(100), int2hex(255) +} {00 64 FF} +do_execsql_test 1.1 { + CREATE TABLE t1(i, h); + CREATE INDEX i1 ON t1(h); + WITH data(a, b) AS ( + SELECT 1, int2hex(1) + UNION ALL + SELECT a+1, int2hex(a+1) FROM data WHERE a<40 + ) + INSERT INTO t1 SELECT * FROM data; +} {} + +foreach f {oom ioerr} { + do_faultsim_test 1.2 -faults $f* -prep { + set sql1 { SELECT i FROM t1 WHERE (i%2)==0 } + set sql2 { SELECT i FROM t1 WHERE (i%2)==0 ORDER BY h } + set ::s1 [sqlite3_prepare db $sql1 -1 dummy] + set ::s2 [sqlite3_prepare db $sql2 -1 dummy] + + for {set i 0} {$i < 10} {incr i} { sqlite3_step $::s1 } + for {set i 0} {$i < 3} {incr i} { sqlite3_step $::s2 } + + execsql { + BEGIN; DELETE FROM t1 WHERE (i%2) + } + } -body { + execsql { ROLLBACK } + } -test { + + set res1 [list] + set res2 [list] + while {"SQLITE_ROW" == [sqlite3_step $::s1]} { + lappend res1 [sqlite3_column_text $::s1 0] + } + while {"SQLITE_ROW" == [sqlite3_step $::s2]} { + lappend res2 [sqlite3_column_text $::s2 0] + } + set rc1 [sqlite3_finalize $::s1] + set rc2 [sqlite3_finalize $::s2] + + catchsql { ROLLBACK } + + if {$rc1=="SQLITE_OK" && $rc2=="SQLITE_OK" + && $res1=="22 24 26 28 30 32 34 36 38 40" + && $res2=="8 10 12 14 16 18 20 22 24 26 28 30 32 34 36 38 40" + } { + # This is Ok. + } elseif {$rc1!="SQLITE_OK" && $rc2!="SQLITE_OK" && $res1=="" &&$res2==""} { + # Also Ok. + } else { + error "statements don't look right" + } + } +} + + +finish_test + + From 6f9c5669e2d54fb4e480b2f1f39fba6907560cc1 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 13 Nov 2014 13:42:39 +0000 Subject: [PATCH 106/133] When a transaction or savepoint rollback occurs, save the positions of all open read-cursors so that they can be restored following the rollback operation. Cherry-pick of check-in [dd03a2802f3f27] FossilOrigin-Name: 402780a9c8df9e7ea898bdca49c1191042fe387a --- manifest | 19 ++++++------ manifest.uuid | 2 +- src/btree.c | 63 ++++++++++++++++++++++++++------------ src/btree.h | 2 +- src/vdbe.c | 3 +- test/rollback2.test | 74 +++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 132 insertions(+), 31 deletions(-) create mode 100644 test/rollback2.test diff --git a/manifest b/manifest index a2c5fb1bb8..1d9fe4d78f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\s%c\sformat\scharacter\sin\ssqlite3VXPrintf()\sso\sthat\sit\scorrectly\shandles\sprecisions\slarger\sthan\s70. -D 2014-11-12T14:12:28.051 +C When\sa\stransaction\sor\ssavepoint\srollback\soccurs,\ssave\sthe\spositions\sof\sall\sopen\sread-cursors\sso\sthat\sthey\scan\sbe\srestored\sfollowing\sthe\srollback\soperation.\s\sCherry-pick\sof\scheck-in\s[dd03a2802f3f27] +D 2014-11-13T13:42:39.738 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,8 +172,8 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c 8cdfeb0c8a6d8bdad3faefae418eb3dc767051b6 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 944b84f72fd9048acde31400d89439976bf766b0 -F src/btree.h 97c4d0ef860f151e096e97eef78b87f7b4a8d899 +F src/btree.c 2c15850c5c9a26b10cdf92f9a29c74e299dc3674 +F src/btree.h a4afc6b06f5a1dd2076d15aa168baec44fc0121b F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 9dc2bd94347b878c89627000c92b0c8d97ec2919 F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 @@ -289,7 +289,7 @@ F src/update.c 3c4ecc282accf12d39edb8d524cf089645e55a13 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 4006c01772bd8d8ac4306d523bbcee41d3e392d8 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c 0b4ffa1aeeb7a10fefc6497dd0159bf5d9828464 +F src/vdbe.c a6b604364c7cbb079c083418e7359d1d665f2ef0 F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 F src/vdbeInt.h e2a060a55ee18a6ab973353a5e2ec7ee569bf787 F src/vdbeapi.c 37a6c6ae284a97bcace365f2f0a225680c0499d9 @@ -786,6 +786,7 @@ F test/releasetest.mk 2eced2f9ae701fd0a29e714a241760503ccba25a F test/releasetest.tcl a4279c890698584feb2ffc86735857a4e4474180 F test/resolver01.test 33abf37ff8335e6bf98f2b45a0af3e06996ccd9a F test/rollback.test 458fe73eb3ffdfdf9f6ba3e9b7350a6220414dea +F test/rollback2.test 552abaab8e721b6060a727d639896427059e51ec F test/rowhash.test 0bc1d31415e4575d10cacf31e1a66b5cc0f8be81 F test/rowid.test 742b5741584a8a44fd83e856cc2896688401d645 F test/rtree.test 0c8d9dd458d6824e59683c19ab2ffa9ef946f798 @@ -1204,8 +1205,8 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P e1017745e183f5d7429ce787ec2feef946a24f0f -Q +08a27440f19b7fc884464832e6105af1bf008172 -R 3a2fc5ef3a825f22ce8bbd1b4d69c828 +P 839a6df9f98b90fb593534a62145d9c913540bae +Q +dd03a2802f3f276525f3cef9a93f825dd8606626 +R 0801523eb21d9763755a3afd3e84ae8b U drh -Z 3e707181121665e3dbca925048596208 +Z 995ce8d43edc83cec464fe7495c36085 diff --git a/manifest.uuid b/manifest.uuid index 55a1630790..38b0ca2922 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -839a6df9f98b90fb593534a62145d9c913540bae \ No newline at end of file +402780a9c8df9e7ea898bdca49c1191042fe387a \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 9d7a05157d..41e097af53 100644 --- a/src/btree.c +++ b/src/btree.c @@ -3472,29 +3472,52 @@ int sqlite3BtreeCommit(Btree *p){ ** that belong to other database connections that happen to be ** sharing the cache with pBtree. ** -** This routine gets called when a rollback occurs. The writeOnly -** flag is set to 1 if the transaction did not make any schema -** changes, in which case the read cursors can continue operating. -** If schema changes did occur in the transaction, then both read -** and write cursors must both be tripped. +** This routine gets called when a rollback occurs. If the writeOnly +** flag is true, then only write-cursors need be tripped - read-only +** cursors save their current positions so that they may continue +** following the rollback. Or, if writeOnly is false, all cursors are +** tripped. In general, writeOnly is false if the transaction being +** rolled back modified the database schema. In this case b-tree root +** pages may be moved or deleted from the database altogether, making +** it unsafe for read cursors to continue. +** +** If the writeOnly flag is true and an error is encountered while +** saving the current position of a read-only cursor, all cursors, +** including all read-cursors are tripped. +** +** SQLITE_OK is returned if successful, or if an error occurs while +** saving a cursor position, an SQLite error code. */ -void sqlite3BtreeTripAllCursors(Btree *pBtree, int errCode, int writeOnly){ +int sqlite3BtreeTripAllCursors(Btree *pBtree, int errCode, int writeOnly){ BtCursor *p; + int rc = SQLITE_OK; + assert( (writeOnly==0 || writeOnly==1) && BTCF_WriteFlag==1 ); - if( pBtree==0 ) return; - sqlite3BtreeEnter(pBtree); - for(p=pBtree->pBt->pCursor; p; p=p->pNext){ - int i; - if( writeOnly && (p->curFlags & BTCF_WriteFlag)==0 ) continue; - sqlite3BtreeClearCursor(p); - p->eState = CURSOR_FAULT; - p->skipNext = errCode; - for(i=0; i<=p->iPage; i++){ - releasePage(p->apPage[i]); - p->apPage[i] = 0; + if( pBtree ){ + sqlite3BtreeEnter(pBtree); + for(p=pBtree->pBt->pCursor; p; p=p->pNext){ + int i; + if( writeOnly && (p->curFlags & BTCF_WriteFlag)==0 ){ + if( p->eState==CURSOR_VALID ){ + int rc = saveCursorPosition(p); + if( rc!=SQLITE_OK ){ + (void)sqlite3BtreeTripAllCursors(pBtree, rc, 0); + break; + } + } + }else{ + sqlite3BtreeClearCursor(p); + p->eState = CURSOR_FAULT; + p->skipNext = errCode; + } + for(i=0; i<=p->iPage; i++){ + releasePage(p->apPage[i]); + p->apPage[i] = 0; + } } + sqlite3BtreeLeave(pBtree); } - sqlite3BtreeLeave(pBtree); + return rc; } /* @@ -3523,7 +3546,9 @@ int sqlite3BtreeRollback(Btree *p, int tripCode, int writeOnly){ rc = SQLITE_OK; } if( tripCode ){ - sqlite3BtreeTripAllCursors(p, tripCode, writeOnly); + int rc2 = sqlite3BtreeTripAllCursors(p, tripCode, writeOnly); + assert( rc==SQLITE_OK || (writeOnly==0 && rc2==SQLITE_OK) ); + if( rc2!=SQLITE_OK ) rc = rc2; } btreeIntegrity(p); diff --git a/src/btree.h b/src/btree.h index f724269a3e..fabedd9a53 100644 --- a/src/btree.h +++ b/src/btree.h @@ -116,7 +116,7 @@ int sqlite3BtreeIncrVacuum(Btree *); int sqlite3BtreeDropTable(Btree*, int, int*); int sqlite3BtreeClearTable(Btree*, int, int*); int sqlite3BtreeClearTableOfCursor(BtCursor*); -void sqlite3BtreeTripAllCursors(Btree*, int, int); +int sqlite3BtreeTripAllCursors(Btree*, int, int); void sqlite3BtreeGetMeta(Btree *pBtree, int idx, u32 *pValue); int sqlite3BtreeUpdateMeta(Btree*, int idx, u32 value); diff --git a/src/vdbe.c b/src/vdbe.c index 9462bd71f8..1ad8aab753 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2827,8 +2827,9 @@ case OP_Savepoint: { if( p1==SAVEPOINT_ROLLBACK ){ isSchemaChange = (db->flags & SQLITE_InternChanges)!=0; for(ii=0; iinDb; ii++){ - sqlite3BtreeTripAllCursors(db->aDb[ii].pBt, SQLITE_ABORT, + rc = sqlite3BtreeTripAllCursors(db->aDb[ii].pBt, SQLITE_ABORT, isSchemaChange==0); + if( rc!=SQLITE_OK ) goto abort_due_to_error; } }else{ isSchemaChange = 0; diff --git a/test/rollback2.test b/test/rollback2.test new file mode 100644 index 0000000000..9637f0c0e5 --- /dev/null +++ b/test/rollback2.test @@ -0,0 +1,74 @@ +# 2014 November 12 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set ::testprefix rollback2 + +proc int2hex {i} { format %.2X $i } +db func int2hex int2hex + +do_execsql_test 1.0 { + SELECT int2hex(0), int2hex(100), int2hex(255) +} {00 64 FF} +do_execsql_test 1.1 { + CREATE TABLE t1(i, h); + CREATE INDEX i1 ON t1(h); + WITH data(a, b) AS ( + SELECT 1, int2hex(1) + UNION ALL + SELECT a+1, int2hex(a+1) FROM data WHERE a<40 + ) + INSERT INTO t1 SELECT * FROM data; +} {} + + +proc do_rollback_test {tn args} { + set A(-setup) "" + set A(-select) "" + set A(-result) "" + set A(-rollback) ROLLBACK + + array set O $args + foreach k [array names O] { + if {[info exists A($k)]==0} { error "unknown option: $k" } + set A($k) $O($k) + } + + for {set iRollback 0} 1 {incr iRollback} { + catch { db eval ROLLBACK } + set res [list] + db eval $A(-setup) + + set i 0 + db eval $A(-select) x { + if {$i==$iRollback} { db eval $A(-rollback) } + foreach k $x(*) { lappend res $x($k) } + incr i + } + + do_test $tn.$iRollback [list set {} $res] [list {*}$A(-result)] + if {$i < $iRollback} break + } +} + +do_rollback_test 2 -setup { + BEGIN; + DELETE FROM t1 WHERE (i%2)==1; +} -select { + SELECT i FROM t1 WHERE (i%2)==0 +} -result { + 2 4 6 8 10 12 14 16 18 20 22 24 26 28 30 32 34 36 38 40 +} + +finish_test + From fad01993b7f9a5d519dd5d344dd5e4c8a44ccc06 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 13 Nov 2014 14:18:25 +0000 Subject: [PATCH 107/133] Have calls to sqlite3_backup_init() fail if there is already a read or read-write transaction open on the destination database. FossilOrigin-Name: 169b5505498c0a7ee2b5dbb2ba13c41dfaa7c62f --- manifest | 15 ++++++----- manifest.uuid | 2 +- src/backup.c | 27 ++++++++++++++++---- test/backup.test | 1 + test/backup5.test | 65 +++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 97 insertions(+), 13 deletions(-) create mode 100644 test/backup5.test diff --git a/manifest b/manifest index dddb788b2d..4f0d48c405 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sfurther\stests\sfor\srollback\soperations\sin\sthe\spresence\sof\songoing\sselects. -D 2014-11-12T17:45:37.113 +C Have\scalls\sto\ssqlite3_backup_init()\sfail\sif\sthere\sis\salready\sa\sread\sor\sread-write\stransaction\sopen\son\sthe\sdestination\sdatabase. +D 2014-11-13T14:18:25.531 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -170,7 +170,7 @@ F src/alter.c ba266a779bc7ce10e52e59e7d3dc79fa342e8fdb F src/analyze.c afbcca663c3f3625340b8e30d440cd7a97ded6bc F src/attach.c f4e94df2d1826feda65eb0939f7f6f5f923a0ad9 F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 -F src/backup.c 86b311af1ad35decd1f2f0cee457d344acb83c3b +F src/backup.c 7ddee9c7d505e07e959a575b18498f17c71e53ea F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 F src/btree.c d5d991b518fa5bebc64037dfeb98a48051d864d7 @@ -353,9 +353,10 @@ F test/autovacuum.test 941892505d2c0f410a0cb5970dfa1c7c4e5f6e74 F test/autovacuum_ioerr2.test 8a367b224183ad801e0e24dcb7d1501f45f244b4 F test/avtrans.test 0252654f4295ddda3b2cce0e894812259e655a85 F test/backcompat.test 19a1f337c68419b020a7481dd272a472c4ad8ef4 -F test/backup.test c9cdd23a495864b9edf75a9fa66f5cb7e10fcf62 +F test/backup.test b79299a536a4c6d919094786595b95be56d02014 F test/backup2.test 34986ef926ea522911a51dfdb2f8e99b7b75ebcf F test/backup4.test 2a2e4a64388090b152de753fd9e123f28f6a3bd4 +F test/backup5.test ee5da6d7fe5082f5b9b0bbfa31d016f52412a2e4 F test/backup_ioerr.test 4c3c7147cee85b024ecf6e150e090c32fdbb5135 F test/backup_malloc.test 7162d604ec2b4683c4b3799a48657fb8b5e2d450 F test/badutf.test d5360fc31f643d37a973ab0d8b4fb85799c3169f @@ -1220,7 +1221,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P dd03a2802f3f276525f3cef9a93f825dd8606626 -R c0a8c133338658022754212d7071964f +P eaf3aae014f59c8d37aa20aa31d54cf13f9e86fc +R cf416376a3614eef6301ba322bd3cc43 U dan -Z c8dc7391cf2e61af731936b53ad2fd87 +Z 499ccc085277a0250fd3c5f22e93f6be diff --git a/manifest.uuid b/manifest.uuid index 634cdcad8d..73dcaa8447 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -eaf3aae014f59c8d37aa20aa31d54cf13f9e86fc \ No newline at end of file +169b5505498c0a7ee2b5dbb2ba13c41dfaa7c62f \ No newline at end of file diff --git a/src/backup.c b/src/backup.c index 57f7f447ea..e3f869035e 100644 --- a/src/backup.c +++ b/src/backup.c @@ -122,6 +122,20 @@ static int setDestPgsz(sqlite3_backup *p){ return rc; } +/* +** Check that there is no open read-transaction on the b-tree passed as the +** second argument. If there is not, return SQLITE_OK. Otherwise, if there +** is an open read-transaction, return SQLITE_ERROR and leave an error +** message in database handle db. +*/ +static int checkReadTransaction(sqlite3 *db, Btree *p){ + if( sqlite3BtreeIsInReadTrans(p) ){ + sqlite3ErrorWithMsg(db, SQLITE_ERROR, "destination database is in use"); + return SQLITE_ERROR; + } + return SQLITE_OK; +} + /* ** Create an sqlite3_backup process to copy the contents of zSrcDb from ** connection handle pSrcDb to zDestDb in pDestDb. If successful, return @@ -181,12 +195,15 @@ sqlite3_backup *sqlite3_backup_init( p->iNext = 1; p->isAttached = 0; - if( 0==p->pSrc || 0==p->pDest || setDestPgsz(p)==SQLITE_NOMEM ){ + if( 0==p->pSrc || 0==p->pDest + || setDestPgsz(p)==SQLITE_NOMEM + || checkReadTransaction(pDestDb, p->pDest)!=SQLITE_OK + ){ /* One (or both) of the named databases did not exist or an OOM - ** error was hit. The error has already been written into the - ** pDestDb handle. All that is left to do here is free the - ** sqlite3_backup structure. - */ + ** error was hit. Or there is a transaction open on the destination + ** database. The error has already been written into the pDestDb + ** handle. All that is left to do here is free the sqlite3_backup + ** structure. */ sqlite3_free(p); p = 0; } diff --git a/test/backup.test b/test/backup.test index 444619c68c..3b1e1db9e5 100644 --- a/test/backup.test +++ b/test/backup.test @@ -217,6 +217,7 @@ foreach nPagePerStep {1 200} { INSERT INTO ${file_dest}.t1 VALUES(1, randstr(1000,1000)) " $db_dest } + execsql COMMIT $db_dest } # Backup the source database. diff --git a/test/backup5.test b/test/backup5.test new file mode 100644 index 0000000000..c789adfa69 --- /dev/null +++ b/test/backup5.test @@ -0,0 +1,65 @@ +# 2014 November 13 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix backup5 + +forcedelete test2.db + +do_execsql_test 1.0 { + CREATE TABLE t1(a, b); + CREATE TABLE t2(a, b); + INSERT INTO t2 VALUES(1, 1); + INSERT INTO t2 VALUES(2, 2); + INSERT INTO t2 VALUES(3, 3); +} + +do_test 1.1 { + forcecopy test.db test.db2 + db eval { + DROP TABLE t2; + INSERT INTO t1 VALUES(zeroblob(1000), zeroblob(1000)); + INSERT INTO t1 VALUES(randomblob(1000), randomblob(1000)); + } +} {} + +do_test 1.2 { + sqlite3 db2 test.db2 + set stmt [sqlite3_prepare_v2 db2 "SELECT * FROM t2" -1 dummy] + sqlite3_step $stmt +} {SQLITE_ROW} + +do_test 1.3 { + list [catch { sqlite3_backup B db2 main db main } msg] $msg +} {1 {sqlite3_backup_init() failed}} + +do_test 1.4 { + sqlite3_errmsg db2 +} {destination database is in use} + +do_test 1.5 { + sqlite3_reset $stmt + sqlite3_backup B db2 main db main + B step 200 + B finish +} {SQLITE_OK} + +do_test 1.6 { + list [sqlite3_step $stmt] [sqlite3_finalize $stmt] +} {SQLITE_ERROR SQLITE_ERROR} + +do_test 1.7 { + sqlite3_errmsg db2 +} {no such table: t2} + +finish_test From 8ac1a67eff83ed80083e94deb70a366cf1a9568e Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 13 Nov 2014 14:30:56 +0000 Subject: [PATCH 108/133] Modify the documentation for sqlite3_backup_init() to indicate that it will fail if there is already a read or read-write transaction open on the destination database. FossilOrigin-Name: ef03a203351a6002e2b1075139717e4234c816cd --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/sqlite.h.in | 4 ++++ 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 4f0d48c405..7a15358036 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Have\scalls\sto\ssqlite3_backup_init()\sfail\sif\sthere\sis\salready\sa\sread\sor\sread-write\stransaction\sopen\son\sthe\sdestination\sdatabase. -D 2014-11-13T14:18:25.531 +C Modify\sthe\sdocumentation\sfor\ssqlite3_backup_init()\sto\sindicate\sthat\sit\swill\sfail\sif\sthere\sis\salready\sa\sread\sor\sread-write\stransaction\sopen\son\sthe\sdestination\sdatabase. +D 2014-11-13T14:30:56.983 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -230,7 +230,7 @@ F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c bc28d5992109717c87804e2eb1a08a7c8cc7a2fd -F src/sqlite.h.in 0e6612f84936cca29166f2c66068e0227a13fdf6 +F src/sqlite.h.in 0c5c0df7e4e436dfc5592511325bf4a96f6a638d F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 71b0bf1a7fc55b5cb374f7579fd140e730a6e0f4 @@ -1221,7 +1221,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P eaf3aae014f59c8d37aa20aa31d54cf13f9e86fc -R cf416376a3614eef6301ba322bd3cc43 +P 169b5505498c0a7ee2b5dbb2ba13c41dfaa7c62f +R 91cc925fca98dfe2ae31ca6f9eebcf27 U dan -Z 499ccc085277a0250fd3c5f22e93f6be +Z 09c231773b0ef1121faa87d2329ad833 diff --git a/manifest.uuid b/manifest.uuid index 73dcaa8447..018b18d5f0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -169b5505498c0a7ee2b5dbb2ba13c41dfaa7c62f \ No newline at end of file +ef03a203351a6002e2b1075139717e4234c816cd \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 2812bf511d..4427f39d06 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -6848,6 +6848,10 @@ typedef struct sqlite3_backup sqlite3_backup; ** must be different or else sqlite3_backup_init(D,N,S,M) will fail with ** an error. ** +** ^A call to sqlite3_backup_init() will fail, returning SQLITE_ERROR, if +** there is already a read or read-write transaction open on the +** destination database. +** ** ^If an error occurs within sqlite3_backup_init(D,N,S,M), then NULL is ** returned and an error code and error message are stored in the ** destination [database connection] D. From dd715f7c57f78f96fa00f8cb299d082f59ddbfb5 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 14 Nov 2014 15:28:33 +0000 Subject: [PATCH 109/133] Do not automatically remove the DISTINCT keyword from "a IN (SELECT DISTINCT ...)" expressions. Fix for [db87229497]. FossilOrigin-Name: 55e453aadbb676dda07f0fa537d39ce184ef636c --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/expr.c | 1 - test/in5.test | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 56 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 7a15358036..1b20e13d73 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Modify\sthe\sdocumentation\sfor\ssqlite3_backup_init()\sto\sindicate\sthat\sit\swill\sfail\sif\sthere\sis\salready\sa\sread\sor\sread-write\stransaction\sopen\son\sthe\sdestination\sdatabase. -D 2014-11-13T14:30:56.983 +C Do\snot\sautomatically\sremove\sthe\sDISTINCT\skeyword\sfrom\s"a\sIN\s(SELECT\sDISTINCT\s...)"\sexpressions.\sFix\sfor\s[db87229497]. +D 2014-11-14T15:28:33.929 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -182,7 +182,7 @@ F src/complete.c c4ba6e0626bb94bc77a0861735f3382fcf7cc818 F src/ctime.c df19848891c8a553c80e6f5a035e768280952d1a F src/date.c 93594514aae68de117ca4a2a0d6cc63eddf26744 F src/delete.c 0750b1eb4d96cd3fb2c798599a3a7c85e92f1417 -F src/expr.c 0391a657df4959eaf2a2fd7d77de5ebe750686ee +F src/expr.c a3ff05db5709d628c23890db862e30f3dd9dc428 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c da985ae673efef2c712caef825a5d2edb087ead7 F src/func.c ba47c1671ab3cfdafa6e9d6ee490939ea578adee @@ -627,7 +627,7 @@ F test/in.test 047c4671328e9032ab95666a67021adbbd36e98e F test/in2.test 5d4c61d17493c832f7d2d32bef785119e87bde75 F test/in3.test 3cbf58c87f4052cee3a58b37b6389777505aa0c0 F test/in4.test d2b38cba404bc4320f4fe1b595b3d163f212c068 -F test/in5.test 99f9a40af01711b06d2d614ecfe96129f334fba3 +F test/in5.test 1de657472fa9ac2924be25c2c959ac5ca1aae554 F test/incrblob.test e81846d214f3637622620fbde7cd526781cfe328 F test/incrblob2.test bf4d549aa4a466d7fbe3e3a3693d3861263d5600 F test/incrblob3.test d8d036fde015d4a159cd3cbae9d29003b37227a4 @@ -1221,7 +1221,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 169b5505498c0a7ee2b5dbb2ba13c41dfaa7c62f -R 91cc925fca98dfe2ae31ca6f9eebcf27 +P ef03a203351a6002e2b1075139717e4234c816cd +R bb54553354aa14551393c5e5701542c8 U dan -Z 09c231773b0ef1121faa87d2329ad833 +Z f91e19e3f2d1e58cdccd86465c86ab70 diff --git a/manifest.uuid b/manifest.uuid index 018b18d5f0..c5dbef8e09 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ef03a203351a6002e2b1075139717e4234c816cd \ No newline at end of file +55e453aadbb676dda07f0fa537d39ce184ef636c \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 13a9cb46fd..25f0be400f 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1857,7 +1857,6 @@ int sqlite3CodeSubselect( assert( (pExpr->iTable&0x0000FFFF)==pExpr->iTable ); pSelect->iLimit = 0; testcase( pSelect->selFlags & SF_Distinct ); - pSelect->selFlags &= ~SF_Distinct; testcase( pKeyInfo==0 ); /* Caused by OOM in sqlite3KeyInfoAlloc() */ if( sqlite3Select(pParse, pSelect, &dest) ){ sqlite3KeyInfoUnref(pKeyInfo); diff --git a/test/in5.test b/test/in5.test index 8a43b8d44a..67d212589d 100644 --- a/test/in5.test +++ b/test/in5.test @@ -12,6 +12,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +set testprefix in5 do_test in5-1.1 { execsql { @@ -135,4 +136,51 @@ do_test in5-5.3 { }] } {0} +#------------------------------------------------------------------------- +# At one point SQLite was removing the DISTINCT keyword from expressions +# similar to: +# +# IN (SELECT DISTINCT FROM...) +# +# However, there are a few obscure cases where this is incorrect. For +# example, if the SELECT features a LIMIT clause, or if the collation +# sequence or affinity used by the DISTINCT does not match the one used +# by the IN(...) expression. +# +do_execsql_test 6.1.1 { + CREATE TABLE t1(a COLLATE nocase); + INSERT INTO t1 VALUES('one'); + INSERT INTO t1 VALUES('ONE'); +} +do_execsql_test 6.1.2 { + SELECT count(*) FROM t1 WHERE a COLLATE BINARY IN (SELECT DISTINCT a FROM t1) +} {1} + +do_execsql_test 6.2.1 { + CREATE TABLE t3(a, b); + INSERT INTO t3 VALUES(1, 1); + INSERT INTO t3 VALUES(1, 2); + INSERT INTO t3 VALUES(1, 3); + INSERT INTO t3 VALUES(2, 4); + INSERT INTO t3 VALUES(2, 5); + INSERT INTO t3 VALUES(2, 6); + INSERT INTO t3 VALUES(3, 7); + INSERT INTO t3 VALUES(3, 8); + INSERT INTO t3 VALUES(3, 9); +} +do_execsql_test 6.2.2 { + SELECT count(*) FROM t3 WHERE b IN (SELECT DISTINCT a FROM t3 LIMIT 5); +} {3} +do_execsql_test 6.2.3 { + SELECT count(*) FROM t3 WHERE b IN (SELECT a FROM t3 LIMIT 5); +} {2} + +do_execsql_test 6.3.1 { + CREATE TABLE x1(a); + CREATE TABLE x2(b); + INSERT INTO x1 VALUES(1), (1), (2); + INSERT INTO x2 VALUES(1), (2); + SELECT count(*) FROM x2 WHERE b IN (SELECT DISTINCT a FROM x1 LIMIT 2); +} {2} + finish_test From 0a9d9d566e24d05afcfdd15368eccb8683331ecc Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 14 Nov 2014 15:42:23 +0000 Subject: [PATCH 110/133] Do not automatically remove the DISTINCT keyword from "a IN (SELECT DISTINCT ...)" expressions. Fix for [db87229497]. FossilOrigin-Name: 98457a57d642b35917eb9ad8f70065e273aad206 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/expr.c | 1 - test/in5.test | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 57 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 1d9fe4d78f..a2350554db 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\sa\stransaction\sor\ssavepoint\srollback\soccurs,\ssave\sthe\spositions\sof\sall\sopen\sread-cursors\sso\sthat\sthey\scan\sbe\srestored\sfollowing\sthe\srollback\soperation.\s\sCherry-pick\sof\scheck-in\s[dd03a2802f3f27] -D 2014-11-13T13:42:39.738 +C Do\snot\sautomatically\sremove\sthe\sDISTINCT\skeyword\sfrom\s"a\sIN\s(SELECT\sDISTINCT\s...)"\sexpressions.\sFix\sfor\s[db87229497]. +D 2014-11-14T15:42:23.965 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -181,7 +181,7 @@ F src/complete.c 535183afb3c75628b78ce82612931ac7cdf26f14 F src/ctime.c bb434068b5308a857b181c2d204a320ff0d6c638 F src/date.c 57a7f9ba9f6b4d5268f5e411739066a611f99036 F src/delete.c 0750b1eb4d96cd3fb2c798599a3a7c85e92f1417 -F src/expr.c fc204d08af06437ddaffe5a1b1f1f6f9e1a55d6d +F src/expr.c 1891cb50510a31e96de8a54579e7d3aef60f0094 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c da985ae673efef2c712caef825a5d2edb087ead7 F src/func.c ba47c1671ab3cfdafa6e9d6ee490939ea578adee @@ -616,7 +616,7 @@ F test/in.test 047c4671328e9032ab95666a67021adbbd36e98e F test/in2.test 5d4c61d17493c832f7d2d32bef785119e87bde75 F test/in3.test 3cbf58c87f4052cee3a58b37b6389777505aa0c0 F test/in4.test d2b38cba404bc4320f4fe1b595b3d163f212c068 -F test/in5.test 99f9a40af01711b06d2d614ecfe96129f334fba3 +F test/in5.test 1de657472fa9ac2924be25c2c959ac5ca1aae554 F test/incrblob.test e81846d214f3637622620fbde7cd526781cfe328 F test/incrblob2.test bf4d549aa4a466d7fbe3e3a3693d3861263d5600 F test/incrblob3.test d8d036fde015d4a159cd3cbae9d29003b37227a4 @@ -1205,8 +1205,8 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 839a6df9f98b90fb593534a62145d9c913540bae -Q +dd03a2802f3f276525f3cef9a93f825dd8606626 -R 0801523eb21d9763755a3afd3e84ae8b +P 402780a9c8df9e7ea898bdca49c1191042fe387a +Q +55e453aadbb676dda07f0fa537d39ce184ef636c +R 0bf0d6ff158ba2352191f711d5e9a856 U drh -Z 995ce8d43edc83cec464fe7495c36085 +Z fc11fd8e7149b148ee2e3386efe4bde7 diff --git a/manifest.uuid b/manifest.uuid index 38b0ca2922..8df489121d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -402780a9c8df9e7ea898bdca49c1191042fe387a \ No newline at end of file +98457a57d642b35917eb9ad8f70065e273aad206 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 1ad9a879a3..881cfd4344 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1836,7 +1836,6 @@ int sqlite3CodeSubselect( assert( (pExpr->iTable&0x0000FFFF)==pExpr->iTable ); pSelect->iLimit = 0; testcase( pSelect->selFlags & SF_Distinct ); - pSelect->selFlags &= ~SF_Distinct; testcase( pKeyInfo==0 ); /* Caused by OOM in sqlite3KeyInfoAlloc() */ if( sqlite3Select(pParse, pSelect, &dest) ){ sqlite3KeyInfoUnref(pKeyInfo); diff --git a/test/in5.test b/test/in5.test index 8a43b8d44a..67d212589d 100644 --- a/test/in5.test +++ b/test/in5.test @@ -12,6 +12,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +set testprefix in5 do_test in5-1.1 { execsql { @@ -135,4 +136,51 @@ do_test in5-5.3 { }] } {0} +#------------------------------------------------------------------------- +# At one point SQLite was removing the DISTINCT keyword from expressions +# similar to: +# +# IN (SELECT DISTINCT FROM...) +# +# However, there are a few obscure cases where this is incorrect. For +# example, if the SELECT features a LIMIT clause, or if the collation +# sequence or affinity used by the DISTINCT does not match the one used +# by the IN(...) expression. +# +do_execsql_test 6.1.1 { + CREATE TABLE t1(a COLLATE nocase); + INSERT INTO t1 VALUES('one'); + INSERT INTO t1 VALUES('ONE'); +} +do_execsql_test 6.1.2 { + SELECT count(*) FROM t1 WHERE a COLLATE BINARY IN (SELECT DISTINCT a FROM t1) +} {1} + +do_execsql_test 6.2.1 { + CREATE TABLE t3(a, b); + INSERT INTO t3 VALUES(1, 1); + INSERT INTO t3 VALUES(1, 2); + INSERT INTO t3 VALUES(1, 3); + INSERT INTO t3 VALUES(2, 4); + INSERT INTO t3 VALUES(2, 5); + INSERT INTO t3 VALUES(2, 6); + INSERT INTO t3 VALUES(3, 7); + INSERT INTO t3 VALUES(3, 8); + INSERT INTO t3 VALUES(3, 9); +} +do_execsql_test 6.2.2 { + SELECT count(*) FROM t3 WHERE b IN (SELECT DISTINCT a FROM t3 LIMIT 5); +} {3} +do_execsql_test 6.2.3 { + SELECT count(*) FROM t3 WHERE b IN (SELECT a FROM t3 LIMIT 5); +} {2} + +do_execsql_test 6.3.1 { + CREATE TABLE x1(a); + CREATE TABLE x2(b); + INSERT INTO x1 VALUES(1), (1), (2); + INSERT INTO x2 VALUES(1), (2); + SELECT count(*) FROM x2 WHERE b IN (SELECT DISTINCT a FROM x1 LIMIT 2); +} {2} + finish_test From 70273d0babf9b470539d443e8714a1ad947d197e Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 14 Nov 2014 19:34:20 +0000 Subject: [PATCH 111/133] Consider using an automatic-index for a scan even if there exists a possible skip-scan that uses one or more "=" operators. FossilOrigin-Name: 93642a65ef3d53ece322ffd85233b68fc9a86c9d --- manifest | 17 ++++++++++------- manifest.uuid | 2 +- src/where.c | 3 ++- test/autoindex3.test | 34 ++++++++++++++++++++++++++++++++++ 4 files changed, 47 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 1b20e13d73..48ea588135 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\sautomatically\sremove\sthe\sDISTINCT\skeyword\sfrom\s"a\sIN\s(SELECT\sDISTINCT\s...)"\sexpressions.\sFix\sfor\s[db87229497]. -D 2014-11-14T15:28:33.929 +C Consider\susing\san\sautomatic-index\sfor\sa\sscan\seven\sif\sthere\sexists\sa\spossible\sskip-scan\sthat\suses\sone\sor\smore\s"="\soperators. +D 2014-11-14T19:34:20.967 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -304,7 +304,7 @@ F src/vtab.c 2a30791bbd7926b589401bd09c3abb33de563793 F src/wal.c fa090966140602f03a621f87d82ee69e66ca63b5 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 3862a1173ae2716bde12f1ab3fb649f1d85b05c2 +F src/where.c 4d72a350fd5eec080583f95044f0f394590d13a3 F src/whereInt.h d3633e9b592103241b74b0ec76185f3e5b8b62e0 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -347,7 +347,7 @@ F test/auth3.test 5cfa94ed90c6617c42b7ba4b133fd79678b251c7 F test/autoinc.test c58912526998a39e11f66b533e23cfabea7f25b7 F test/autoindex1.test 6ff78b94f43a59616c06c11c55b12935173506d7 F test/autoindex2.test 60d2fc6f38364308ce73a9beb01b47ded38697de -F test/autoindex3.test 8254f689c3241081fad52b7bea18ba53e07e14a2 +F test/autoindex3.test a3be0d1a53a7d2edff208a5e442312957047e972 F test/autoindex4.test fc807f9efd158bec60f5dfdf34ebe46fb274612d F test/autovacuum.test 941892505d2c0f410a0cb5970dfa1c7c4e5f6e74 F test/autovacuum_ioerr2.test 8a367b224183ad801e0e24dcb7d1501f45f244b4 @@ -1221,7 +1221,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ef03a203351a6002e2b1075139717e4234c816cd -R bb54553354aa14551393c5e5701542c8 +P 55e453aadbb676dda07f0fa537d39ce184ef636c +R 0a82574686b12650ce6ac2a5e463073b +T *branch * experimental-autoindex-fix +T *sym-experimental-autoindex-fix * +T -sym-trunk * U dan -Z f91e19e3f2d1e58cdccd86465c86ab70 +Z 13c044e2b6669432b7a0bb70427e69a9 diff --git a/manifest.uuid b/manifest.uuid index c5dbef8e09..769887276c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -55e453aadbb676dda07f0fa537d39ce184ef636c \ No newline at end of file +93642a65ef3d53ece322ffd85233b68fc9a86c9d \ No newline at end of file diff --git a/src/where.c b/src/where.c index c3641c7cc7..2a06e3f479 100644 --- a/src/where.c +++ b/src/where.c @@ -4130,8 +4130,9 @@ static WhereLoop **whereLoopFindLesser( /* Any loop using an appliation-defined index (or PRIMARY KEY or ** UNIQUE constraint) with one or more == constraints is better - ** than an automatic index. */ + ** than an automatic index. Unless it is a skip-scan. */ if( (p->wsFlags & WHERE_AUTO_INDEX)!=0 + && (pTemplate->nSkip)==0 && (pTemplate->wsFlags & WHERE_INDEXED)!=0 && (pTemplate->wsFlags & WHERE_COLUMN_EQ)!=0 && (p->prereq & pTemplate->prereq)==pTemplate->prereq diff --git a/test/autoindex3.test b/test/autoindex3.test index 33053bba67..c99a175c6d 100644 --- a/test/autoindex3.test +++ b/test/autoindex3.test @@ -17,6 +17,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +set testprefix autoindex3 # The t1b and t2d indexes are not very selective. It used to be that # the autoindex mechanism would create automatic indexes on t1(b) or @@ -54,5 +55,38 @@ do_execsql_test autoindex3-140 { EXPLAIN QUERY PLAN SELECT * FROM t1, t2 WHERE d IN (5,b) AND x=y; } {/AUTO/} +reset_db +do_execsql_test 210 { + CREATE TABLE v(b, d, e); + CREATE TABLE u(a, b, c); + ANALYZE sqlite_master; + INSERT INTO "sqlite_stat1" VALUES('u','uab','40000 400 1'); + INSERT INTO "sqlite_stat1" VALUES('v','vbde','40000 400 1 1'); + INSERT INTO "sqlite_stat1" VALUES('v','ve','40000 21'); + + CREATE INDEX uab on u(a, b); + CREATE INDEX ve on v(e); + CREATE INDEX vbde on v(b,d,e); + + DROP TABLE IF EXISTS sqlite_stat4; + ANALYZE sqlite_master; +} + +# At one point, SQLite was using the inferior plan: +# +# 0|0|1|SEARCH TABLE v USING INDEX ve (e>?) +# 0|1|0|SEARCH TABLE u USING COVERING INDEX uab (ANY(a) AND b=?) +# +# on the basis that the real index "uab" must be better than the automatic +# index. This is not right - a skip-scan is not necessarily better than an +# automatic index scan. +# +do_eqp_test 220 { + select count(*) from u, v where u.b = v.b and v.e > 34; +} { + 0 0 1 {SEARCH TABLE v USING INDEX ve (e>?)} + 0 1 0 {SEARCH TABLE u USING AUTOMATIC COVERING INDEX (b=?)} +} + finish_test From f9df2fbdcd43eb7f223da88ce8cdcf70766bd945 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 15 Nov 2014 19:08:13 +0000 Subject: [PATCH 112/133] Adding the "noskipscan" token to an sqlite_stat1.stat field prevents an index for being used with the skip-scan algorithm. FossilOrigin-Name: 00fe09505792cd0d104b2da9d040f023e30fa871 --- manifest | 23 +++++++++++++---------- manifest.uuid | 2 +- src/analyze.c | 2 ++ src/sqliteInt.h | 1 + src/where.c | 1 + test/skipscan1.test | 19 +++++++++++++++++++ 6 files changed, 37 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 1b20e13d73..1c607eedd4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\sautomatically\sremove\sthe\sDISTINCT\skeyword\sfrom\s"a\sIN\s(SELECT\sDISTINCT\s...)"\sexpressions.\sFix\sfor\s[db87229497]. -D 2014-11-14T15:28:33.929 +C Adding\sthe\s"noskipscan"\stoken\sto\san\ssqlite_stat1.stat\sfield\sprevents\san\nindex\sfor\sbeing\sused\swith\sthe\sskip-scan\salgorithm. +D 2014-11-15T19:08:13.305 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -167,7 +167,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c ba266a779bc7ce10e52e59e7d3dc79fa342e8fdb -F src/analyze.c afbcca663c3f3625340b8e30d440cd7a97ded6bc +F src/analyze.c 951fd859852dfbced4a58b73954f168eeb9d0772 F src/attach.c f4e94df2d1826feda65eb0939f7f6f5f923a0ad9 F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 7ddee9c7d505e07e959a575b18498f17c71e53ea @@ -233,7 +233,7 @@ F src/shell.c bc28d5992109717c87804e2eb1a08a7c8cc7a2fd F src/sqlite.h.in 0c5c0df7e4e436dfc5592511325bf4a96f6a638d F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d -F src/sqliteInt.h 71b0bf1a7fc55b5cb374f7579fd140e730a6e0f4 +F src/sqliteInt.h c9e95b8fa9aee30d46387735c5be73fa58886e38 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 81712116e826b0089bb221b018929536b2b5406f F src/table.c f142bba7903e93ca8d113a5b8877a108ad1a27dc @@ -304,7 +304,7 @@ F src/vtab.c 2a30791bbd7926b589401bd09c3abb33de563793 F src/wal.c fa090966140602f03a621f87d82ee69e66ca63b5 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 3862a1173ae2716bde12f1ab3fb649f1d85b05c2 +F src/where.c cb89128d24ddb5fe3ca290166d66968f2dc0c3c8 F src/whereInt.h d3633e9b592103241b74b0ec76185f3e5b8b62e0 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -855,7 +855,7 @@ F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3 F test/show_speedtest1_rtree.tcl 32e6c5f073d7426148a6936a0408f4b5b169aba5 F test/shrink.test 8c70f62b6e8eb4d54533de6d65bd06b1b9a17868 F test/sidedelete.test f0ad71abe6233e3b153100f3b8d679b19a488329 -F test/skipscan1.test 7e15e1cc524524e7b2c4595ec85c75501d22f4ff +F test/skipscan1.test 2ddfe5d168462170c4487f534e2a99fb006b2076 F test/skipscan2.test d1d1450952b7275f0b0a3a981f0230532743951a F test/skipscan3.test ec5bab3f81c7038b43450e7b3062e04a198bdbb5 F test/skipscan5.test 67817a4b6857c47e0e33ba3e506da6f23ef68de2 @@ -1221,7 +1221,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ef03a203351a6002e2b1075139717e4234c816cd -R bb54553354aa14551393c5e5701542c8 -U dan -Z f91e19e3f2d1e58cdccd86465c86ab70 +P 55e453aadbb676dda07f0fa537d39ce184ef636c +R 59717354988087765927b6c6b6246253 +T *branch * noskipscan-token +T *sym-noskipscan-token * +T -sym-trunk * +U drh +Z 9d226c4fa6bf82a1f1a22a271191f33c diff --git a/manifest.uuid b/manifest.uuid index c5dbef8e09..1ab180da36 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -55e453aadbb676dda07f0fa537d39ce184ef636c \ No newline at end of file +00fe09505792cd0d104b2da9d040f023e30fa871 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 597885237c..769a16565a 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1466,6 +1466,8 @@ static void decodeIntArray( pIndex->bUnordered = 1; }else if( sqlite3_strglob("sz=[0-9]*", z)==0 ){ pIndex->szIdxRow = sqlite3LogEst(sqlite3Atoi(z+3)); + }else if( sqlite3_strglob("noskipscan*", z)==0 ){ + pIndex->noSkipScan = 1; } #ifdef SQLITE_ENABLE_COSTMULT else if( sqlite3_strglob("costmult=[0-9]*",z)==0 ){ diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 4d272b06a1..f3d6ce015e 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1795,6 +1795,7 @@ struct Index { unsigned uniqNotNull:1; /* True if UNIQUE and NOT NULL for all columns */ unsigned isResized:1; /* True if resizeIndexObject() has been called */ unsigned isCovering:1; /* True if this is a covering index */ + unsigned noSkipScan:1; /* Do not try to use skip-scan if true */ #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 int nSample; /* Number of elements in aSample[] */ int nSampleCol; /* Size of IndexSample.anEq[] and so on */ diff --git a/src/where.c b/src/where.c index c3641c7cc7..db6cf94dea 100644 --- a/src/where.c +++ b/src/where.c @@ -4587,6 +4587,7 @@ static int whereLoopAddBtreeIndex( assert( 42==sqlite3LogEst(18) ); if( saved_nEq==saved_nSkip && saved_nEq+1nKeyCol + && pProbe->noSkipScan==0 && pProbe->aiRowLogEst[saved_nEq+1]>=42 /* TUNING: Minimum for skip-scan */ && (rc = whereLoopResize(db, pNew, pNew->nLTerm+1))==SQLITE_OK ){ diff --git a/test/skipscan1.test b/test/skipscan1.test index 6b9f1209a5..4f996df972 100644 --- a/test/skipscan1.test +++ b/test/skipscan1.test @@ -273,4 +273,23 @@ do_execsql_test skipscan1-6.3 { EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE b=1; } {~/ANY/} +# If the sqlite_stat1 entry includes the "noskipscan" token, then never use +# skipscan with that index. +# +do_execsql_test skipscan1-7.1 { + UPDATE sqlite_stat1 SET stat='500000 125000 1 sz=100'; + ANALYZE sqlite_master; + EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE b=1; +} {/ANY/} +do_execsql_test skipscan1-7.2 { + UPDATE sqlite_stat1 SET stat='500000 125000 1 noskipscan sz=100'; + ANALYZE sqlite_master; + EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE b=1; +} {~/ANY/} +do_execsql_test skipscan1-7.3 { + UPDATE sqlite_stat1 SET stat='500000 125000 1 sz=100 noskipscan'; + ANALYZE sqlite_master; + EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE b=1; +} {~/ANY/} + finish_test From 302f583aa4dc6a77ca9042dca3143a3bf8a134f0 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 17 Nov 2014 15:22:08 +0000 Subject: [PATCH 113/133] Update a couple of test cases to account for the fact that ROLLBACK does not always abort all running SELECT statements. FossilOrigin-Name: eba171e980fa4491dfee9d7e4df50c87a0ebbf87 --- manifest | 17 ++++++++--------- manifest.uuid | 2 +- test/ioerr2.test | 8 +++++++- test/shared_err.test | 8 +++++++- 4 files changed, 23 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index a2350554db..27ecc3c872 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\sautomatically\sremove\sthe\sDISTINCT\skeyword\sfrom\s"a\sIN\s(SELECT\sDISTINCT\s...)"\sexpressions.\sFix\sfor\s[db87229497]. -D 2014-11-14T15:42:23.965 +C Update\sa\scouple\sof\stest\scases\sto\saccount\sfor\sthe\sfact\sthat\sROLLBACK\sdoes\snot\salways\sabort\sall\srunning\sSELECT\sstatements. +D 2014-11-17T15:22:08.993 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -648,7 +648,7 @@ F test/interrupt.test dfe9a67a94b0b2d8f70545ba1a6cca10780d71cc F test/intpkey.test 7506090fc08e028712a8bf47e5f54111947e3844 F test/io.test 3a7abcef18727cc0f2399e04b0e8903eccae50f8 F test/ioerr.test 2a24bd6ed5a8b062e64bfe1f6cf94fb25e92210d -F test/ioerr2.test 9d71166f8466eda510f1af6137bdabaa82b5408d +F test/ioerr2.test 2593563599e2cc6b6b4fcf5878b177bdd5d8df26 F test/ioerr3.test d3cec5e1a11ad6d27527d0d38573fbff14c71bdd F test/ioerr4.test f130fe9e71008577b342b8874d52984bd04ede2c F test/ioerr5.test 2edfa4fb0f896f733071303b42224df8bedd9da4 @@ -830,7 +830,7 @@ F test/shared7.test a81e99f83e6c51b02ac99c96fb3a2a7b5978c956 F test/shared8.test 00a07bf5e1337ecf72e94542bdefdc330d7a2538 F test/shared9.test 5f2a8f79b4d6c7d107a01ffa1ed05ae7e6333e21 F test/sharedA.test 0cdf1a76dfa00e6beee66af5b534b1e8df2720f5 -F test/shared_err.test 0079c05c97d88cfa03989b7c20a8b266983087aa +F test/shared_err.test 2f2aee20db294b9924e81f6ccbe60f19e21e8506 F test/sharedlock.test 5ede3c37439067c43b0198f580fd374ebf15d304 F test/shell1.test d60946b5fde4d85fe06db7331dfe89011f564350 F test/shell2.test c57da3a381c099b02c813ba156298d5c2f5c93a3 @@ -1205,8 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 402780a9c8df9e7ea898bdca49c1191042fe387a -Q +55e453aadbb676dda07f0fa537d39ce184ef636c -R 0bf0d6ff158ba2352191f711d5e9a856 -U drh -Z fc11fd8e7149b148ee2e3386efe4bde7 +P 98457a57d642b35917eb9ad8f70065e273aad206 +R d8acddabccfe794436f7272593be21d9 +U dan +Z 30deb4af13a2947d821f81a6f506118a diff --git a/manifest.uuid b/manifest.uuid index 8df489121d..bf9ab0f15d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -98457a57d642b35917eb9ad8f70065e273aad206 \ No newline at end of file +eba171e980fa4491dfee9d7e4df50c87a0ebbf87 \ No newline at end of file diff --git a/test/ioerr2.test b/test/ioerr2.test index 5150ace3ab..c08c3453f3 100644 --- a/test/ioerr2.test +++ b/test/ioerr2.test @@ -112,6 +112,12 @@ foreach bPersist [list 0 1] { } } +# When this test was written, an IO error within the UPDATE statement caused +# a rollback, which tripped all read-cursors, causing the outer SELECT to +# fail with "abort due to ROLLBACK". Now, the loop continues until the UPDATE +# is run successfully. At this point the next IO error occurs within the +# SELECT - throwing the "disk I/O error" that the test case now expects. +# do_test ioerr2-5 { execsql { CREATE TABLE t2 AS SELECT * FROM t1; @@ -130,7 +136,7 @@ do_test ioerr2-5 { } } msg] list $rc $msg -} {1 {abort due to ROLLBACK}} +} {1 {disk I/O error}} ;# used to be "{1 {abort due to ROLLBACK}}" if {$::tcl_platform(platform) == "unix"} { # Cause the call to xAccess used by [pragma temp_store_directory] to diff --git a/test/shared_err.test b/test/shared_err.test index 17add94bb7..96e5ee4540 100644 --- a/test/shared_err.test +++ b/test/shared_err.test @@ -446,9 +446,15 @@ do_malloc_test shared_err-8 -tclprep { } {1} db2 close } + +# When this test case was written, OOM errors in write statements would +# cause transaction rollback, which would trip cursors in other statements, +# aborting them. This no longer happens. +# do_test shared_malloc-8.X { # Test that one or more queries were aborted due to the malloc() failure. - expr $::aborted>=1 + # expr $::aborted>=1 + expr $::aborted==0 } {1} # This test is designed to catch a specific bug that was present during From 13835c41e824eb2b6a7d504c4df2066fe4ad6c52 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 17 Nov 2014 15:32:47 +0000 Subject: [PATCH 114/133] Fix a bug in the sqlite3TripAllCursors() routine that prevents it from reporting errors. It is unknown at this time whether or not this omission can result in any incorrect result in an actual query. FossilOrigin-Name: 42588207ff5451cb785c394633e1ab631fb82f01 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/btree.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 27ecc3c872..a0d49916be 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sa\scouple\sof\stest\scases\sto\saccount\sfor\sthe\sfact\sthat\sROLLBACK\sdoes\snot\salways\sabort\sall\srunning\sSELECT\sstatements. -D 2014-11-17T15:22:08.993 +C Fix\sa\sbug\sin\sthe\ssqlite3TripAllCursors()\sroutine\sthat\sprevents\sit\sfrom\nreporting\serrors.\s\sIt\sis\sunknown\sat\sthis\stime\swhether\sor\snot\sthis\somission\scan\nresult\sin\sany\sincorrect\sresult\sin\san\sactual\squery. +D 2014-11-17T15:32:47.066 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c 8cdfeb0c8a6d8bdad3faefae418eb3dc767051b6 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 2c15850c5c9a26b10cdf92f9a29c74e299dc3674 +F src/btree.c 2087125172421c78fc0c02f953b7d660f2a2598e F src/btree.h a4afc6b06f5a1dd2076d15aa168baec44fc0121b F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 9dc2bd94347b878c89627000c92b0c8d97ec2919 @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 98457a57d642b35917eb9ad8f70065e273aad206 -R d8acddabccfe794436f7272593be21d9 -U dan -Z 30deb4af13a2947d821f81a6f506118a +P eba171e980fa4491dfee9d7e4df50c87a0ebbf87 +R c97ad964463de041aa0a8f758aac4ee1 +U drh +Z ed993b1ef595900db9792d9ac234a3c7 diff --git a/manifest.uuid b/manifest.uuid index bf9ab0f15d..7891ecfbd0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -eba171e980fa4491dfee9d7e4df50c87a0ebbf87 \ No newline at end of file +42588207ff5451cb785c394633e1ab631fb82f01 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 41e097af53..111870548b 100644 --- a/src/btree.c +++ b/src/btree.c @@ -3499,7 +3499,7 @@ int sqlite3BtreeTripAllCursors(Btree *pBtree, int errCode, int writeOnly){ int i; if( writeOnly && (p->curFlags & BTCF_WriteFlag)==0 ){ if( p->eState==CURSOR_VALID ){ - int rc = saveCursorPosition(p); + rc = saveCursorPosition(p); if( rc!=SQLITE_OK ){ (void)sqlite3BtreeTripAllCursors(pBtree, rc, 0); break; From 77b1deec400c40676dfca2986dbf659af548c40f Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 17 Nov 2014 17:13:06 +0000 Subject: [PATCH 115/133] When a SELECT statement is terminated by a ROLLBACK TO operation, make the error message be "abort due to ROLLBACK" rather than "callback requested query abort". FossilOrigin-Name: 34fc4a082c192830e48f643549c04a4f91912b8b --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbe.c | 3 ++- test/savepoint7.test | 4 ++-- 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index a0d49916be..93ef894b6e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sbug\sin\sthe\ssqlite3TripAllCursors()\sroutine\sthat\sprevents\sit\sfrom\nreporting\serrors.\s\sIt\sis\sunknown\sat\sthis\stime\swhether\sor\snot\sthis\somission\scan\nresult\sin\sany\sincorrect\sresult\sin\san\sactual\squery. -D 2014-11-17T15:32:47.066 +C When\sa\sSELECT\sstatement\sis\sterminated\sby\sa\sROLLBACK\sTO\soperation,\smake\sthe\nerror\smessage\sbe\s"abort\sdue\sto\sROLLBACK"\srather\sthan\s\n"callback\srequested\squery\sabort". +D 2014-11-17T17:13:06.964 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/update.c 3c4ecc282accf12d39edb8d524cf089645e55a13 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 4006c01772bd8d8ac4306d523bbcee41d3e392d8 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c a6b604364c7cbb079c083418e7359d1d665f2ef0 +F src/vdbe.c 3e8f9787d9a5fe24077f7af0baa2d11042471f12 F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 F src/vdbeInt.h e2a060a55ee18a6ab973353a5e2ec7ee569bf787 F src/vdbeapi.c 37a6c6ae284a97bcace365f2f0a225680c0499d9 @@ -797,7 +797,7 @@ F test/savepoint3.test e328085853b14898d78ceea00dfe7db18bb6a9ec F test/savepoint4.test c8f8159ade6d2acd9128be61e1230f1c1edc6cc0 F test/savepoint5.test 0735db177e0ebbaedc39812c8d065075d563c4fd F test/savepoint6.test f41279c5e137139fa5c21485773332c7adb98cd7 -F test/savepoint7.test 1c8f26b1e2a4221b0214e222ce12a97a59918eb2 +F test/savepoint7.test db3db281486c925095f305aad09fe806e5188ff3 F test/schema.test 8f7999be894260f151adf15c2c7540f1c6d6a481 F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5 F test/schema3.test 1bc1008e1f8cb5654b248c55f27249366eb7ed38 @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P eba171e980fa4491dfee9d7e4df50c87a0ebbf87 -R c97ad964463de041aa0a8f758aac4ee1 +P 42588207ff5451cb785c394633e1ab631fb82f01 +R dfbcaabb48c2f5c435e2c7179a62bd27 U drh -Z ed993b1ef595900db9792d9ac234a3c7 +Z c435b7b174ccd5699f4e15dc48114455 diff --git a/manifest.uuid b/manifest.uuid index 7891ecfbd0..d4b77b5f9f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -42588207ff5451cb785c394633e1ab631fb82f01 \ No newline at end of file +34fc4a082c192830e48f643549c04a4f91912b8b \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 1ad8aab753..9a8db1a638 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2827,7 +2827,8 @@ case OP_Savepoint: { if( p1==SAVEPOINT_ROLLBACK ){ isSchemaChange = (db->flags & SQLITE_InternChanges)!=0; for(ii=0; iinDb; ii++){ - rc = sqlite3BtreeTripAllCursors(db->aDb[ii].pBt, SQLITE_ABORT, + rc = sqlite3BtreeTripAllCursors(db->aDb[ii].pBt, + SQLITE_ABORT_ROLLBACK, isSchemaChange==0); if( rc!=SQLITE_OK ) goto abort_due_to_error; } diff --git a/test/savepoint7.test b/test/savepoint7.test index 908ec571f5..d8a02f1f80 100644 --- a/test/savepoint7.test +++ b/test/savepoint7.test @@ -78,7 +78,7 @@ do_test savepoint7-2.1 { } msg] db eval {RELEASE x1} list $rc $msg [db eval {SELECT * FROM t2}] -} {1 {callback requested query abort} {}} +} {1 {abort due to ROLLBACK} {}} do_test savepoint7-2.2 { db eval {DELETE FROM t2;} @@ -93,6 +93,6 @@ do_test savepoint7-2.2 { } } msg] list $rc $msg [db eval {SELECT * FROM t2}] -} {1 {callback requested query abort} {}} +} {1 {abort due to ROLLBACK} {}} finish_test From d816e00d13e0f1bb4704086760e0a394a61e9981 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 17 Nov 2014 19:25:15 +0000 Subject: [PATCH 116/133] Improved comments on the BtCursor.skipNext field. No changes to code. FossilOrigin-Name: e956e7db057d1112badf5e0671cea95201385b44 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btreeInt.h | 10 ++++++++-- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 93ef894b6e..d851cc2e87 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\sa\sSELECT\sstatement\sis\sterminated\sby\sa\sROLLBACK\sTO\soperation,\smake\sthe\nerror\smessage\sbe\s"abort\sdue\sto\sROLLBACK"\srather\sthan\s\n"callback\srequested\squery\sabort". -D 2014-11-17T17:13:06.964 +C Improved\scomments\son\sthe\sBtCursor.skipNext\sfield.\s\sNo\schanges\sto\scode. +D 2014-11-17T19:25:15.092 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -174,7 +174,7 @@ F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 F src/btree.c 2087125172421c78fc0c02f953b7d660f2a2598e F src/btree.h a4afc6b06f5a1dd2076d15aa168baec44fc0121b -F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 +F src/btreeInt.h 3363e18fd76f69a27a870b25221b2345b3fd4d21 F src/build.c 9dc2bd94347b878c89627000c92b0c8d97ec2919 F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 F src/complete.c 535183afb3c75628b78ce82612931ac7cdf26f14 @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 42588207ff5451cb785c394633e1ab631fb82f01 -R dfbcaabb48c2f5c435e2c7179a62bd27 +P 34fc4a082c192830e48f643549c04a4f91912b8b +R b667da582f38b5593e8a936d8e9b63b7 U drh -Z c435b7b174ccd5699f4e15dc48114455 +Z 1e479ba79d4cbfb8037d6ba7f0cc99ff diff --git a/manifest.uuid b/manifest.uuid index d4b77b5f9f..a049e1fe86 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -34fc4a082c192830e48f643549c04a4f91912b8b \ No newline at end of file +e956e7db057d1112badf5e0671cea95201385b44 \ No newline at end of file diff --git a/src/btreeInt.h b/src/btreeInt.h index 2368e6c884..a28a6a297e 100644 --- a/src/btreeInt.h +++ b/src/btreeInt.h @@ -489,6 +489,11 @@ struct CellInfo { ** ** Fields in this structure are accessed under the BtShared.mutex ** found at self->pBt->mutex. +** +** skipNext meaning: +** eState==SKIPNEXT && skipNext>0: Next sqlite3BtreeNext() is no-op. +** eState==SKIPNEXT && skipNext<0: Next sqlite3BtreePrevious() is no-op. +** eState==FAULT: Cursor fault with skipNext as error code. */ struct BtCursor { Btree *pBtree; /* The Btree to which this cursor belongs */ @@ -501,7 +506,8 @@ struct BtCursor { void *pKey; /* Saved key that was cursor last known position */ Pgno pgnoRoot; /* The root page of this tree */ int nOvflAlloc; /* Allocated size of aOverflow[] array */ - int skipNext; /* Prev() is noop if negative. Next() is noop if positive */ + int skipNext; /* Prev() is noop if negative. Next() is noop if positive. + ** Error code if eState==CURSOR_FAULT */ u8 curFlags; /* zero or more BTCF_* flags defined below */ u8 eState; /* One of the CURSOR_XXX constants (see below) */ u8 hints; /* As configured by CursorSetHints() */ @@ -547,7 +553,7 @@ struct BtCursor { ** on a different connection that shares the BtShared cache with this ** cursor. The error has left the cache in an inconsistent state. ** Do nothing else with this cursor. Any attempt to use the cursor -** should return the error code stored in BtCursor.skip +** should return the error code stored in BtCursor.skipNext */ #define CURSOR_INVALID 0 #define CURSOR_VALID 1 From 2b8669a9da3296ce0ca99da34f9b158ee1eb4d1e Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 17 Nov 2014 19:42:48 +0000 Subject: [PATCH 117/133] Avoid calling sqlite3BtreeKeysize() on a b-tree cursor in SKIPNEXT or SKIPPREV state. FossilOrigin-Name: 54e7d3fcb1ab21c03ffef1af93ae029a2901098a --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbe.c | 4 ++++ test/misc8.test | 37 +++++++++++++++++++++++++++++++++++++ 4 files changed, 49 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 1b20e13d73..12a819fd2f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\sautomatically\sremove\sthe\sDISTINCT\skeyword\sfrom\s"a\sIN\s(SELECT\sDISTINCT\s...)"\sexpressions.\sFix\sfor\s[db87229497]. -D 2014-11-14T15:28:33.929 +C Avoid\scalling\ssqlite3BtreeKeysize()\son\sa\sb-tree\scursor\sin\sSKIPNEXT\sor\sSKIPPREV\sstate. +D 2014-11-17T19:42:48.262 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/update.c 3c4ecc282accf12d39edb8d524cf089645e55a13 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 3b627daa45c7308c1e36e3dbaa3f9ce7e5c7fa73 F src/vacuum.c 9b30ec729337dd012ed88d4c292922c8ef9cf00c -F src/vdbe.c 5e47308836e9bb5fdb4835fdf88eeab071848d3f +F src/vdbe.c 5d3991d723f00ef86263f4d494e105faba5a5abd F src/vdbe.h 6fc69d9c5e146302c56e163cb4b31d1ee64a18c3 F src/vdbeInt.h 9bb69ff2447c34b6ccc58b34ec35b615f86ead78 F src/vdbeapi.c 07acb615d1e4170e71fc1b0d087f3c53a1ad8e83 @@ -734,7 +734,7 @@ F test/misc4.test 9c078510fbfff05a9869a0b6d8b86a623ad2c4f6 F test/misc5.test 528468b26d03303b1f047146e5eefc941b9069f5 F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test edd0b63e2ee29a256900b0514f6fff27e19e9bb2 -F test/misc8.test e838ec20c9c988bc94812fdb89af26409c20931b +F test/misc8.test fc2754d38892f7dac30c22db3616c2764f117d66 F test/misuse.test 3c34719944ba045cc6c188a4852ba04680728912 F test/mmap1.test 1bfd611b9841eafb44f7d83c0788e146d84a33c9 F test/mmap2.test 9d6dd9ddb4ad2379f29cc78f38ce1e63ed418022 @@ -1221,7 +1221,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ef03a203351a6002e2b1075139717e4234c816cd -R bb54553354aa14551393c5e5701542c8 +P 55e453aadbb676dda07f0fa537d39ce184ef636c +R 6cd25f6196e6f786753b0fb88328cee1 U dan -Z f91e19e3f2d1e58cdccd86465c86ab70 +Z f594bfe54166ed1c0f3ae7a5a057ba04 diff --git a/manifest.uuid b/manifest.uuid index c5dbef8e09..e3e77af8a0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -55e453aadbb676dda07f0fa537d39ce184ef636c \ No newline at end of file +54e7d3fcb1ab21c03ffef1af93ae029a2901098a \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index e4cbeb6565..78c5511e5a 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -4408,6 +4408,10 @@ case OP_Rowid: { /* out2-prerelease */ assert( pC->pCursor!=0 ); rc = sqlite3VdbeCursorRestore(pC); if( rc ) goto abort_due_to_error; + if( pC->nullRow ){ + pOut->flags = MEM_Null; + break; + } rc = sqlite3BtreeKeySize(pC->pCursor, &v); assert( rc==SQLITE_OK ); /* Always so because of CursorRestore() above */ } diff --git a/test/misc8.test b/test/misc8.test index 8c0c126a61..3ff52e56f1 100644 --- a/test/misc8.test +++ b/test/misc8.test @@ -58,4 +58,41 @@ do_catchsql_test misc8-1.7 { } {1 {abort due to ROLLBACK}} +reset_db + +proc dbeval {sql} { db eval $sql } +db func eval dbeval + +do_execsql_test misc8-2.1 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b INTEGER) WITHOUT ROWID; + CREATE TABLE t2(c INTEGER PRIMARY KEY, d INTEGER, x BLOB); + INSERT INTO t1 VALUES(0,0); + INSERT INTO t1 VALUES(10,10); + INSERT INTO t2 VALUES(1,1,zeroblob(200)); + INSERT INTO t2 VALUES(2,2,zeroblob(200)); + INSERT INTO t2 VALUES(3,3,zeroblob(200)); + INSERT INTO t2 VALUES(4,4,zeroblob(200)); + INSERT INTO t2 VALUES(5,5,zeroblob(200)); + INSERT INTO t2 VALUES(6,6,zeroblob(200)); + INSERT INTO t2 VALUES(7,7,zeroblob(200)); + INSERT INTO t2 VALUES(8,8,zeroblob(200)); + INSERT INTO t2 VALUES(9,9,zeroblob(200)); + INSERT INTO t2 VALUES(10,10,zeroblob(200)); + SELECT a, c, eval( + printf('DELETE FROM t2 WHERE c=%d AND %d>5', a+c, a+c) + ) FROM t1, t2; +} { + 0 1 {} 10 1 {} + 0 2 {} 10 2 {} + 0 3 {} 10 3 {} + 0 4 {} 10 4 {} + 0 5 {} 10 5 {} + 0 6 {} 10 {} {} + 0 7 {} 10 {} {} + 0 8 {} 10 {} {} + 0 9 {} 10 {} {} + 0 10 {} 10 {} {} +} + + finish_test From 756d3b3e9d597a3bd04666f4909f43b4da07965e Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 17 Nov 2014 19:44:44 +0000 Subject: [PATCH 118/133] Avoid calling sqlite3BtreeKeysize() on a b-tree cursor in SKIPNEXT or SKIPPREV state. FossilOrigin-Name: 2f2ecb994889acb783616acb7307f8fed962d213 --- manifest | 15 ++++++++------- manifest.uuid | 2 +- src/vdbe.c | 4 ++++ 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index d851cc2e87..55c07fe240 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\scomments\son\sthe\sBtCursor.skipNext\sfield.\s\sNo\schanges\sto\scode. -D 2014-11-17T19:25:15.092 +C Avoid\scalling\ssqlite3BtreeKeysize()\son\sa\sb-tree\scursor\sin\sSKIPNEXT\sor\sSKIPPREV\sstate. +D 2014-11-17T19:44:44.917 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/update.c 3c4ecc282accf12d39edb8d524cf089645e55a13 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 4006c01772bd8d8ac4306d523bbcee41d3e392d8 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c 3e8f9787d9a5fe24077f7af0baa2d11042471f12 +F src/vdbe.c 8bc291aa00646d07dab33047520960ea454c5a2f F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 F src/vdbeInt.h e2a060a55ee18a6ab973353a5e2ec7ee569bf787 F src/vdbeapi.c 37a6c6ae284a97bcace365f2f0a225680c0499d9 @@ -1205,7 +1205,8 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 34fc4a082c192830e48f643549c04a4f91912b8b -R b667da582f38b5593e8a936d8e9b63b7 -U drh -Z 1e479ba79d4cbfb8037d6ba7f0cc99ff +P e956e7db057d1112badf5e0671cea95201385b44 +Q +54e7d3fcb1ab21c03ffef1af93ae029a2901098a +R b1492d8f4cbcc74596556f240b662de2 +U dan +Z 45bbf8c9451b95c29c15305e9bbd4586 diff --git a/manifest.uuid b/manifest.uuid index a049e1fe86..bec82c6bb3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e956e7db057d1112badf5e0671cea95201385b44 \ No newline at end of file +2f2ecb994889acb783616acb7307f8fed962d213 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 9a8db1a638..366c7a0166 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -4405,6 +4405,10 @@ case OP_Rowid: { /* out2-prerelease */ assert( pC->pCursor!=0 ); rc = sqlite3VdbeCursorRestore(pC); if( rc ) goto abort_due_to_error; + if( pC->nullRow ){ + pOut->flags = MEM_Null; + break; + } rc = sqlite3BtreeKeySize(pC->pCursor, &v); assert( rc==SQLITE_OK ); /* Always so because of CursorRestore() above */ } From c5352b996a590c7a9ab8233545c3cb40d3ce125c Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 17 Nov 2014 20:33:07 +0000 Subject: [PATCH 119/133] Remove code from sqlite3BtreeKeySize() made unreachable by the previous check-in. FossilOrigin-Name: 57c4aa988c8eda3cc513c1e5df5804d88bee99a0 --- manifest | 15 +++++++-------- manifest.uuid | 2 +- src/btree.c | 10 +++------- 3 files changed, 11 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 55c07fe240..d98c6b6486 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\scalling\ssqlite3BtreeKeysize()\son\sa\sb-tree\scursor\sin\sSKIPNEXT\sor\sSKIPPREV\sstate. -D 2014-11-17T19:44:44.917 +C Remove\scode\sfrom\ssqlite3BtreeKeySize()\smade\sunreachable\sby\sthe\sprevious\ncheck-in. +D 2014-11-17T20:33:07.735 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c 8cdfeb0c8a6d8bdad3faefae418eb3dc767051b6 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 2087125172421c78fc0c02f953b7d660f2a2598e +F src/btree.c c961588f01bd95d37b90359220c640f9763a3f58 F src/btree.h a4afc6b06f5a1dd2076d15aa168baec44fc0121b F src/btreeInt.h 3363e18fd76f69a27a870b25221b2345b3fd4d21 F src/build.c 9dc2bd94347b878c89627000c92b0c8d97ec2919 @@ -1205,8 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P e956e7db057d1112badf5e0671cea95201385b44 -Q +54e7d3fcb1ab21c03ffef1af93ae029a2901098a -R b1492d8f4cbcc74596556f240b662de2 -U dan -Z 45bbf8c9451b95c29c15305e9bbd4586 +P 2f2ecb994889acb783616acb7307f8fed962d213 +R 3350d2f69701d38a07608c1007a483bc +U drh +Z df1a082a18c57f32c53583132171df67 diff --git a/manifest.uuid b/manifest.uuid index bec82c6bb3..2223a6ffff 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2f2ecb994889acb783616acb7307f8fed962d213 \ No newline at end of file +57c4aa988c8eda3cc513c1e5df5804d88bee99a0 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 111870548b..7ea66e0d3b 100644 --- a/src/btree.c +++ b/src/btree.c @@ -3883,13 +3883,9 @@ int sqlite3BtreeCursorIsValid(BtCursor *pCur){ */ int sqlite3BtreeKeySize(BtCursor *pCur, i64 *pSize){ assert( cursorHoldsMutex(pCur) ); - assert( pCur->eState==CURSOR_INVALID || pCur->eState==CURSOR_VALID ); - if( pCur->eState!=CURSOR_VALID ){ - *pSize = 0; - }else{ - getCellInfo(pCur); - *pSize = pCur->info.nKey; - } + assert( pCur->eState==CURSOR_VALID ); + getCellInfo(pCur); + *pSize = pCur->info.nKey; return SQLITE_OK; } From 4429c20b866e038b155d17a347686abc6102d455 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 18 Nov 2014 02:44:32 +0000 Subject: [PATCH 120/133] Add an ALWAYS() to an always-true conditional in the WAL rollback logic. FossilOrigin-Name: c5eae8a60d474131fbfa4d0c2b459005267e8be4 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/wal.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index d98c6b6486..e7e986f7fe 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\scode\sfrom\ssqlite3BtreeKeySize()\smade\sunreachable\sby\sthe\sprevious\ncheck-in. -D 2014-11-17T20:33:07.735 +C Add\san\sALWAYS()\sto\san\salways-true\sconditional\sin\sthe\sWAL\srollback\slogic. +D 2014-11-18T02:44:32.636 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -299,7 +299,7 @@ F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f F src/vdbesort.c 975aeffa99acb0991b2f288d30294756bff41438 F src/vdbetrace.c 7e4222955e07dd707a2f360c0eb73452be1cb010 F src/vtab.c cb0c194303fea276b48d7d4b6d970b5a96bde8de -F src/wal.c 73051f1222321712fa4280c495780ba81d302dad +F src/wal.c 095d41f7114d7a8699207f5313488aa88372d540 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 F src/where.c 2947912f1f3d6a7766fe087fd532a5d688d745b1 @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 2f2ecb994889acb783616acb7307f8fed962d213 -R 3350d2f69701d38a07608c1007a483bc +P 57c4aa988c8eda3cc513c1e5df5804d88bee99a0 +R f70435983903607c251f8514c7f3013d U drh -Z df1a082a18c57f32c53583132171df67 +Z f0ae269a66dc8c4b111d5661661171ab diff --git a/manifest.uuid b/manifest.uuid index 2223a6ffff..3ceeab2cdb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -57c4aa988c8eda3cc513c1e5df5804d88bee99a0 \ No newline at end of file +c5eae8a60d474131fbfa4d0c2b459005267e8be4 \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 24540a2cd5..d134a8b52a 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2506,7 +2506,7 @@ int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx){ memcpy(&pWal->hdr, (void *)walIndexHdr(pWal), sizeof(WalIndexHdr)); for(iFrame=pWal->hdr.mxFrame+1; - rc==SQLITE_OK && iFrame<=iMax; + ALWAYS(rc==SQLITE_OK) && iFrame<=iMax; iFrame++ ){ /* This call cannot fail. Unless the page for which the page number From bb8f92529401b09dbc6e9df4ba3e38ae3a0789f3 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 18 Nov 2014 12:28:52 +0000 Subject: [PATCH 121/133] Increment the version number to 3.8.7.2 FossilOrigin-Name: 945a9e687fdfee5f7103d85d131024e85d594ac3 --- VERSION | 2 +- configure | 18 +++++++++--------- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/VERSION b/VERSION index f53c1ed56b..6df6debe9d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.8.7.1 +3.8.7.2 diff --git a/configure b/configure index d253f21a7d..1aedcaf4d3 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.62 for sqlite 3.8.7.1. +# Generated by GNU Autoconf 2.62 for sqlite 3.8.7.2. # # Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, # 2002, 2003, 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc. @@ -743,8 +743,8 @@ SHELL=${CONFIG_SHELL-/bin/sh} # Identity of this package. PACKAGE_NAME='sqlite' PACKAGE_TARNAME='sqlite' -PACKAGE_VERSION='3.8.7.1' -PACKAGE_STRING='sqlite 3.8.7.1' +PACKAGE_VERSION='3.8.7.2' +PACKAGE_STRING='sqlite 3.8.7.2' PACKAGE_BUGREPORT='' # Factoring default headers for most tests. @@ -1483,7 +1483,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures sqlite 3.8.7.1 to adapt to many kinds of systems. +\`configure' configures sqlite 3.8.7.2 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1548,7 +1548,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of sqlite 3.8.7.1:";; + short | recursive ) echo "Configuration of sqlite 3.8.7.2:";; esac cat <<\_ACEOF @@ -1664,7 +1664,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -sqlite configure 3.8.7.1 +sqlite configure 3.8.7.2 generated by GNU Autoconf 2.62 Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, @@ -1678,7 +1678,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by sqlite $as_me 3.8.7.1, which was +It was created by sqlite $as_me 3.8.7.2, which was generated by GNU Autoconf 2.62. Invocation command line was $ $0 $@ @@ -14021,7 +14021,7 @@ exec 6>&1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by sqlite $as_me 3.8.7.1, which was +This file was extended by sqlite $as_me 3.8.7.2, which was generated by GNU Autoconf 2.62. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -14074,7 +14074,7 @@ Report bugs to ." _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_version="\\ -sqlite config.status 3.8.7.1 +sqlite config.status 3.8.7.2 configured by $0, generated by GNU Autoconf 2.62, with options \\"`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`\\" diff --git a/manifest b/manifest index e7e986f7fe..00bcf54505 100644 --- a/manifest +++ b/manifest @@ -1,12 +1,12 @@ -C Add\san\sALWAYS()\sto\san\salways-true\sconditional\sin\sthe\sWAL\srollback\slogic. -D 2014-11-18T02:44:32.636 +C Increment\sthe\sversion\snumber\sto\s3.8.7.2 +D 2014-11-18T12:28:52.482 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 F Makefile.msc e31dee24038965fb6269d6d61073fd6b7e331dec F Makefile.vxworks 034289efa9d591b04b1a73598623119c306cbba0 F README.md 64f270c43c38c46de749e419c22f0ae2f4499fe8 -F VERSION 5cc0baaee7eeee3238f0f7b5871398d17f79d0cd +F VERSION 3978bf46d1599bc324ae171a99c4e8fca7481822 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 F addopcodes.awk 9eb448a552d5c0185cf62c463f9c173cedae3811 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2 @@ -38,7 +38,7 @@ F autoconf/tea/win/rules.vc c511f222b80064096b705dbeb97060ee1d6b6d63 F config.guess 226d9a188c6196f3033ffc651cbc9dcee1a42977 F config.h.in 0921066a13130082764ab4ab6456f7b5bebe56de F config.sub 9ebe4c3b3dab6431ece34f16828b594fb420da55 -F configure 56fe985cf0e59cd594f9b929099d0be40260e667 x +F configure 135207dac9b9ff35a91cdb17871322c26fa5de73 x F configure.ac 4cf9f60785143fa141b10962ccc885d973792e9a F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad F doc/lemon.html 334dbf6621b8fb8790297ec1abf3cfa4621709d1 @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 57c4aa988c8eda3cc513c1e5df5804d88bee99a0 -R f70435983903607c251f8514c7f3013d +P c5eae8a60d474131fbfa4d0c2b459005267e8be4 +R 876a331fea8d7e2c7f47a110195c3ecb U drh -Z f0ae269a66dc8c4b111d5661661171ab +Z 48596521afe0080c14cc5d1c64dcfe43 diff --git a/manifest.uuid b/manifest.uuid index 3ceeab2cdb..a62a0bd661 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c5eae8a60d474131fbfa4d0c2b459005267e8be4 \ No newline at end of file +945a9e687fdfee5f7103d85d131024e85d594ac3 \ No newline at end of file From 58f95c43adb7e26452e3136f55ee40a12c479d7d Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 18 Nov 2014 20:16:27 +0000 Subject: [PATCH 122/133] Update a couple of test cases to account for the fact that ROLLBACK does not always abort all running SELECT statements. FossilOrigin-Name: abccda769a3f6b755c3bf70b5fb31a5e16718ef3 --- manifest | 17 +++++++++-------- manifest.uuid | 2 +- test/ioerr2.test | 8 +++++++- test/shared_err.test | 8 +++++++- 4 files changed, 24 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 12a819fd2f..267f0a0da9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\scalling\ssqlite3BtreeKeysize()\son\sa\sb-tree\scursor\sin\sSKIPNEXT\sor\sSKIPPREV\sstate. -D 2014-11-17T19:42:48.262 +C Update\sa\scouple\sof\stest\scases\sto\saccount\sfor\sthe\sfact\sthat\sROLLBACK\sdoes\snot\salways\sabort\sall\srunning\sSELECT\sstatements. +D 2014-11-18T20:16:27.742 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -659,7 +659,7 @@ F test/interrupt.test dfe9a67a94b0b2d8f70545ba1a6cca10780d71cc F test/intpkey.test 7506090fc08e028712a8bf47e5f54111947e3844 F test/io.test 3a7abcef18727cc0f2399e04b0e8903eccae50f8 F test/ioerr.test 2a24bd6ed5a8b062e64bfe1f6cf94fb25e92210d -F test/ioerr2.test 9d71166f8466eda510f1af6137bdabaa82b5408d +F test/ioerr2.test 2593563599e2cc6b6b4fcf5878b177bdd5d8df26 F test/ioerr3.test d3cec5e1a11ad6d27527d0d38573fbff14c71bdd F test/ioerr4.test f130fe9e71008577b342b8874d52984bd04ede2c F test/ioerr5.test 2edfa4fb0f896f733071303b42224df8bedd9da4 @@ -844,7 +844,7 @@ F test/shared7.test a81e99f83e6c51b02ac99c96fb3a2a7b5978c956 F test/shared8.test 00a07bf5e1337ecf72e94542bdefdc330d7a2538 F test/shared9.test 5f2a8f79b4d6c7d107a01ffa1ed05ae7e6333e21 F test/sharedA.test 0cdf1a76dfa00e6beee66af5b534b1e8df2720f5 -F test/shared_err.test 0079c05c97d88cfa03989b7c20a8b266983087aa +F test/shared_err.test 2f2aee20db294b9924e81f6ccbe60f19e21e8506 F test/sharedlock.test 5ede3c37439067c43b0198f580fd374ebf15d304 F test/shell1.test d60946b5fde4d85fe06db7331dfe89011f564350 F test/shell2.test c57da3a381c099b02c813ba156298d5c2f5c93a3 @@ -1221,7 +1221,8 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 55e453aadbb676dda07f0fa537d39ce184ef636c -R 6cd25f6196e6f786753b0fb88328cee1 -U dan -Z f594bfe54166ed1c0f3ae7a5a057ba04 +P 54e7d3fcb1ab21c03ffef1af93ae029a2901098a +Q +eba171e980fa4491dfee9d7e4df50c87a0ebbf87 +R 07c45ea3d962860999af22ae8c2af8c8 +U drh +Z f4bad3f3683413628206e5a23517cfd8 diff --git a/manifest.uuid b/manifest.uuid index e3e77af8a0..22da5e0eba 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -54e7d3fcb1ab21c03ffef1af93ae029a2901098a \ No newline at end of file +abccda769a3f6b755c3bf70b5fb31a5e16718ef3 \ No newline at end of file diff --git a/test/ioerr2.test b/test/ioerr2.test index 5150ace3ab..c08c3453f3 100644 --- a/test/ioerr2.test +++ b/test/ioerr2.test @@ -112,6 +112,12 @@ foreach bPersist [list 0 1] { } } +# When this test was written, an IO error within the UPDATE statement caused +# a rollback, which tripped all read-cursors, causing the outer SELECT to +# fail with "abort due to ROLLBACK". Now, the loop continues until the UPDATE +# is run successfully. At this point the next IO error occurs within the +# SELECT - throwing the "disk I/O error" that the test case now expects. +# do_test ioerr2-5 { execsql { CREATE TABLE t2 AS SELECT * FROM t1; @@ -130,7 +136,7 @@ do_test ioerr2-5 { } } msg] list $rc $msg -} {1 {abort due to ROLLBACK}} +} {1 {disk I/O error}} ;# used to be "{1 {abort due to ROLLBACK}}" if {$::tcl_platform(platform) == "unix"} { # Cause the call to xAccess used by [pragma temp_store_directory] to diff --git a/test/shared_err.test b/test/shared_err.test index 17add94bb7..96e5ee4540 100644 --- a/test/shared_err.test +++ b/test/shared_err.test @@ -446,9 +446,15 @@ do_malloc_test shared_err-8 -tclprep { } {1} db2 close } + +# When this test case was written, OOM errors in write statements would +# cause transaction rollback, which would trip cursors in other statements, +# aborting them. This no longer happens. +# do_test shared_malloc-8.X { # Test that one or more queries were aborted due to the malloc() failure. - expr $::aborted>=1 + # expr $::aborted>=1 + expr $::aborted==0 } {1} # This test is designed to catch a specific bug that was present during From bea3b976a95b00968d6f3ca496d5c1dfd3104527 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 18 Nov 2014 20:22:05 +0000 Subject: [PATCH 123/133] Fix a bug in the sqlite3TripAllCursors() routine that prevents it from reporting errors. It is unknown at this time whether or not this omission can result in any incorrect result in an actual query. FossilOrigin-Name: 2896f2640ab3e102ee248d20fb68c497817524eb --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/btree.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 267f0a0da9..9cb41e8715 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sa\scouple\sof\stest\scases\sto\saccount\sfor\sthe\sfact\sthat\sROLLBACK\sdoes\snot\salways\sabort\sall\srunning\sSELECT\sstatements. -D 2014-11-18T20:16:27.742 +C Fix\sa\sbug\sin\sthe\ssqlite3TripAllCursors()\sroutine\sthat\sprevents\sit\sfrom\nreporting\serrors.\s\sIt\sis\sunknown\sat\sthis\stime\swhether\sor\snot\sthis\somission\scan\nresult\sin\sany\sincorrect\sresult\sin\san\sactual\squery. +D 2014-11-18T20:22:05.030 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -173,7 +173,7 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 7ddee9c7d505e07e959a575b18498f17c71e53ea F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c d5d991b518fa5bebc64037dfeb98a48051d864d7 +F src/btree.c 75edb585cc2c66615e0ea01a48807a7bfae4f2fe F src/btree.h e31a3a3ebdedb1caf9bda3ad5dbab3db9b780f6e F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 67bb05b1077e0cdaccb2e36bfcbe7a5df9ed31e8 @@ -1221,8 +1221,8 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 54e7d3fcb1ab21c03ffef1af93ae029a2901098a -Q +eba171e980fa4491dfee9d7e4df50c87a0ebbf87 -R 07c45ea3d962860999af22ae8c2af8c8 +P abccda769a3f6b755c3bf70b5fb31a5e16718ef3 +Q +42588207ff5451cb785c394633e1ab631fb82f01 +R be8390e45e84a659cdb889947026b04a U drh -Z f4bad3f3683413628206e5a23517cfd8 +Z 45d41fa50f6cb7c97c28d0b284fe4014 diff --git a/manifest.uuid b/manifest.uuid index 22da5e0eba..922bec9a78 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -abccda769a3f6b755c3bf70b5fb31a5e16718ef3 \ No newline at end of file +2896f2640ab3e102ee248d20fb68c497817524eb \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index f153a0cc45..7a49fb1a79 100644 --- a/src/btree.c +++ b/src/btree.c @@ -3541,7 +3541,7 @@ int sqlite3BtreeTripAllCursors(Btree *pBtree, int errCode, int writeOnly){ int i; if( writeOnly && (p->curFlags & BTCF_WriteFlag)==0 ){ if( p->eState==CURSOR_VALID ){ - int rc = saveCursorPosition(p); + rc = saveCursorPosition(p); if( rc!=SQLITE_OK ){ (void)sqlite3BtreeTripAllCursors(pBtree, rc, 0); break; From 664f85dd59c7995b2651ca9de390607dbfadb419 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 19 Nov 2014 14:05:41 +0000 Subject: [PATCH 124/133] Add an ALWAYS on an always-true branch in wal.c. Fix the ANALYZE command so that it resets the "unordered" and "noskipscan" flags on indices when reloading the sqlite_stat1 table. FossilOrigin-Name: 9ed97a85feee6593faefa2b54cc4cf9a60f515f9 --- manifest | 15 +++++++-------- manifest.uuid | 2 +- src/analyze.c | 2 ++ src/wal.c | 2 +- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 272d0369e1..5976aae2e5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Adding\sthe\s"noskipscan"\stoken\sto\san\ssqlite_stat1.stat\sfield\sprevents\nan\sindex\sfor\sbeing\sused\swith\sthe\sskip-scan\salgorithm. -D 2014-11-18T21:54:31.890 +C Add\san\sALWAYS\son\san\salways-true\sbranch\sin\swal.c.\nFix\sthe\sANALYZE\scommand\sso\sthat\sit\sresets\sthe\s"unordered"\sand\s"noskipscan"\nflags\son\sindices\swhen\sreloading\sthe\ssqlite_stat1\stable. +D 2014-11-19T14:05:41.209 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -167,7 +167,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c ba266a779bc7ce10e52e59e7d3dc79fa342e8fdb -F src/analyze.c 951fd859852dfbced4a58b73954f168eeb9d0772 +F src/analyze.c c59f238a39aacece176f8bb7dfece40deb268ee5 F src/attach.c f4e94df2d1826feda65eb0939f7f6f5f923a0ad9 F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 7ddee9c7d505e07e959a575b18498f17c71e53ea @@ -301,7 +301,7 @@ F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f F src/vdbesort.c 87f3923483113d1c95d84640becb4e4946f27d9a F src/vdbetrace.c 7e4222955e07dd707a2f360c0eb73452be1cb010 F src/vtab.c 2a30791bbd7926b589401bd09c3abb33de563793 -F src/wal.c fa090966140602f03a621f87d82ee69e66ca63b5 +F src/wal.c 486e644b3b8aa5ad066f625bc428aa8ff7001405 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 F src/where.c e275cb74731a3351a9da6ba8280bd5054db6192d @@ -1221,8 +1221,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P c52f7971e90cac1058d6b66c9e334cbc8607def3 00fe09505792cd0d104b2da9d040f023e30fa871 -R 9bae3f4f5e0db167ef1a706a77e76141 -T +closed 00fe09505792cd0d104b2da9d040f023e30fa871 +P 4461bf045d8eecf98478035efcdba3f41c709bc5 +R b83c6fa5464847f6b6472d2244f6f040 U drh -Z fec927bef6596f9a5e5eec2871a731e7 +Z cd93530e8709a199e7d3e18ae995b78d diff --git a/manifest.uuid b/manifest.uuid index fd301fc3c9..5abcc58c68 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4461bf045d8eecf98478035efcdba3f41c709bc5 \ No newline at end of file +9ed97a85feee6593faefa2b54cc4cf9a60f515f9 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 769a16565a..ffb5c22b63 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1438,6 +1438,8 @@ static void decodeIntArray( if( z==0 ) z = ""; #else assert( z!=0 ); + pIndex->bUnordered = 0; + pIndex->noSkipScan = 0; #endif for(i=0; *z && ihdr, (void *)walIndexHdr(pWal), sizeof(WalIndexHdr)); for(iFrame=pWal->hdr.mxFrame+1; - rc==SQLITE_OK && iFrame<=iMax; + ALWAYS(rc==SQLITE_OK) && iFrame<=iMax; iFrame++ ){ /* This call cannot fail. Unless the page for which the page number From 42a7b4bac94983bfde72f3779e6ee15bfe847335 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 19 Nov 2014 14:31:12 +0000 Subject: [PATCH 125/133] Completely remove an assert() that had previously been commented out. FossilOrigin-Name: 89b3c1c4555c98c633089b21cdd2a1a3a1e751eb --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbe.c | 1 - 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 5976aae2e5..b9bc7630da 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\san\sALWAYS\son\san\salways-true\sbranch\sin\swal.c.\nFix\sthe\sANALYZE\scommand\sso\sthat\sit\sresets\sthe\s"unordered"\sand\s"noskipscan"\nflags\son\sindices\swhen\sreloading\sthe\ssqlite_stat1\stable. -D 2014-11-19T14:05:41.209 +C Completely\sremove\san\sassert()\sthat\shad\spreviously\sbeen\scommented\sout. +D 2014-11-19T14:31:12.601 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/update.c 3c4ecc282accf12d39edb8d524cf089645e55a13 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 3b627daa45c7308c1e36e3dbaa3f9ce7e5c7fa73 F src/vacuum.c 9b30ec729337dd012ed88d4c292922c8ef9cf00c -F src/vdbe.c 5563459c06c434bc43131044fcf8164654008ebd +F src/vdbe.c 16914136ea1e8b18366868671ddb019159687f47 F src/vdbe.h 6fc69d9c5e146302c56e163cb4b31d1ee64a18c3 F src/vdbeInt.h 9bb69ff2447c34b6ccc58b34ec35b615f86ead78 F src/vdbeapi.c 07acb615d1e4170e71fc1b0d087f3c53a1ad8e83 @@ -1221,7 +1221,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 4461bf045d8eecf98478035efcdba3f41c709bc5 -R b83c6fa5464847f6b6472d2244f6f040 +P 9ed97a85feee6593faefa2b54cc4cf9a60f515f9 +R 612c462b22665e254a8e686035c1cd93 U drh -Z cd93530e8709a199e7d3e18ae995b78d +Z cb045903d6d20b681c939d9aa729532d diff --git a/manifest.uuid b/manifest.uuid index 5abcc58c68..7fd5801d3e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9ed97a85feee6593faefa2b54cc4cf9a60f515f9 \ No newline at end of file +89b3c1c4555c98c633089b21cdd2a1a3a1e751eb \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index f883412943..31b42a32bf 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -3808,7 +3808,6 @@ case OP_Found: { /* jump, in3 */ ); if( pIdxKey==0 ) goto no_mem; assert( pIn3->flags & MEM_Blob ); - /* assert( (pIn3->flags & MEM_Zero)==0 ); // zeroblobs already expanded */ ExpandBlob(pIn3); sqlite3VdbeRecordUnpack(pC->pKeyInfo, pIn3->n, pIn3->z, pIdxKey); } From 113762a284699ca55192eec1b482d695fcb281e6 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 19 Nov 2014 16:36:25 +0000 Subject: [PATCH 126/133] Add new requirements marks associated with the file format documentation. No changes to code. FossilOrigin-Name: 6d00bcca6ed1903fb17275752cab71c14392355b --- manifest | 18 ++++++++-------- manifest.uuid | 2 +- src/btree.c | 57 +++++++++++++++++++++++++++++++++++++++++++++---- src/expr.c | 5 ++++- src/pager.c | 2 +- src/sqlite.h.in | 6 +++--- 6 files changed, 71 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index b9bc7630da..a251e14322 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Completely\sremove\san\sassert()\sthat\shad\spreviously\sbeen\scommented\sout. -D 2014-11-19T14:31:12.601 +C Add\snew\srequirements\smarks\sassociated\swith\sthe\sfile\sformat\sdocumentation.\nNo\schanges\sto\scode. +D 2014-11-19T16:36:25.272 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -173,7 +173,7 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 7ddee9c7d505e07e959a575b18498f17c71e53ea F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c b562da29eb370aaac8015026827c2e2fb70ae990 +F src/btree.c d0a7cfc8788ad028c86020637df70c789615196e F src/btree.h e31a3a3ebdedb1caf9bda3ad5dbab3db9b780f6e F src/btreeInt.h 3363e18fd76f69a27a870b25221b2345b3fd4d21 F src/build.c 67bb05b1077e0cdaccb2e36bfcbe7a5df9ed31e8 @@ -182,7 +182,7 @@ F src/complete.c c4ba6e0626bb94bc77a0861735f3382fcf7cc818 F src/ctime.c df19848891c8a553c80e6f5a035e768280952d1a F src/date.c 93594514aae68de117ca4a2a0d6cc63eddf26744 F src/delete.c 0750b1eb4d96cd3fb2c798599a3a7c85e92f1417 -F src/expr.c a3ff05db5709d628c23890db862e30f3dd9dc428 +F src/expr.c 73de4c0da2eed6b149d40a05c589dfeb2c4a87a1 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c da985ae673efef2c712caef825a5d2edb087ead7 F src/func.c ba47c1671ab3cfdafa6e9d6ee490939ea578adee @@ -216,7 +216,7 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c fb587121840f690101336879adfa6d0b2cd0e8c7 F src/os_win.c a9e500dd963fb1f67d7860e58b5772abe6123862 F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 -F src/pager.c 8d97b3633f098fef817656dcbf167ca904511d78 +F src/pager.c b8764f90c135482988268eec93d7f5cdb89d687a F src/pager.h d1eee3c3f741be247ce6d82752a178515fc8578b F src/parse.y 5dfead8aed90cb0c7c1115898ee2266804daff45 F src/pcache.c ace1b67632deeaa84859b4c16c27711dfb7db3d4 @@ -230,7 +230,7 @@ F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c bc28d5992109717c87804e2eb1a08a7c8cc7a2fd -F src/sqlite.h.in 0c5c0df7e4e436dfc5592511325bf4a96f6a638d +F src/sqlite.h.in dfbdcd6e52e7652648d8e7634a7c191fcac8e6ce F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h c9e95b8fa9aee30d46387735c5be73fa58886e38 @@ -1221,7 +1221,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 9ed97a85feee6593faefa2b54cc4cf9a60f515f9 -R 612c462b22665e254a8e686035c1cd93 +P 89b3c1c4555c98c633089b21cdd2a1a3a1e751eb +R 92805b92bb878e3838f25a4f3bef8761 U drh -Z cb045903d6d20b681c939d9aa729532d +Z 529027073b2f32ceeacba058565c65ab diff --git a/manifest.uuid b/manifest.uuid index 7fd5801d3e..cb62723bc8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -89b3c1c4555c98c633089b21cdd2a1a3a1e751eb \ No newline at end of file +6d00bcca6ed1903fb17275752cab71c14392355b \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 9587e567a3..f61078cac6 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1247,10 +1247,15 @@ static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc, int *pbDefrag){ for(iAddr=hdr+1; (pc = get2byte(&aData[iAddr]))>0; iAddr=pc){ int size; /* Size of the free slot */ + /* EVIDENCE-OF: R-06866-39125 Freeblocks are always connected in order of + ** increasing offset. */ if( pc>usableSize-4 || pc=nByte ){ int x = size - nByte; @@ -1314,6 +1319,11 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ top = get2byte(&data[hdr+5]); if( gap>top ){ if( top==0 ){ + /* EVIDENCE-OF: R-29356-02391 If the database uses a 65536-byte page size + ** and the reserved space is zero (the usual value for reserved space) + ** then the cell content offset of an empty page wants to be 65536. + ** However, that integer is too large to be stored in a 2-byte unsigned + ** integer, so a value of 0 is used in its place. */ top = 65536; }else{ return SQLITE_CORRUPT_BKPT; @@ -1998,6 +2008,9 @@ int sqlite3BtreeOpen( #ifdef SQLITE_SECURE_DELETE pBt->btsFlags |= BTS_SECURE_DELETE; #endif + /* EVIDENCE-OF: R-51873-39618 The page size for a database file is + ** determined by the 2-byte integer located at an offset of 16 bytes from + ** the beginning of the database file. */ pBt->pageSize = (zDbHeader[16]<<8) | (zDbHeader[17]<<16); if( pBt->pageSize<512 || pBt->pageSize>SQLITE_MAX_PAGE_SIZE || ((pBt->pageSize-1)&pBt->pageSize)!=0 ){ @@ -2016,6 +2029,9 @@ int sqlite3BtreeOpen( #endif nReserve = 0; }else{ + /* EVIDENCE-OF: R-37497-42412 The size of the reserved region is + ** determined by the one-byte unsigned integer found at an offset of 20 + ** into the database file header. */ nReserve = zDbHeader[20]; pBt->btsFlags |= BTS_PAGESIZE_FIXED; #ifndef SQLITE_OMIT_AUTOVACUUM @@ -2525,6 +2541,9 @@ static int lockBtree(BtShared *pBt){ u32 usableSize; u8 *page1 = pPage1->aData; rc = SQLITE_NOTADB; + /* EVIDENCE-OF: R-43737-39999 Every valid SQLite database file begins + ** with the following 16 bytes (in hex): 53 51 4c 69 74 65 20 66 6f 72 6d + ** 61 74 20 33 00. */ if( memcmp(page1, zMagicHeader, 16)!=0 ){ goto page1_init_failed; } @@ -2565,15 +2584,21 @@ static int lockBtree(BtShared *pBt){ } #endif - /* The maximum embedded fraction must be exactly 25%. And the minimum - ** embedded fraction must be 12.5% for both leaf-data and non-leaf-data. + /* EVIDENCE-OF: R-15465-20813 The maximum and minimum embedded payload + ** fractions and the leaf payload fraction values must be 64, 32, and 32. + ** ** The original design allowed these amounts to vary, but as of ** version 3.6.0, we require them to be fixed. */ if( memcmp(&page1[21], "\100\040\040",3)!=0 ){ goto page1_init_failed; } + /* EVIDENCE-OF: R-51873-39618 The page size for a database file is + ** determined by the 2-byte integer located at an offset of 16 bytes from + ** the beginning of the database file. */ pageSize = (page1[16]<<8) | (page1[17]<<16); + /* EVIDENCE-OF: R-25008-21688 The size of a page is a power of two + ** between 512 and 65536 inclusive. */ if( ((pageSize-1)&pageSize)!=0 || pageSize>SQLITE_MAX_PAGE_SIZE || pageSize<=256 @@ -2581,6 +2606,13 @@ static int lockBtree(BtShared *pBt){ goto page1_init_failed; } assert( (pageSize & 7)==0 ); + /* EVIDENCE-OF: R-59310-51205 The "reserved space" size in the 1-byte + ** integer at offset 20 is the number of bytes of space at the end of + ** each page to reserve for extensions. + ** + ** EVIDENCE-OF: R-37497-42412 The size of the reserved region is + ** determined by the one-byte unsigned integer found at an offset of 20 + ** into the database file header. */ usableSize = pageSize - page1[20]; if( (u32)pageSize!=pBt->pageSize ){ /* After reading the first page of the database assuming a page size @@ -2601,6 +2633,9 @@ static int lockBtree(BtShared *pBt){ rc = SQLITE_CORRUPT_BKPT; goto page1_init_failed; } + /* EVIDENCE-OF: R-28312-64704 However, the usable size is not allowed to + ** be less than 480. In other words, if the page size is 512, then the + ** reserved space size cannot exceed 32. */ if( usableSize<480 ){ goto page1_init_failed; } @@ -5178,6 +5213,8 @@ static int allocateBtreePage( assert( eMode==BTALLOC_ANY || (nearby>0 && IfNotOmitAV(pBt->autoVacuum)) ); pPage1 = pBt->pPage1; mxPage = btreePagecount(pBt); + /* EVIDENCE-OF: R-05119-02637 The 4-byte big-endian integer at offset 36 + ** stores stores the total number of pages on the freelist. */ n = get4byte(&pPage1->aData[36]); testcase( n==mxPage-1 ); if( n>=mxPage ){ @@ -5224,8 +5261,14 @@ static int allocateBtreePage( do { pPrevTrunk = pTrunk; if( pPrevTrunk ){ + /* EVIDENCE-OF: R-01506-11053 The first integer on a freelist trunk page + ** is the page number of the next freelist trunk page in the list or + ** zero if this is the last freelist trunk page. */ iTrunk = get4byte(&pPrevTrunk->aData[0]); }else{ + /* EVIDENCE-OF: R-59841-13798 The 4-byte big-endian integer at offset 32 + ** stores the page number of the first page of the freelist, or zero if + ** the freelist is empty. */ iTrunk = get4byte(&pPage1->aData[32]); } testcase( iTrunk==mxPage ); @@ -5240,8 +5283,9 @@ static int allocateBtreePage( } assert( pTrunk!=0 ); assert( pTrunk->aData!=0 ); - - k = get4byte(&pTrunk->aData[4]); /* # of leaves on this trunk page */ + /* EVIDENCE-OF: R-13523-04394 The second integer on a freelist trunk page + ** is the number of leaf page pointers to follow. */ + k = get4byte(&pTrunk->aData[4]); if( k==0 && !searchList ){ /* The trunk has no leaves and the list is not being searched. ** So extract the trunk page itself and use it as the newly @@ -5559,6 +5603,11 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ ** for now. At some point in the future (once everyone has upgraded ** to 3.6.0 or later) we should consider fixing the conditional above ** to read "usableSize/4-2" instead of "usableSize/4-8". + ** + ** EVIDENCE-OF: R-19920-11576 However, newer versions of SQLite still + ** avoid using the last six entries in the freelist trunk page array in + ** order that database files created by newer versions of SQLite can be + ** read by older versions of SQLite. */ rc = sqlite3PagerWrite(pTrunk->pDbPage); if( rc==SQLITE_OK ){ diff --git a/src/expr.c b/src/expr.c index 25f0be400f..c28f196221 100644 --- a/src/expr.c +++ b/src/expr.c @@ -3003,7 +3003,10 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ #ifndef SQLITE_OMIT_FLOATING_POINT /* If the column has REAL affinity, it may currently be stored as an - ** integer. Use OP_RealAffinity to make sure it is really real. */ + ** integer. Use OP_RealAffinity to make sure it is really real. + ** + ** EVIDENCE-OF: R-60985-57662 SQLite will convert the value back to + ** floating point when extracting it from the record. */ if( pExpr->iColumn>=0 && pTab->aCol[pExpr->iColumn].affinity==SQLITE_AFF_REAL ){ diff --git a/src/pager.c b/src/pager.c index 997f842d0a..e0ede87f07 100644 --- a/src/pager.c +++ b/src/pager.c @@ -2899,7 +2899,7 @@ static int readDbPage(PgHdr *pPg, u32 iFrame){ ** ** For an encrypted database, the situation is more complex: bytes ** 24..39 of the database are white noise. But the probability of - ** white noising equaling 16 bytes of 0xff is vanishingly small so + ** white noise equaling 16 bytes of 0xff is vanishingly small so ** we should still be ok. */ memset(pPager->dbFileVers, 0xff, sizeof(pPager->dbFileVers)); diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 4427f39d06..a59599a0c5 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -4164,9 +4164,9 @@ int sqlite3_create_function_v2( ** These constant define integer codes that represent the various ** text encodings supported by SQLite. */ -#define SQLITE_UTF8 1 -#define SQLITE_UTF16LE 2 -#define SQLITE_UTF16BE 3 +#define SQLITE_UTF8 1 /* IMP: R-37514-35566 */ +#define SQLITE_UTF16LE 2 /* IMP: R-03371-37637 */ +#define SQLITE_UTF16BE 3 /* IMP: R-51971-34154 */ #define SQLITE_UTF16 4 /* Use native byte order */ #define SQLITE_ANY 5 /* Deprecated */ #define SQLITE_UTF16_ALIGNED 8 /* sqlite3_create_collation only */ From 654858d7f45254f915f0796431699ecc841fdf06 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 20 Nov 2014 02:18:14 +0000 Subject: [PATCH 127/133] Add some requirements marks to the record formatting logic. Comment changes only - the code is unaltered. FossilOrigin-Name: 9a9627e178a67bbfc85366aaea900e674d22fb53 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbe.c | 9 ++++++++- src/vdbeaux.c | 25 +++++++++++++++++++++++-- 4 files changed, 39 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index a251e14322..065eb19989 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\snew\srequirements\smarks\sassociated\swith\sthe\sfile\sformat\sdocumentation.\nNo\schanges\sto\scode. -D 2014-11-19T16:36:25.272 +C Add\ssome\srequirements\smarks\sto\sthe\srecord\sformatting\slogic.\s\sComment\schanges\nonly\s-\sthe\scode\sis\sunaltered. +D 2014-11-20T02:18:14.206 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,11 +291,11 @@ F src/update.c 3c4ecc282accf12d39edb8d524cf089645e55a13 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 3b627daa45c7308c1e36e3dbaa3f9ce7e5c7fa73 F src/vacuum.c 9b30ec729337dd012ed88d4c292922c8ef9cf00c -F src/vdbe.c 16914136ea1e8b18366868671ddb019159687f47 +F src/vdbe.c ec1f55acef4864520ca2017b9f0d60c2ac1b8b78 F src/vdbe.h 6fc69d9c5e146302c56e163cb4b31d1ee64a18c3 F src/vdbeInt.h 9bb69ff2447c34b6ccc58b34ec35b615f86ead78 F src/vdbeapi.c 07acb615d1e4170e71fc1b0d087f3c53a1ad8e83 -F src/vdbeaux.c 9b0a251b6dfab349dd6c6efb40062eb7386b26f5 +F src/vdbeaux.c e3ae27459a4eb986d2892de2d7f309d18c9971ef F src/vdbeblob.c 4af4bfb71f6df7778397b4a0ebc1879793276778 F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f F src/vdbesort.c 87f3923483113d1c95d84640becb4e4946f27d9a @@ -1221,7 +1221,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 89b3c1c4555c98c633089b21cdd2a1a3a1e751eb -R 92805b92bb878e3838f25a4f3bef8761 +P 6d00bcca6ed1903fb17275752cab71c14392355b +R b487443c841a5e8e1bfed86b80cd7f65 U drh -Z 529027073b2f32ceeacba058565c65ab +Z bbc37d068b0ef41277590cc500b3cc54 diff --git a/manifest.uuid b/manifest.uuid index cb62723bc8..a704e79b38 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6d00bcca6ed1903fb17275752cab71c14392355b \ No newline at end of file +9a9627e178a67bbfc85366aaea900e674d22fb53 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 31b42a32bf..822bf80bb8 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2636,7 +2636,10 @@ case OP_MakeRecord: { nHdr += serial_type<=127 ? 1 : sqlite3VarintLen(serial_type); }while( (--pRec)>=pData0 ); - /* Add the initial header varint and total the size */ + /* EVIDENCE-OF: R-22564-11647 The header begins with a single varint + ** which determines the total number of bytes in the header. The varint + ** value is the size of the header in bytes including the size varint + ** itself. */ testcase( nHdr==126 ); testcase( nHdr==127 ); if( nHdr<=126 ){ @@ -2670,7 +2673,11 @@ case OP_MakeRecord: { pRec = pData0; do{ serial_type = pRec->uTemp; + /* EVIDENCE-OF: R-06529-47362 Following the size varint are one or more + ** additional varints, one per column. */ i += putVarint32(&zNewRecord[i], serial_type); /* serial type */ + /* EVIDENCE-OF: R-64536-51728 The values for each column in the record + ** immediately follow the header. */ j += sqlite3VdbeSerialPut(&zNewRecord[j], pRec, serial_type); /* content */ }while( (++pRec)<=pLast ); assert( i==nHdr ); diff --git a/src/vdbeaux.c b/src/vdbeaux.c index d8ee5c8e8c..145bf453ad 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -3051,10 +3051,14 @@ static u32 SQLITE_NOINLINE serialGet( u32 y = FOUR_BYTE_UINT(buf+4); x = (x<<32) + y; if( serial_type==6 ){ + /* EVIDENCE-OF: R-29851-52272 Value is a big-endian 64-bit + ** twos-complement integer. */ pMem->u.i = *(i64*)&x; pMem->flags = MEM_Int; testcase( pMem->u.i<0 ); }else{ + /* EVIDENCE-OF: R-57343-49114 Value is a big-endian IEEE 754-2008 64-bit + ** floating point number. */ #if !defined(NDEBUG) && !defined(SQLITE_OMIT_FLOATING_POINT) /* Verify that integers and floating point values use the same ** byte order. Or, that if SQLITE_MIXED_ENDIAN_64BIT_FLOAT is @@ -3082,35 +3086,46 @@ u32 sqlite3VdbeSerialGet( switch( serial_type ){ case 10: /* Reserved for future use */ case 11: /* Reserved for future use */ - case 0: { /* NULL */ + case 0: { /* Null */ + /* EVIDENCE-OF: R-24078-09375 Value is a NULL. */ pMem->flags = MEM_Null; break; } - case 1: { /* 1-byte signed integer */ + case 1: { + /* EVIDENCE-OF: R-44885-25196 Value is an 8-bit twos-complement + ** integer. */ pMem->u.i = ONE_BYTE_INT(buf); pMem->flags = MEM_Int; testcase( pMem->u.i<0 ); return 1; } case 2: { /* 2-byte signed integer */ + /* EVIDENCE-OF: R-49794-35026 Value is a big-endian 16-bit + ** twos-complement integer. */ pMem->u.i = TWO_BYTE_INT(buf); pMem->flags = MEM_Int; testcase( pMem->u.i<0 ); return 2; } case 3: { /* 3-byte signed integer */ + /* EVIDENCE-OF: R-37839-54301 Value is a big-endian 24-bit + ** twos-complement integer. */ pMem->u.i = THREE_BYTE_INT(buf); pMem->flags = MEM_Int; testcase( pMem->u.i<0 ); return 3; } case 4: { /* 4-byte signed integer */ + /* EVIDENCE-OF: R-01849-26079 Value is a big-endian 32-bit + ** twos-complement integer. */ pMem->u.i = FOUR_BYTE_INT(buf); pMem->flags = MEM_Int; testcase( pMem->u.i<0 ); return 4; } case 5: { /* 6-byte signed integer */ + /* EVIDENCE-OF: R-50385-09674 Value is a big-endian 48-bit + ** twos-complement integer. */ pMem->u.i = FOUR_BYTE_UINT(buf+2) + (((i64)1)<<32)*TWO_BYTE_INT(buf); pMem->flags = MEM_Int; testcase( pMem->u.i<0 ); @@ -3124,11 +3139,17 @@ u32 sqlite3VdbeSerialGet( } case 8: /* Integer 0 */ case 9: { /* Integer 1 */ + /* EVIDENCE-OF: R-12976-22893 Value is the integer 0. */ + /* EVIDENCE-OF: R-18143-12121 Value is the integer 1. */ pMem->u.i = serial_type-8; pMem->flags = MEM_Int; return 0; } default: { + /* EVIDENCE-OF: R-14606-31564 Value is a BLOB that is (N-12)/2 bytes in + ** length. + ** EVIDENCE-OF: R-28401-00140 Value is a string in the text encoding and + ** (N-13)/2 bytes in length. */ static const u16 aFlag[] = { MEM_Blob|MEM_Ephem, MEM_Str|MEM_Ephem }; pMem->z = (char *)buf; pMem->n = (serial_type-12)/2; From 1b40e63f9bdf8f51241a0511a7c9bb663fe419ed Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 20 Nov 2014 02:58:10 +0000 Subject: [PATCH 128/133] Fix the encoding of some integers to use the minimum amount of space: -128, -32768, -8388608, -217483648, and -140737488355328. FossilOrigin-Name: 2d7c8da5f16e64eaa7b0c2d66898682ea3d102a0 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbeaux.c | 4 +--- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 065eb19989..fe5a6d1bb4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\ssome\srequirements\smarks\sto\sthe\srecord\sformatting\slogic.\s\sComment\schanges\nonly\s-\sthe\scode\sis\sunaltered. -D 2014-11-20T02:18:14.206 +C Fix\sthe\sencoding\sof\ssome\sintegers\sto\suse\sthe\sminimum\samount\sof\sspace:\n-128,\s-32768,\s-8388608,\s-217483648,\sand\s-140737488355328. +D 2014-11-20T02:58:10.344 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -295,7 +295,7 @@ F src/vdbe.c ec1f55acef4864520ca2017b9f0d60c2ac1b8b78 F src/vdbe.h 6fc69d9c5e146302c56e163cb4b31d1ee64a18c3 F src/vdbeInt.h 9bb69ff2447c34b6ccc58b34ec35b615f86ead78 F src/vdbeapi.c 07acb615d1e4170e71fc1b0d087f3c53a1ad8e83 -F src/vdbeaux.c e3ae27459a4eb986d2892de2d7f309d18c9971ef +F src/vdbeaux.c 5ce4f414147a3bc3cbcf00ec57f2606c25791629 F src/vdbeblob.c 4af4bfb71f6df7778397b4a0ebc1879793276778 F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f F src/vdbesort.c 87f3923483113d1c95d84640becb4e4946f27d9a @@ -1221,7 +1221,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 6d00bcca6ed1903fb17275752cab71c14392355b -R b487443c841a5e8e1bfed86b80cd7f65 +P 9a9627e178a67bbfc85366aaea900e674d22fb53 +R acb3727f628d804856a9646910d389b3 U drh -Z bbc37d068b0ef41277590cc500b3cc54 +Z ae13a50368fe7d7b53b7e4276c8588bd diff --git a/manifest.uuid b/manifest.uuid index a704e79b38..1ae875991f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9a9627e178a67bbfc85366aaea900e674d22fb53 \ No newline at end of file +2d7c8da5f16e64eaa7b0c2d66898682ea3d102a0 \ No newline at end of file diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 145bf453ad..b8f5bca2c3 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -2883,9 +2883,7 @@ u32 sqlite3VdbeSerialType(Mem *pMem, int file_format){ i64 i = pMem->u.i; u64 u; if( i<0 ){ - if( i<(-MAX_6BYTE) ) return 6; - /* Previous test prevents: u = -(-9223372036854775808) */ - u = -i; + u = ~i; }else{ u = i; } From fdab02635c2304acc99b04272235463e1ed1ac51 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 20 Nov 2014 15:30:50 +0000 Subject: [PATCH 129/133] Ensure that when the number of cells on a page drops to zero that the freelist and fragment counter are both cleared. Also add evidence marks corresponding to file-format documentation. FossilOrigin-Name: ef9fbc08b0a047042deeb2d6007d67028fefb9e2 --- manifest | 12 +++--- manifest.uuid | 2 +- src/btree.c | 102 ++++++++++++++++++++++++++++++++++++++++---------- 3 files changed, 89 insertions(+), 27 deletions(-) diff --git a/manifest b/manifest index fe5a6d1bb4..7541704f57 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\sencoding\sof\ssome\sintegers\sto\suse\sthe\sminimum\samount\sof\sspace:\n-128,\s-32768,\s-8388608,\s-217483648,\sand\s-140737488355328. -D 2014-11-20T02:58:10.344 +C Ensure\sthat\swhen\sthe\snumber\sof\scells\son\sa\spage\sdrops\sto\szero\sthat\sthe\sfreelist\nand\sfragment\scounter\sare\sboth\scleared.\s\sAlso\sadd\sevidence\smarks\scorresponding\nto\sfile-format\sdocumentation. +D 2014-11-20T15:30:50.141 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -173,7 +173,7 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 7ddee9c7d505e07e959a575b18498f17c71e53ea F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c d0a7cfc8788ad028c86020637df70c789615196e +F src/btree.c 4db5e06ca2d1a5437be7075251fa702c76179b0e F src/btree.h e31a3a3ebdedb1caf9bda3ad5dbab3db9b780f6e F src/btreeInt.h 3363e18fd76f69a27a870b25221b2345b3fd4d21 F src/build.c 67bb05b1077e0cdaccb2e36bfcbe7a5df9ed31e8 @@ -1221,7 +1221,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 9a9627e178a67bbfc85366aaea900e674d22fb53 -R acb3727f628d804856a9646910d389b3 +P 2d7c8da5f16e64eaa7b0c2d66898682ea3d102a0 +R aa709859c03f6b1c53283cb1512cbe99 U drh -Z ae13a50368fe7d7b53b7e4276c8588bd +Z 12bf55f57cbfdde86aed8c230487a6ad diff --git a/manifest.uuid b/manifest.uuid index 1ae875991f..e8ed272102 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2d7c8da5f16e64eaa7b0c2d66898682ea3d102a0 \ No newline at end of file +ef9fbc08b0a047042deeb2d6007d67028fefb9e2 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index f61078cac6..5ae5986cfa 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1139,6 +1139,11 @@ static void ptrmapPutOvflPtr(MemPage *pPage, u8 *pCell, int *pRC){ ** end of the page and all free space is collected into one ** big FreeBlk that occurs in between the header and cell ** pointer array and the cell content area. +** +** EVIDENCE-OF: R-44582-60138 SQLite may from time to time reorganize a +** b-tree page so that there are no freeblocks or fragment bytes, all +** unused bytes are contained in the unallocated space region, and all +** cells are packed tightly at the end of the page. */ static int defragmentPage(MemPage *pPage){ int i; /* Loop counter */ @@ -1262,6 +1267,8 @@ static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc, int *pbDefrag){ testcase( x==4 ); testcase( x==3 ); if( x<4 ){ + /* EVIDENCE-OF: R-11498-58022 In a well-formed b-tree page, the total + ** number of bytes in fragments may not exceed 60. */ if( aData[hdr+7]>=60 ){ if( pbDefrag ) *pbDefrag = 1; return 0; @@ -1316,19 +1323,13 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ assert( pPage->cellOffset == hdr + 12 - 4*pPage->leaf ); gap = pPage->cellOffset + 2*pPage->nCell; assert( gap<=65536 ); - top = get2byte(&data[hdr+5]); - if( gap>top ){ - if( top==0 ){ - /* EVIDENCE-OF: R-29356-02391 If the database uses a 65536-byte page size - ** and the reserved space is zero (the usual value for reserved space) - ** then the cell content offset of an empty page wants to be 65536. - ** However, that integer is too large to be stored in a 2-byte unsigned - ** integer, so a value of 0 is used in its place. */ - top = 65536; - }else{ - return SQLITE_CORRUPT_BKPT; - } - } + /* EVIDENCE-OF: R-29356-02391 If the database uses a 65536-byte page size + ** and the reserved space is zero (the usual value for reserved space) + ** then the cell content offset of an empty page wants to be 65536. + ** However, that integer is too large to be stored in a 2-byte unsigned + ** integer, so a value of 0 is used in its place. */ + top = get2byteNotZero(&data[hdr+5]); + if( gap>top ) return SQLITE_CORRUPT_BKPT; /* If there is enough space between gap and top for one more cell pointer ** array entry offset, and if the freelist is not empty, then search the @@ -1497,18 +1498,32 @@ static int decodeFlags(MemPage *pPage, int flagByte){ pPage->childPtrSize = 4-4*pPage->leaf; pBt = pPage->pBt; if( flagByte==(PTF_LEAFDATA | PTF_INTKEY) ){ + /* EVIDENCE-OF: R-03640-13415 A value of 5 means the page is an interior + ** table b-tree page. */ + assert( (PTF_LEAFDATA|PTF_INTKEY)==5 ); + /* EVIDENCE-OF: R-20501-61796 A value of 13 means the page is a leaf + ** table b-tree page. */ + assert( (PTF_LEAFDATA|PTF_INTKEY|PTF_LEAF)==13 ); pPage->intKey = 1; pPage->intKeyLeaf = pPage->leaf; pPage->noPayload = !pPage->leaf; pPage->maxLocal = pBt->maxLeaf; pPage->minLocal = pBt->minLeaf; }else if( flagByte==PTF_ZERODATA ){ + /* EVIDENCE-OF: R-27225-53936 A value of 2 means the page is an interior + ** index b-tree page. */ + assert( (PTF_ZERODATA)==2 ); + /* EVIDENCE-OF: R-16571-11615 A value of 10 means the page is a leaf + ** index b-tree page. */ + assert( (PTF_ZERODATA|PTF_LEAF)==10 ); pPage->intKey = 0; pPage->intKeyLeaf = 0; pPage->noPayload = 0; pPage->maxLocal = pBt->maxLocal; pPage->minLocal = pBt->minLocal; }else{ + /* EVIDENCE-OF: R-47608-56469 Any other value for the b-tree page type is + ** an error. */ return SQLITE_CORRUPT_BKPT; } pPage->max1bytePayload = pBt->max1bytePayload; @@ -1548,21 +1563,33 @@ static int btreeInitPage(MemPage *pPage){ hdr = pPage->hdrOffset; data = pPage->aData; + /* EVIDENCE-OF: R-28594-02890 The one-byte flag at offset 0 indicating + ** the b-tree page type. */ if( decodeFlags(pPage, data[hdr]) ) return SQLITE_CORRUPT_BKPT; assert( pBt->pageSize>=512 && pBt->pageSize<=65536 ); pPage->maskPage = (u16)(pBt->pageSize - 1); pPage->nOverflow = 0; usableSize = pBt->usableSize; - pPage->cellOffset = cellOffset = hdr + 12 - 4*pPage->leaf; + pPage->cellOffset = cellOffset = hdr + 8 + pPage->childPtrSize; pPage->aDataEnd = &data[usableSize]; pPage->aCellIdx = &data[cellOffset]; + /* EVIDENCE-OF: R-58015-48175 The two-byte integer at offset 5 designates + ** the start of the cell content area. A zero value for this integer is + ** interpreted as 65536. */ top = get2byteNotZero(&data[hdr+5]); + /* EVIDENCE-OF: R-37002-32774 The two-byte integer at offset 3 gives the + ** number of cells on the page. */ pPage->nCell = get2byte(&data[hdr+3]); if( pPage->nCell>MX_CELL(pBt) ){ /* To many cells for a single page. The page must be corrupt */ return SQLITE_CORRUPT_BKPT; } testcase( pPage->nCell==MX_CELL(pBt) ); + /* EVIDENCE-OF: R-24089-57979 If a page contains no cells (which is only + ** possible for a root page of a table that contains no rows) then the + ** offset to the cell content area will equal the page size minus the + ** bytes of reserved space. */ + assert( pPage->nCell>0 || top==usableSize || CORRUPT_DB ); /* A malformed database page might cause us to read past the end ** of page when parsing a cell. @@ -1596,13 +1623,20 @@ static int btreeInitPage(MemPage *pPage){ } #endif - /* Compute the total free space on the page */ + /* Compute the total free space on the page + ** EVIDENCE-OF: R-23588-34450 The two-byte integer at offset 1 gives the + ** start of the first freeblock on the page, or is zero if there are no + ** freeblocks. */ pc = get2byte(&data[hdr+1]); - nFree = data[hdr+7] + top; + nFree = data[hdr+7] + top; /* Init nFree to non-freeblock free space */ while( pc>0 ){ u16 next, size; if( pciCellLast ){ - /* Start of free block is off the page */ + /* EVIDENCE-OF: R-55530-52930 In a well-formed b-tree page, there will + ** always be at least one cell before the first freeblock. + ** + ** Or, the freeblock is off the end of the page + */ return SQLITE_CORRUPT_BKPT; } next = get2byte(&data[pc]); @@ -5959,9 +5993,17 @@ static void dropCell(MemPage *pPage, int idx, int sz, int *pRC){ return; } pPage->nCell--; - memmove(ptr, ptr+2, 2*(pPage->nCell - idx)); - put2byte(&data[hdr+3], pPage->nCell); - pPage->nFree += 2; + if( pPage->nCell==0 ){ + memset(&data[hdr+1], 0, 4); + data[hdr+7] = 0; + put2byte(&data[hdr+5], pPage->pBt->usableSize); + pPage->nFree = pPage->pBt->usableSize - pPage->hdrOffset + - pPage->childPtrSize - 8; + }else{ + memmove(ptr, ptr+2, 2*(pPage->nCell - idx)); + put2byte(&data[hdr+3], pPage->nCell); + pPage->nFree += 2; + } } /* @@ -8609,8 +8651,14 @@ static int checkTreePage( assert( contentOffset<=usableSize ); /* Enforced by btreeInitPage() */ memset(hit+contentOffset, 0, usableSize-contentOffset); memset(hit, 1, contentOffset); + /* EVIDENCE-OF: R-37002-32774 The two-byte integer at offset 3 gives the + ** number of cells on the page. */ nCell = get2byte(&data[hdr+3]); + /* EVIDENCE-OF: R-23882-45353 The cell pointer array of a b-tree page + ** immediately follows the b-tree page header. */ cellStart = hdr + 12 - 4*pPage->leaf; + /* EVIDENCE-OF: R-02776-14802 The cell pointer array consists of K 2-byte + ** integer offsets to the cell contents. */ for(i=0; i=pc; j--) hit[j]++; } } + /* EVIDENCE-OF: R-20690-50594 The second field of the b-tree page header + ** is the offset of the first freeblock, or zero if there are no + ** freeblocks on the page. */ i = get2byte(&data[hdr+1]); while( i>0 ){ int size, j; @@ -8633,7 +8684,13 @@ static int checkTreePage( size = get2byte(&data[i+2]); assert( i+size<=usableSize ); /* Enforced by btreeInitPage() */ for(j=i+size-1; j>=i; j--) hit[j]++; + /* EVIDENCE-OF: R-58208-19414 The first 2 bytes of a freeblock are a + ** big-endian integer which is the offset in the b-tree page of the next + ** freeblock in the chain, or zero if the freeblock is the last on the + ** chain. */ j = get2byte(&data[i]); + /* EVIDENCE-OF: R-06866-39125 Freeblocks are always connected in order of + ** increasing offset. */ assert( j==0 || j>i+size ); /* Enforced by btreeInitPage() */ assert( j<=usableSize-4 ); /* Enforced by btreeInitPage() */ i = j; @@ -8647,6 +8704,11 @@ static int checkTreePage( break; } } + /* EVIDENCE-OF: R-43263-13491 The total number of bytes in all fragments + ** is stored in the fifth field of the b-tree page header. + ** EVIDENCE-OF: R-07161-27322 The one-byte integer at offset 7 gives the + ** number of fragmented free bytes within the cell content area. + */ if( cnt!=data[hdr+7] ){ checkAppendMsg(pCheck, "Fragmentation of %d bytes reported as %d on page %d", From 5e3b49bc42c1a10222571bf433045bb03c7057e9 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 20 Nov 2014 19:22:26 +0000 Subject: [PATCH 130/133] Add requirements marks on the built-in collating functions. FossilOrigin-Name: 4b608b62ac8d4eafdb76192b3b5db272332a4bfd --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/main.c | 19 +++++++++++++++---- 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 7541704f57..0e5910fc06 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Ensure\sthat\swhen\sthe\snumber\sof\scells\son\sa\spage\sdrops\sto\szero\sthat\sthe\sfreelist\nand\sfragment\scounter\sare\sboth\scleared.\s\sAlso\sadd\sevidence\smarks\scorresponding\nto\sfile-format\sdocumentation. -D 2014-11-20T15:30:50.141 +C Add\srequirements\smarks\son\sthe\sbuilt-in\scollating\sfunctions. +D 2014-11-20T19:22:26.830 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -195,7 +195,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 F src/loadext.c de741e66e5ddc1598d904d7289239696e40ed994 -F src/main.c d3310d5ed56e246bf1589e47eeaca8be582bd4b8 +F src/main.c 54d0f4896cebc61ae5f831937464953780fe5346 F src/malloc.c 740db54387204c9a2eb67c6d98e68b08e9ef4eab F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c faf615aafd8be74a71494dfa027c113ea5c6615f @@ -1221,7 +1221,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 2d7c8da5f16e64eaa7b0c2d66898682ea3d102a0 -R aa709859c03f6b1c53283cb1512cbe99 +P ef9fbc08b0a047042deeb2d6007d67028fefb9e2 +R 87fc16da633f081b9c3d6138857b30da U drh -Z 12bf55f57cbfdde86aed8c230487a6ad +Z 4f4a01f4cbeff0ca684ca414f4595d57 diff --git a/manifest.uuid b/manifest.uuid index e8ed272102..de14eeaf45 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ef9fbc08b0a047042deeb2d6007d67028fefb9e2 \ No newline at end of file +4b608b62ac8d4eafdb76192b3b5db272332a4bfd \ No newline at end of file diff --git a/src/main.c b/src/main.c index f223b71f74..1d34b13d6f 100644 --- a/src/main.c +++ b/src/main.c @@ -773,13 +773,20 @@ static int binCollFunc( ){ int rc, n; n = nKey1mallocFailed ){ goto opendb_out; } + /* EVIDENCE-OF: R-08308-17224 The default collating function for all + ** strings is BINARY. + */ db->pDfltColl = sqlite3FindCollSeq(db, SQLITE_UTF8, "BINARY", 0); assert( db->pDfltColl!=0 ); - /* Also add a UTF-8 case-insensitive collation sequence. */ - createCollation(db, "NOCASE", SQLITE_UTF8, 0, nocaseCollatingFunc, 0); - /* Parse the filename/URI argument. */ db->openFlags = flags; rc = sqlite3ParseUri(zVfs, zFilename, &flags, &db->pVfs, &zOpen, &zErrMsg); From 341eca7f68e2ad7a6102c98342c23ddaf65f3c89 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 20 Nov 2014 23:03:42 +0000 Subject: [PATCH 131/133] Updates to requirements tags on the mutex documentation. FossilOrigin-Name: fcf8b7e4c6c3893e2004a28dc9f0f677907b4ba9 --- manifest | 12 ++++---- manifest.uuid | 2 +- src/sqlite.h.in | 80 +++++++++++++++++++++++-------------------------- 3 files changed, 45 insertions(+), 49 deletions(-) diff --git a/manifest b/manifest index 0e5910fc06..3b9c283125 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\srequirements\smarks\son\sthe\sbuilt-in\scollating\sfunctions. -D 2014-11-20T19:22:26.830 +C Updates\sto\srequirements\stags\son\sthe\smutex\sdocumentation. +D 2014-11-20T23:03:42.154 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -230,7 +230,7 @@ F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c bc28d5992109717c87804e2eb1a08a7c8cc7a2fd -F src/sqlite.h.in dfbdcd6e52e7652648d8e7634a7c191fcac8e6ce +F src/sqlite.h.in c63db0117aeb749ca02b6016dbbbccbbbd9a141d F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h c9e95b8fa9aee30d46387735c5be73fa58886e38 @@ -1221,7 +1221,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ef9fbc08b0a047042deeb2d6007d67028fefb9e2 -R 87fc16da633f081b9c3d6138857b30da +P 4b608b62ac8d4eafdb76192b3b5db272332a4bfd +R b10e3d696841d651117121c42237df0a U drh -Z 4f4a01f4cbeff0ca684ca414f4595d57 +Z 5d29b887364b9011d92605ad266bc7f8 diff --git a/manifest.uuid b/manifest.uuid index de14eeaf45..fb1afe2201 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4b608b62ac8d4eafdb76192b3b5db272332a4bfd \ No newline at end of file +fcf8b7e4c6c3893e2004a28dc9f0f677907b4ba9 \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index a59599a0c5..f1b917c308 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -5900,34 +5900,34 @@ int sqlite3_vfs_unregister(sqlite3_vfs*); ** ** The SQLite source code contains multiple implementations ** of these mutex routines. An appropriate implementation -** is selected automatically at compile-time. ^(The following +** is selected automatically at compile-time. The following ** implementations are available in the SQLite core: ** **
    **
  • SQLITE_MUTEX_PTHREADS **
  • SQLITE_MUTEX_W32 **
  • SQLITE_MUTEX_NOOP -**
)^ +** ** -** ^The SQLITE_MUTEX_NOOP implementation is a set of routines +** The SQLITE_MUTEX_NOOP implementation is a set of routines ** that does no real locking and is appropriate for use in -** a single-threaded application. ^The SQLITE_MUTEX_PTHREADS and +** a single-threaded application. The SQLITE_MUTEX_PTHREADS and ** SQLITE_MUTEX_W32 implementations are appropriate for use on Unix ** and Windows. ** -** ^(If SQLite is compiled with the SQLITE_MUTEX_APPDEF preprocessor +** If SQLite is compiled with the SQLITE_MUTEX_APPDEF preprocessor ** macro defined (with "-DSQLITE_MUTEX_APPDEF=1"), then no mutex ** implementation is included with the library. In this case the ** application must supply a custom mutex implementation using the ** [SQLITE_CONFIG_MUTEX] option of the sqlite3_config() function ** before calling sqlite3_initialize() or any other public sqlite3_ -** function that calls sqlite3_initialize().)^ +** function that calls sqlite3_initialize(). ** ** ^The sqlite3_mutex_alloc() routine allocates a new -** mutex and returns a pointer to it. ^If it returns NULL -** that means that a mutex could not be allocated. ^SQLite -** will unwind its stack and return an error. ^(The argument -** to sqlite3_mutex_alloc() is one of these integer constants: +** mutex and returns a pointer to it. ^The sqlite3_mutex_alloc() +** routine returns NULL if it is unable to allocate the requested +** mutex. The argument to sqlite3_mutex_alloc() must one of these +** integer constants: ** **
    **
  • SQLITE_MUTEX_FAST @@ -5940,7 +5940,8 @@ int sqlite3_vfs_unregister(sqlite3_vfs*); **
  • SQLITE_MUTEX_STATIC_PMEM **
  • SQLITE_MUTEX_STATIC_APP1 **
  • SQLITE_MUTEX_STATIC_APP2 -**
)^ +**
  • SQLITE_MUTEX_STATIC_APP3 +** ** ** ^The first two constants (SQLITE_MUTEX_FAST and SQLITE_MUTEX_RECURSIVE) ** cause sqlite3_mutex_alloc() to create @@ -5948,14 +5949,14 @@ int sqlite3_vfs_unregister(sqlite3_vfs*); ** is used but not necessarily so when SQLITE_MUTEX_FAST is used. ** The mutex implementation does not need to make a distinction ** between SQLITE_MUTEX_RECURSIVE and SQLITE_MUTEX_FAST if it does -** not want to. ^SQLite will only request a recursive mutex in -** cases where it really needs one. ^If a faster non-recursive mutex +** not want to. SQLite will only request a recursive mutex in +** cases where it really needs one. If a faster non-recursive mutex ** implementation is available on the host platform, the mutex subsystem ** might return such a mutex in response to SQLITE_MUTEX_FAST. ** ** ^The other allowed parameters to sqlite3_mutex_alloc() (anything other ** than SQLITE_MUTEX_FAST and SQLITE_MUTEX_RECURSIVE) each return -** a pointer to a static preexisting mutex. ^Six static mutexes are +** a pointer to a static preexisting mutex. ^Nine static mutexes are ** used by the current version of SQLite. Future versions of SQLite ** may add additional static mutexes. Static mutexes are for internal ** use by SQLite only. Applications that use SQLite mutexes should @@ -5964,16 +5965,13 @@ int sqlite3_vfs_unregister(sqlite3_vfs*); ** ** ^Note that if one of the dynamic mutex parameters (SQLITE_MUTEX_FAST ** or SQLITE_MUTEX_RECURSIVE) is used then sqlite3_mutex_alloc() -** returns a different mutex on every call. ^But for the static +** returns a different mutex on every call. ^For the static ** mutex types, the same mutex is returned on every call that has ** the same type number. ** ** ^The sqlite3_mutex_free() routine deallocates a previously -** allocated dynamic mutex. ^SQLite is careful to deallocate every -** dynamic mutex that it allocates. The dynamic mutexes must not be in -** use when they are deallocated. Attempting to deallocate a static -** mutex results in undefined behavior. ^SQLite never deallocates -** a static mutex. +** allocated dynamic mutex. Attempting to deallocate a static +** mutex results in undefined behavior. ** ** ^The sqlite3_mutex_enter() and sqlite3_mutex_try() routines attempt ** to enter a mutex. ^If another thread is already within the mutex, @@ -5981,23 +5979,21 @@ int sqlite3_vfs_unregister(sqlite3_vfs*); ** SQLITE_BUSY. ^The sqlite3_mutex_try() interface returns [SQLITE_OK] ** upon successful entry. ^(Mutexes created using ** SQLITE_MUTEX_RECURSIVE can be entered multiple times by the same thread. -** In such cases the, +** In such cases, the ** mutex must be exited an equal number of times before another thread -** can enter.)^ ^(If the same thread tries to enter any other -** kind of mutex more than once, the behavior is undefined. -** SQLite will never exhibit -** such behavior in its own use of mutexes.)^ +** can enter.)^ If the same thread tries to enter any mutex other +** than an SQLITE_MUTEX_RECURSIVE more than once, the behavior is undefined. ** ** ^(Some systems (for example, Windows 95) do not support the operation ** implemented by sqlite3_mutex_try(). On those systems, sqlite3_mutex_try() -** will always return SQLITE_BUSY. The SQLite core only ever uses -** sqlite3_mutex_try() as an optimization so this is acceptable behavior.)^ +** will always return SQLITE_BUSY. The SQLite core only ever uses +** sqlite3_mutex_try() as an optimization so this is acceptable +** behavior.)^ ** ** ^The sqlite3_mutex_leave() routine exits a mutex that was -** previously entered by the same thread. ^(The behavior +** previously entered by the same thread. The behavior ** is undefined if the mutex is not currently entered by the -** calling thread or is not currently allocated. SQLite will -** never do either.)^ +** calling thread or is not currently allocated. ** ** ^If the argument to sqlite3_mutex_enter(), sqlite3_mutex_try(), or ** sqlite3_mutex_leave() is a NULL pointer, then all three routines @@ -6018,9 +6014,9 @@ void sqlite3_mutex_leave(sqlite3_mutex*); ** used to allocate and use mutexes. ** ** Usually, the default mutex implementations provided by SQLite are -** sufficient, however the user has the option of substituting a custom +** sufficient, however the application has the option of substituting a custom ** implementation for specialized deployments or systems for which SQLite -** does not provide a suitable implementation. In this case, the user +** does not provide a suitable implementation. In this case, the application ** creates and populates an instance of this structure to pass ** to sqlite3_config() along with the [SQLITE_CONFIG_MUTEX] option. ** Additionally, an instance of this structure can be used as an @@ -6061,13 +6057,13 @@ void sqlite3_mutex_leave(sqlite3_mutex*); ** (i.e. it is acceptable to provide an implementation that segfaults if ** it is passed a NULL pointer). ** -** The xMutexInit() method must be threadsafe. ^It must be harmless to +** The xMutexInit() method must be threadsafe. It must be harmless to ** invoke xMutexInit() multiple times within the same process and without ** intervening calls to xMutexEnd(). Second and subsequent calls to ** xMutexInit() must be no-ops. ** -** ^xMutexInit() must not use SQLite memory allocation ([sqlite3_malloc()] -** and its associates). ^Similarly, xMutexAlloc() must not use SQLite memory +** xMutexInit() must not use SQLite memory allocation ([sqlite3_malloc()] +** and its associates). Similarly, xMutexAlloc() must not use SQLite memory ** allocation for a static mutex. ^However xMutexAlloc() may use SQLite ** memory allocation for a fast or recursive mutex. ** @@ -6093,29 +6089,29 @@ struct sqlite3_mutex_methods { ** CAPI3REF: Mutex Verification Routines ** ** The sqlite3_mutex_held() and sqlite3_mutex_notheld() routines -** are intended for use inside assert() statements. ^The SQLite core +** are intended for use inside assert() statements. The SQLite core ** never uses these routines except inside an assert() and applications -** are advised to follow the lead of the core. ^The SQLite core only +** are advised to follow the lead of the core. The SQLite core only ** provides implementations for these routines when it is compiled -** with the SQLITE_DEBUG flag. ^External mutex implementations +** with the SQLITE_DEBUG flag. External mutex implementations ** are only required to provide these routines if SQLITE_DEBUG is ** defined and if NDEBUG is not defined. ** -** ^These routines should return true if the mutex in their argument +** These routines should return true if the mutex in their argument ** is held or not held, respectively, by the calling thread. ** -** ^The implementation is not required to provide versions of these +** The implementation is not required to provide versions of these ** routines that actually work. If the implementation does not provide working ** versions of these routines, it should at least provide stubs that always ** return true so that one does not get spurious assertion failures. ** -** ^If the argument to sqlite3_mutex_held() is a NULL pointer then +** If the argument to sqlite3_mutex_held() is a NULL pointer then ** the routine should return 1. This seems counter-intuitive since ** clearly the mutex cannot be held if it does not exist. But ** the reason the mutex does not exist is because the build is not ** using mutexes. And we do not want the assert() containing the ** call to sqlite3_mutex_held() to fail, so a non-zero return is -** the appropriate thing to do. ^The sqlite3_mutex_notheld() +** the appropriate thing to do. The sqlite3_mutex_notheld() ** interface should also return 1 when given a NULL pointer. */ #ifndef NDEBUG From 2d8233157da0be2fa529b14c4f625f3c15ec38af Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 20 Nov 2014 23:11:30 +0000 Subject: [PATCH 132/133] Fix a benign test error on PRAGMA collation_list introduced by a recent checkin. FossilOrigin-Name: 332cc9591d05508ac9cb56fde2b82e20e0342d1f --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/pragma.test | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 3b9c283125..3641e5837e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Updates\sto\srequirements\stags\son\sthe\smutex\sdocumentation. -D 2014-11-20T23:03:42.154 +C Fix\sa\sbenign\stest\serror\son\sPRAGMA\scollation_list\nintroduced\sby\sa\srecent\scheckin. +D 2014-11-20T23:11:30.405 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -777,7 +777,7 @@ F test/pcache.test b09104b03160aca0d968d99e8cd2c5b1921a993d F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 F test/percentile.test b98fc868d71eb5619d42a1702e9ab91718cbed54 F test/permutations.test cef25f5e8499a15846eccd06785f17f4180407ab -F test/pragma.test 19d0241a007bcdd77fc2606ec60fc60357e7fc8b +F test/pragma.test 49ac8a73c0daa574824538fed28727d1259fe735 F test/pragma2.test aea7b3d82c76034a2df2b38a13745172ddc0bc13 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 F test/printf2.test b4acd4bf8734243257f01ddefa17c4fb090acc8a @@ -1221,7 +1221,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 4b608b62ac8d4eafdb76192b3b5db272332a4bfd -R b10e3d696841d651117121c42237df0a +P fcf8b7e4c6c3893e2004a28dc9f0f677907b4ba9 +R 97435df5b02a85b2d9795aaf4e4e7c6e U drh -Z 5d29b887364b9011d92605ad266bc7f8 +Z ee8d2a1291f4df2e216b5b6437eb1599 diff --git a/manifest.uuid b/manifest.uuid index fb1afe2201..aa16af69b0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fcf8b7e4c6c3893e2004a28dc9f0f677907b4ba9 \ No newline at end of file +332cc9591d05508ac9cb56fde2b82e20e0342d1f \ No newline at end of file diff --git a/test/pragma.test b/test/pragma.test index 539d867366..e660ab0fe7 100644 --- a/test/pragma.test +++ b/test/pragma.test @@ -1190,13 +1190,13 @@ ifcapable schema_pragmas { execsql2 { pragma collation_list; } - } {seq 0 name NOCASE seq 1 name RTRIM seq 2 name BINARY} + } {seq 0 name RTRIM seq 1 name NOCASE seq 2 name BINARY} do_test pragma-11.2 { db collate New_Collation blah... execsql { pragma collation_list; } - } {0 New_Collation 1 NOCASE 2 RTRIM 3 BINARY} + } {0 New_Collation 1 RTRIM 2 NOCASE 3 BINARY} } ifcapable schema_pragmas&&tempdb { From 643091f07104b3dd2e3793286ddb1f1b5eb99f87 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 20 Nov 2014 23:21:23 +0000 Subject: [PATCH 133/133] Fix a typo in a requirements mark on the abs() SQL function. FossilOrigin-Name: b1e6c02f8b9a2afaa12ac15a33e3f698c3be27d6 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/func.c | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 3641e5837e..3722a0fbac 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sbenign\stest\serror\son\sPRAGMA\scollation_list\nintroduced\sby\sa\srecent\scheckin. -D 2014-11-20T23:11:30.405 +C Fix\sa\stypo\sin\sa\srequirements\smark\son\sthe\sabs()\sSQL\sfunction. +D 2014-11-20T23:21:23.554 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -185,7 +185,7 @@ F src/delete.c 0750b1eb4d96cd3fb2c798599a3a7c85e92f1417 F src/expr.c 73de4c0da2eed6b149d40a05c589dfeb2c4a87a1 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c da985ae673efef2c712caef825a5d2edb087ead7 -F src/func.c ba47c1671ab3cfdafa6e9d6ee490939ea578adee +F src/func.c 6d3c4ebd72aa7923ce9b110a7dc15f9b8c548430 F src/global.c 6ded36dda9466fc1c9a3c5492ded81d79bf3977d F src/hash.c 4263fbc955f26c2e8cdc0cf214bc42435aa4e4f5 F src/hash.h c8f3c31722cf3277d03713909761e152a5b81094 @@ -1221,7 +1221,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P fcf8b7e4c6c3893e2004a28dc9f0f677907b4ba9 -R 97435df5b02a85b2d9795aaf4e4e7c6e +P 332cc9591d05508ac9cb56fde2b82e20e0342d1f +R 5693b17271f5aafc5aa0ee222d419f56 U drh -Z ee8d2a1291f4df2e216b5b6437eb1599 +Z 26e3a9a90b782d10bf5cba52cec231a6 diff --git a/manifest.uuid b/manifest.uuid index aa16af69b0..2d35c62379 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -332cc9591d05508ac9cb56fde2b82e20e0342d1f \ No newline at end of file +b1e6c02f8b9a2afaa12ac15a33e3f698c3be27d6 \ No newline at end of file diff --git a/src/func.c b/src/func.c index cf556e2439..a057993413 100644 --- a/src/func.c +++ b/src/func.c @@ -157,8 +157,8 @@ static void absFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ default: { /* Because sqlite3_value_double() returns 0.0 if the argument is not ** something that can be converted into a number, we have: - ** IMP: R-57326-31541 Abs(X) return 0.0 if X is a string or blob that - ** cannot be converted to a numeric value. + ** IMP: R-01992-00519 Abs(X) returns 0.0 if X is a string or blob + ** that cannot be converted to a numeric value. */ double rVal = sqlite3_value_double(argv[0]); if( rVal<0 ) rVal = -rVal;