diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index 48ab980921..fc71584909 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -319,8 +319,11 @@ struct Fts5IndexIter { i64 iRowid; const u8 *pData; int nData; + u8 bEof; }; +#define sqlite3Fts5IterEof(x) ((x)->bEof) + /* ** Values used as part of the flags argument passed to IndexQuery(). */ @@ -329,6 +332,12 @@ struct Fts5IndexIter { #define FTS5INDEX_QUERY_TEST_NOIDX 0x0004 /* Do not use prefix index */ #define FTS5INDEX_QUERY_SCAN 0x0008 /* Scan query (fts5vocab) */ +/* The following are used internally by the fts5_index.c module. They are +** defined here only to make it easier to avoid clashes with the flags +** above. */ +#define FTS5INDEX_QUERY_SKIPEMPTY 0x0010 +#define FTS5INDEX_QUERY_NOOUTPUT 0x0020 + /* ** Create/destroy an Fts5Index object. */ @@ -384,7 +393,6 @@ int sqlite3Fts5IndexQuery( ** The various operations on open token or token prefix iterators opened ** using sqlite3Fts5IndexQuery(). */ -int sqlite3Fts5IterEof(Fts5IndexIter*); int sqlite3Fts5IterNext(Fts5IndexIter*); int sqlite3Fts5IterNextFrom(Fts5IndexIter*, i64 iMatch); i64 sqlite3Fts5IterRowid(Fts5IndexIter*); diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index 7c8e3c87c1..5006d88d0f 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -303,7 +303,7 @@ static i64 fts5ExprSynonymRowid(Fts5ExprTerm *pTerm, int bDesc, int *pbEof){ assert( bDesc==0 || bDesc==1 ); for(p=pTerm; p; p=p->pSynonym){ if( 0==sqlite3Fts5IterEof(p->pIter) ){ - i64 iRowid = sqlite3Fts5IterRowid(p->pIter); + i64 iRowid = p->pIter->iRowid; if( bRetValid==0 || (bDesc!=(iRowidpSynonym ); for(p=pTerm; p; p=p->pSynonym){ Fts5IndexIter *pIter = p->pIter; - if( sqlite3Fts5IterEof(pIter)==0 && sqlite3Fts5IterRowid(pIter)==iRowid ){ + if( sqlite3Fts5IterEof(pIter)==0 && pIter->iRowid==iRowid ){ if( pIter->nData==0 ) continue; if( nIter==nAlloc ){ int nByte = sizeof(Fts5PoslistReader) * nAlloc * 2; @@ -631,72 +631,6 @@ static int fts5ExprNearIsMatch(int *pRc, Fts5ExprNearset *pNear){ } } -/* -** Advance the first term iterator in the first phrase of pNear. Set output -** variable *pbEof to true if it reaches EOF or if an error occurs. -** -** Return SQLITE_OK if successful, or an SQLite error code if an error -** occurs. -*/ -static int fts5ExprNearAdvanceFirst( - Fts5Expr *pExpr, /* Expression pPhrase belongs to */ - Fts5ExprNode *pNode, /* FTS5_STRING or FTS5_TERM node */ - int bFromValid, - i64 iFrom -){ - Fts5ExprTerm *pTerm = &pNode->pNear->apPhrase[0]->aTerm[0]; - int rc = SQLITE_OK; - - pNode->bNomatch = 0; - if( pTerm->pSynonym ){ - int bEof = 1; - Fts5ExprTerm *p; - - /* Find the firstest rowid any synonym points to. */ - i64 iRowid = fts5ExprSynonymRowid(pTerm, pExpr->bDesc, 0); - - /* Advance each iterator that currently points to iRowid. Or, if iFrom - ** is valid - each iterator that points to a rowid before iFrom. */ - for(p=pTerm; p; p=p->pSynonym){ - if( sqlite3Fts5IterEof(p->pIter)==0 ){ - i64 ii = sqlite3Fts5IterRowid(p->pIter); - if( ii==iRowid - || (bFromValid && ii!=iFrom && (ii>iFrom)==pExpr->bDesc) - ){ - if( bFromValid ){ - rc = sqlite3Fts5IterNextFrom(p->pIter, iFrom); - }else{ - rc = sqlite3Fts5IterNext(p->pIter); - } - if( rc!=SQLITE_OK ) break; - if( sqlite3Fts5IterEof(p->pIter)==0 ){ - bEof = 0; - } - }else{ - bEof = 0; - } - } - } - - /* Set the EOF flag if either all synonym iterators are at EOF or an - ** error has occurred. */ - pNode->bEof = (rc || bEof); - }else{ - Fts5IndexIter *pIter = pTerm->pIter; - - assert( Fts5NodeIsString(pNode) ); - if( bFromValid ){ - rc = sqlite3Fts5IterNextFrom(pIter, iFrom); - }else{ - rc = sqlite3Fts5IterNext(pIter); - } - - pNode->bEof = (rc || sqlite3Fts5IterEof(pIter)); - } - - return rc; -} - /* ** Advance iterator pIter until it points to a value equal to or laster ** than the initial value of *piLast. If this means the iterator points @@ -716,7 +650,7 @@ static int fts5ExprAdvanceto( i64 iLast = *piLast; i64 iRowid; - iRowid = sqlite3Fts5IterRowid(pIter); + iRowid = pIter->iRowid; if( (bDesc==0 && iLast>iRowid) || (bDesc && iLastiRowid; assert( (bDesc==0 && iRowid>=iLast) || (bDesc==1 && iRowid<=iLast) ); } *piLast = iRowid; @@ -745,7 +679,7 @@ static int fts5ExprSynonymAdvanceto( for(p=pTerm; rc==SQLITE_OK && p; p=p->pSynonym){ if( sqlite3Fts5IterEof(p->pIter)==0 ){ - i64 iRowid = sqlite3Fts5IterRowid(p->pIter); + i64 iRowid = p->pIter->iRowid; if( (bDesc==0 && iLast>iRowid) || (bDesc && iLastpIter, iLast); } @@ -809,104 +743,6 @@ static int fts5ExprNearTest( } } -static int fts5ExprTokenTest( - Fts5Expr *pExpr, /* Expression that pNear is a part of */ - Fts5ExprNode *pNode /* The "NEAR" node (FTS5_TERM) */ -){ - /* As this "NEAR" object is actually a single phrase that consists - ** of a single term only, grab pointers into the poslist managed by the - ** fts5_index.c iterator object. This is much faster than synthesizing - ** a new poslist the way we have to for more complicated phrase or NEAR - ** expressions. */ - Fts5ExprPhrase *pPhrase = pNode->pNear->apPhrase[0]; - Fts5IndexIter *pIter = pPhrase->aTerm[0].pIter; - - assert( pNode->eType==FTS5_TERM ); - assert( pNode->pNear->nPhrase==1 && pPhrase->nTerm==1 ); - assert( pPhrase->aTerm[0].pSynonym==0 ); - - pPhrase->poslist.n = pIter->nData; - if( pExpr->pConfig->eDetail==FTS5_DETAIL_FULL ){ - pPhrase->poslist.p = (u8*)pIter->pData; - } - pNode->iRowid = pIter->iRowid; - pNode->bNomatch = (pPhrase->poslist.n==0); - return SQLITE_OK; -} - -/* -** All individual term iterators in pNear are guaranteed to be valid when -** this function is called. This function checks if all term iterators -** point to the same rowid, and if not, advances them until they do. -** If an EOF is reached before this happens, *pbEof is set to true before -** returning. -** -** SQLITE_OK is returned if an error occurs, or an SQLite error code -** otherwise. It is not considered an error code if an iterator reaches -** EOF. -*/ -static int fts5ExprNearNextMatch( - Fts5Expr *pExpr, /* Expression pPhrase belongs to */ - Fts5ExprNode *pNode -){ - Fts5ExprNearset *pNear = pNode->pNear; - Fts5ExprPhrase *pLeft = pNear->apPhrase[0]; - int rc = SQLITE_OK; - i64 iLast; /* Lastest rowid any iterator points to */ - int i, j; /* Phrase and token index, respectively */ - int bMatch; /* True if all terms are at the same rowid */ - const int bDesc = pExpr->bDesc; - - /* Check that this node should not be FTS5_TERM */ - assert( pNear->nPhrase>1 - || pNear->apPhrase[0]->nTerm>1 - || pNear->apPhrase[0]->aTerm[0].pSynonym - ); - - /* Initialize iLast, the "lastest" rowid any iterator points to. If the - ** iterator skips through rowids in the default ascending order, this means - ** the maximum rowid. Or, if the iterator is "ORDER BY rowid DESC", then it - ** means the minimum rowid. */ - if( pLeft->aTerm[0].pSynonym ){ - iLast = fts5ExprSynonymRowid(&pLeft->aTerm[0], bDesc, 0); - }else{ - iLast = sqlite3Fts5IterRowid(pLeft->aTerm[0].pIter); - } - - do { - bMatch = 1; - for(i=0; inPhrase; i++){ - Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; - for(j=0; jnTerm; j++){ - Fts5ExprTerm *pTerm = &pPhrase->aTerm[j]; - if( pTerm->pSynonym ){ - i64 iRowid = fts5ExprSynonymRowid(pTerm, bDesc, 0); - if( iRowid==iLast ) continue; - bMatch = 0; - if( fts5ExprSynonymAdvanceto(pTerm, bDesc, &iLast, &rc) ){ - pNode->bNomatch = 0; - pNode->bEof = 1; - return rc; - } - }else{ - Fts5IndexIter *pIter = pPhrase->aTerm[j].pIter; - i64 iRowid = sqlite3Fts5IterRowid(pIter); - if( iRowid==iLast ) continue; - bMatch = 0; - if( fts5ExprAdvanceto(pIter, bDesc, &iLast, &rc, &pNode->bEof) ){ - return rc; - } - } - } - } - }while( bMatch==0 ); - - pNode->iRowid = iLast; - pNode->bNomatch = ((0==fts5ExprNearTest(&rc, pExpr, pNode)) && rc==SQLITE_OK); - assert( pNode->bEof==0 || pNode->bNomatch==0 ); - - return rc; -} /* ** Initialize all term iterators in the pNear object. If any term is found @@ -957,10 +793,6 @@ static int fts5ExprNearInitAll( return rc; } -/* fts5ExprNodeNext() calls fts5ExprNodeNextMatch(). And vice-versa. */ -static int fts5ExprNodeNextMatch(Fts5Expr*, Fts5ExprNode*); - - /* ** If pExpr is an ASC iterator, this function returns a value with the ** same sign as: @@ -1012,10 +844,275 @@ static void fts5ExprNodeZeroPoslist(Fts5ExprNode *pNode){ } + +/* +** Compare the values currently indicated by the two nodes as follows: +** +** res = (*p1) - (*p2) +** +** Nodes that point to values that come later in the iteration order are +** considered to be larger. Nodes at EOF are the largest of all. +** +** This means that if the iteration order is ASC, then numerically larger +** rowids are considered larger. Or if it is the default DESC, numerically +** smaller rowids are larger. +*/ +static int fts5NodeCompare( + Fts5Expr *pExpr, + Fts5ExprNode *p1, + Fts5ExprNode *p2 +){ + if( p2->bEof ) return -1; + if( p1->bEof ) return +1; + return fts5RowidCmp(pExpr, p1->iRowid, p2->iRowid); +} + +/* +** All individual term iterators in pNear are guaranteed to be valid when +** this function is called. This function checks if all term iterators +** point to the same rowid, and if not, advances them until they do. +** If an EOF is reached before this happens, *pbEof is set to true before +** returning. +** +** SQLITE_OK is returned if an error occurs, or an SQLite error code +** otherwise. It is not considered an error code if an iterator reaches +** EOF. +*/ +static int fts5ExprNodeTest_STRING( + Fts5Expr *pExpr, /* Expression pPhrase belongs to */ + Fts5ExprNode *pNode +){ + Fts5ExprNearset *pNear = pNode->pNear; + Fts5ExprPhrase *pLeft = pNear->apPhrase[0]; + int rc = SQLITE_OK; + i64 iLast; /* Lastest rowid any iterator points to */ + int i, j; /* Phrase and token index, respectively */ + int bMatch; /* True if all terms are at the same rowid */ + const int bDesc = pExpr->bDesc; + + /* Check that this node should not be FTS5_TERM */ + assert( pNear->nPhrase>1 + || pNear->apPhrase[0]->nTerm>1 + || pNear->apPhrase[0]->aTerm[0].pSynonym + ); + + /* Initialize iLast, the "lastest" rowid any iterator points to. If the + ** iterator skips through rowids in the default ascending order, this means + ** the maximum rowid. Or, if the iterator is "ORDER BY rowid DESC", then it + ** means the minimum rowid. */ + if( pLeft->aTerm[0].pSynonym ){ + iLast = fts5ExprSynonymRowid(&pLeft->aTerm[0], bDesc, 0); + }else{ + iLast = pLeft->aTerm[0].pIter->iRowid; + } + + do { + bMatch = 1; + for(i=0; inPhrase; i++){ + Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; + for(j=0; jnTerm; j++){ + Fts5ExprTerm *pTerm = &pPhrase->aTerm[j]; + if( pTerm->pSynonym ){ + i64 iRowid = fts5ExprSynonymRowid(pTerm, bDesc, 0); + if( iRowid==iLast ) continue; + bMatch = 0; + if( fts5ExprSynonymAdvanceto(pTerm, bDesc, &iLast, &rc) ){ + pNode->bNomatch = 0; + pNode->bEof = 1; + return rc; + } + }else{ + Fts5IndexIter *pIter = pPhrase->aTerm[j].pIter; + if( pIter->iRowid==iLast ) continue; + bMatch = 0; + if( fts5ExprAdvanceto(pIter, bDesc, &iLast, &rc, &pNode->bEof) ){ + return rc; + } + } + } + } + }while( bMatch==0 ); + + pNode->iRowid = iLast; + pNode->bNomatch = ((0==fts5ExprNearTest(&rc, pExpr, pNode)) && rc==SQLITE_OK); + assert( pNode->bEof==0 || pNode->bNomatch==0 ); + + return rc; +} + +/* +** Advance the first term iterator in the first phrase of pNear. Set output +** variable *pbEof to true if it reaches EOF or if an error occurs. +** +** Return SQLITE_OK if successful, or an SQLite error code if an error +** occurs. +*/ +static int fts5ExprNodeNext_STRING( + Fts5Expr *pExpr, /* Expression pPhrase belongs to */ + Fts5ExprNode *pNode, /* FTS5_STRING or FTS5_TERM node */ + int bFromValid, + i64 iFrom +){ + Fts5ExprTerm *pTerm = &pNode->pNear->apPhrase[0]->aTerm[0]; + int rc = SQLITE_OK; + + pNode->bNomatch = 0; + if( pTerm->pSynonym ){ + int bEof = 1; + Fts5ExprTerm *p; + + /* Find the firstest rowid any synonym points to. */ + i64 iRowid = fts5ExprSynonymRowid(pTerm, pExpr->bDesc, 0); + + /* Advance each iterator that currently points to iRowid. Or, if iFrom + ** is valid - each iterator that points to a rowid before iFrom. */ + for(p=pTerm; p; p=p->pSynonym){ + if( sqlite3Fts5IterEof(p->pIter)==0 ){ + i64 ii = p->pIter->iRowid; + if( ii==iRowid + || (bFromValid && ii!=iFrom && (ii>iFrom)==pExpr->bDesc) + ){ + if( bFromValid ){ + rc = sqlite3Fts5IterNextFrom(p->pIter, iFrom); + }else{ + rc = sqlite3Fts5IterNext(p->pIter); + } + if( rc!=SQLITE_OK ) break; + if( sqlite3Fts5IterEof(p->pIter)==0 ){ + bEof = 0; + } + }else{ + bEof = 0; + } + } + } + + /* Set the EOF flag if either all synonym iterators are at EOF or an + ** error has occurred. */ + pNode->bEof = (rc || bEof); + }else{ + Fts5IndexIter *pIter = pTerm->pIter; + + assert( Fts5NodeIsString(pNode) ); + if( bFromValid ){ + rc = sqlite3Fts5IterNextFrom(pIter, iFrom); + }else{ + rc = sqlite3Fts5IterNext(pIter); + } + + pNode->bEof = (rc || sqlite3Fts5IterEof(pIter)); + } + + if( pNode->bEof==0 ){ + assert( rc==SQLITE_OK ); + rc = fts5ExprNodeTest_STRING(pExpr, pNode); + } + + return rc; +} + + +static int fts5ExprNodeTest_TERM( + Fts5Expr *pExpr, /* Expression that pNear is a part of */ + Fts5ExprNode *pNode /* The "NEAR" node (FTS5_TERM) */ +){ + /* As this "NEAR" object is actually a single phrase that consists + ** of a single term only, grab pointers into the poslist managed by the + ** fts5_index.c iterator object. This is much faster than synthesizing + ** a new poslist the way we have to for more complicated phrase or NEAR + ** expressions. */ + Fts5ExprPhrase *pPhrase = pNode->pNear->apPhrase[0]; + Fts5IndexIter *pIter = pPhrase->aTerm[0].pIter; + + assert( pNode->eType==FTS5_TERM ); + assert( pNode->pNear->nPhrase==1 && pPhrase->nTerm==1 ); + assert( pPhrase->aTerm[0].pSynonym==0 ); + + pPhrase->poslist.n = pIter->nData; + if( pExpr->pConfig->eDetail==FTS5_DETAIL_FULL ){ + pPhrase->poslist.p = (u8*)pIter->pData; + } + pNode->iRowid = pIter->iRowid; + pNode->bNomatch = (pPhrase->poslist.n==0); + return SQLITE_OK; +} + +/* +** xNext() method for a node of type FTS5_TERM. +*/ +static int fts5ExprNodeNext_TERM( + Fts5Expr *pExpr, + Fts5ExprNode *pNode, + int bFromValid, + i64 iFrom +){ + int rc; + Fts5IndexIter *pIter = pNode->pNear->apPhrase[0]->aTerm[0].pIter; + + assert( pNode->bEof==0 ); + if( bFromValid ){ + rc = sqlite3Fts5IterNextFrom(pIter, iFrom); + }else{ + rc = sqlite3Fts5IterNext(pIter); + } + if( rc==SQLITE_OK && sqlite3Fts5IterEof(pIter)==0 ){ + rc = fts5ExprNodeTest_TERM(pExpr, pNode); + }else{ + pNode->bEof = 1; + pNode->bNomatch = 0; + } + return rc; +} + +static void fts5ExprNodeTest_OR( + Fts5Expr *pExpr, /* Expression of which pNode is a part */ + Fts5ExprNode *pNode /* Expression node to test */ +){ + Fts5ExprNode *pNext = pNode->apChild[0]; + int i; + + for(i=1; inChild; i++){ + Fts5ExprNode *pChild = pNode->apChild[i]; + int cmp = fts5NodeCompare(pExpr, pNext, pChild); + if( cmp>0 || (cmp==0 && pChild->bNomatch==0) ){ + pNext = pChild; + } + } + pNode->iRowid = pNext->iRowid; + pNode->bEof = pNext->bEof; + pNode->bNomatch = pNext->bNomatch; +} + +static int fts5ExprNodeNext_OR( + Fts5Expr *pExpr, + Fts5ExprNode *pNode, + int bFromValid, + i64 iFrom +){ + int i; + i64 iLast = pNode->iRowid; + + for(i=0; inChild; i++){ + Fts5ExprNode *p1 = pNode->apChild[i]; + assert( p1->bEof || fts5RowidCmp(pExpr, p1->iRowid, iLast)>=0 ); + if( p1->bEof==0 ){ + if( (p1->iRowid==iLast) + || (bFromValid && fts5RowidCmp(pExpr, p1->iRowid, iFrom)<0) + ){ + int rc = fts5ExprNodeNext(pExpr, p1, bFromValid, iFrom); + if( rc!=SQLITE_OK ) return rc; + } + } + } + + fts5ExprNodeTest_OR(pExpr, pNode); + return SQLITE_OK; +} + /* ** Argument pNode is an FTS5_AND node. */ -static int fts5ExprAndNextRowid( +static int fts5ExprNodeTest_AND( Fts5Expr *pExpr, /* Expression pPhrase belongs to */ Fts5ExprNode *pAnd /* FTS5_AND node to advance */ ){ @@ -1030,15 +1127,11 @@ static int fts5ExprAndNextRowid( bMatch = 1; for(iChild=0; iChildnChild; iChild++){ Fts5ExprNode *pChild = pAnd->apChild[iChild]; - if( 0 && pChild->eType==FTS5_STRING ){ - /* TODO */ - }else{ - int cmp = fts5RowidCmp(pExpr, iLast, pChild->iRowid); - if( cmp>0 ){ - /* Advance pChild until it points to iLast or laster */ - rc = fts5ExprNodeNext(pExpr, pChild, 1, iLast); - if( rc!=SQLITE_OK ) return rc; - } + int cmp = fts5RowidCmp(pExpr, iLast, pChild->iRowid); + if( cmp>0 ){ + /* Advance pChild until it points to iLast or laster */ + rc = fts5ExprNodeNext(pExpr, pChild, 1, iLast); + if( rc!=SQLITE_OK ) return rc; } /* If the child node is now at EOF, so is the parent AND node. Otherwise, @@ -1068,155 +1161,66 @@ static int fts5ExprAndNextRowid( return SQLITE_OK; } - -/* -** Compare the values currently indicated by the two nodes as follows: -** -** res = (*p1) - (*p2) -** -** Nodes that point to values that come later in the iteration order are -** considered to be larger. Nodes at EOF are the largest of all. -** -** This means that if the iteration order is ASC, then numerically larger -** rowids are considered larger. Or if it is the default DESC, numerically -** smaller rowids are larger. -*/ -static int fts5NodeCompare( - Fts5Expr *pExpr, - Fts5ExprNode *p1, - Fts5ExprNode *p2 -){ - if( p2->bEof ) return -1; - if( p1->bEof ) return +1; - return fts5RowidCmp(pExpr, p1->iRowid, p2->iRowid); -} - -/* -** xNext() method for a node of type FTS5_TERM. -*/ -static int fts5ExprNodeNext_Term( +static int fts5ExprNodeNext_AND( Fts5Expr *pExpr, Fts5ExprNode *pNode, int bFromValid, i64 iFrom ){ - int rc; - Fts5IndexIter *pIter = pNode->pNear->apPhrase[0]->aTerm[0].pIter; - - assert( pNode->bEof==0 ); - if( bFromValid ){ - rc = sqlite3Fts5IterNextFrom(pIter, iFrom); - }else{ - rc = sqlite3Fts5IterNext(pIter); - } - if( rc==SQLITE_OK && sqlite3Fts5IterEof(pIter)==0 ){ - rc = fts5ExprTokenTest(pExpr, pNode); - }else{ - pNode->bEof = 1; - pNode->bNomatch = 0; + int rc = fts5ExprNodeNext(pExpr, pNode->apChild[0], bFromValid, iFrom); + if( rc==SQLITE_OK ){ + rc = fts5ExprNodeTest_AND(pExpr, pNode); } return rc; } -/* -** Advance node iterator pNode, part of expression pExpr. If argument -** bFromValid is zero, then pNode is advanced exactly once. Or, if argument -** bFromValid is non-zero, then pNode is advanced until it is at or past -** rowid value iFrom. Whether "past" means "less than" or "greater than" -** depends on whether this is an ASC or DESC iterator. -*/ -static int fts5ExprNodeNext_Fallback( - Fts5Expr *pExpr, - Fts5ExprNode *pNode, - int bFromValid, - i64 iFrom +static int fts5ExprNodeTest_NOT( + Fts5Expr *pExpr, /* Expression pPhrase belongs to */ + Fts5ExprNode *pNode /* FTS5_NOT node to advance */ ){ int rc = SQLITE_OK; + Fts5ExprNode *p1 = pNode->apChild[0]; + Fts5ExprNode *p2 = pNode->apChild[1]; + assert( pNode->nChild==2 ); - if( pNode->bEof==0 ){ - switch( pNode->eType ){ - case FTS5_STRING: { - rc = fts5ExprNearAdvanceFirst(pExpr, pNode, bFromValid, iFrom); - break; - }; - - case FTS5_TERM: { - Fts5IndexIter *pIter = pNode->pNear->apPhrase[0]->aTerm[0].pIter; - if( bFromValid ){ - rc = sqlite3Fts5IterNextFrom(pIter, iFrom); - }else{ - rc = sqlite3Fts5IterNext(pIter); - } - if( rc==SQLITE_OK && sqlite3Fts5IterEof(pIter)==0 ){ - assert( rc==SQLITE_OK ); - rc = fts5ExprTokenTest(pExpr, pNode); - }else{ - pNode->bEof = 1; - pNode->bNomatch = 0; - } - return rc; - }; - - case FTS5_AND: { - Fts5ExprNode *pLeft = pNode->apChild[0]; - rc = fts5ExprNodeNext(pExpr, pLeft, bFromValid, iFrom); - break; - } - - case FTS5_OR: { - int i; - i64 iLast = pNode->iRowid; - - for(i=0; rc==SQLITE_OK && inChild; i++){ - Fts5ExprNode *p1 = pNode->apChild[i]; - assert( p1->bEof || fts5RowidCmp(pExpr, p1->iRowid, iLast)>=0 ); - if( p1->bEof==0 ){ - if( (p1->iRowid==iLast) - || (bFromValid && fts5RowidCmp(pExpr, p1->iRowid, iFrom)<0) - ){ - rc = fts5ExprNodeNext(pExpr, p1, bFromValid, iFrom); - } - } - } - - break; - } - - default: assert( pNode->eType==FTS5_NOT ); { - assert( pNode->nChild==2 ); - rc = fts5ExprNodeNext(pExpr, pNode->apChild[0], bFromValid, iFrom); - break; - } - } - - if( rc==SQLITE_OK ){ - rc = fts5ExprNodeNextMatch(pExpr, pNode); + while( rc==SQLITE_OK && p1->bEof==0 ){ + int cmp = fts5NodeCompare(pExpr, p1, p2); + if( cmp>0 ){ + rc = fts5ExprNodeNext(pExpr, p2, 1, p1->iRowid); + cmp = fts5NodeCompare(pExpr, p1, p2); } + assert( rc!=SQLITE_OK || cmp<=0 ); + if( cmp || p2->bNomatch ) break; + rc = fts5ExprNodeNext(pExpr, p1, 0, 0); + } + pNode->bEof = p1->bEof; + pNode->bNomatch = p1->bNomatch; + pNode->iRowid = p1->iRowid; + if( p1->bEof ){ + fts5ExprNodeZeroPoslist(p2); } - - /* Assert that if bFromValid was true, either: - ** - ** a) an error occurred, or - ** b) the node is now at EOF, or - ** c) the node is now at or past rowid iFrom. - */ - assert( bFromValid==0 - || rc!=SQLITE_OK /* a */ - || pNode->bEof /* b */ - || pNode->iRowid==iFrom || pExpr->bDesc==(pNode->iRowidbNomatch==0 || rc==SQLITE_OK ); return rc; } +static int fts5ExprNodeNext_NOT( + Fts5Expr *pExpr, + Fts5ExprNode *pNode, + int bFromValid, + i64 iFrom +){ + int rc = fts5ExprNodeNext(pExpr, pNode->apChild[0], bFromValid, iFrom); + if( rc==SQLITE_OK ){ + rc = fts5ExprNodeTest_NOT(pExpr, pNode); + } + return rc; +} /* ** If pNode currently points to a match, this function returns SQLITE_OK ** without modifying it. Otherwise, pNode is advanced until it does point ** to a match or EOF is reached. */ -static int fts5ExprNodeNextMatch( +static int fts5ExprNodeTest( Fts5Expr *pExpr, /* Expression of which pNode is a part */ Fts5ExprNode *pNode /* Expression node to test */ ){ @@ -1225,59 +1229,27 @@ static int fts5ExprNodeNextMatch( switch( pNode->eType ){ case FTS5_STRING: { - /* Advance the iterators until they all point to the same rowid */ - rc = fts5ExprNearNextMatch(pExpr, pNode); + rc = fts5ExprNodeTest_STRING(pExpr, pNode); break; } case FTS5_TERM: { - rc = fts5ExprTokenTest(pExpr, pNode); + rc = fts5ExprNodeTest_TERM(pExpr, pNode); break; } case FTS5_AND: { - rc = fts5ExprAndNextRowid(pExpr, pNode); + rc = fts5ExprNodeTest_AND(pExpr, pNode); break; } case FTS5_OR: { - Fts5ExprNode *pNext = pNode->apChild[0]; - int i; - - for(i=1; inChild; i++){ - Fts5ExprNode *pChild = pNode->apChild[i]; - int cmp = fts5NodeCompare(pExpr, pNext, pChild); - if( cmp>0 || (cmp==0 && pChild->bNomatch==0) ){ - pNext = pChild; - } - } - pNode->iRowid = pNext->iRowid; - pNode->bEof = pNext->bEof; - pNode->bNomatch = pNext->bNomatch; + fts5ExprNodeTest_OR(pExpr, pNode); break; } default: assert( pNode->eType==FTS5_NOT ); { - Fts5ExprNode *p1 = pNode->apChild[0]; - Fts5ExprNode *p2 = pNode->apChild[1]; - assert( pNode->nChild==2 ); - - while( rc==SQLITE_OK && p1->bEof==0 ){ - int cmp = fts5NodeCompare(pExpr, p1, p2); - if( cmp>0 ){ - rc = fts5ExprNodeNext(pExpr, p2, 1, p1->iRowid); - cmp = fts5NodeCompare(pExpr, p1, p2); - } - assert( rc!=SQLITE_OK || cmp<=0 ); - if( cmp || p2->bNomatch ) break; - rc = fts5ExprNodeNext(pExpr, p1, 0, 0); - } - pNode->bEof = p1->bEof; - pNode->bNomatch = p1->bNomatch; - pNode->iRowid = p1->iRowid; - if( p1->bEof ){ - fts5ExprNodeZeroPoslist(p2); - } + rc = fts5ExprNodeTest_NOT(pExpr, pNode); break; } } @@ -1329,7 +1301,7 @@ static int fts5ExprNodeFirst(Fts5Expr *pExpr, Fts5ExprNode *pNode){ } if( rc==SQLITE_OK ){ - rc = fts5ExprNodeNextMatch(pExpr, pNode); + rc = fts5ExprNodeTest(pExpr, pNode); } return rc; } @@ -1365,7 +1337,8 @@ int sqlite3Fts5ExprFirst(Fts5Expr *p, Fts5Index *pIdx, i64 iFirst, int bDesc){ } /* If the iterator is not at a real match, skip forward until it is. */ - while( pRoot->bNomatch && rc==SQLITE_OK && pRoot->bEof==0 ){ + while( pRoot->bNomatch ){ + assert( pRoot->bEof==0 && rc==SQLITE_OK ); rc = fts5ExprNodeNext(p, pRoot, 0, 0); } } @@ -1690,13 +1663,14 @@ int sqlite3Fts5ExprClonePhrase( pNew->apExprPhrase[0] = sCtx.pPhrase; pNew->pRoot->pNear->apPhrase[0] = sCtx.pPhrase; pNew->pRoot->pNear->nPhrase = 1; - pNew->pRoot->xNext = fts5ExprNodeNext_Fallback; sCtx.pPhrase->pNode = pNew->pRoot; if( pOrig->nTerm==1 && pOrig->aTerm[0].pSynonym==0 ){ pNew->pRoot->eType = FTS5_TERM; + pNew->pRoot->xNext = fts5ExprNodeNext_TERM; }else{ pNew->pRoot->eType = FTS5_STRING; + pNew->pRoot->xNext = fts5ExprNodeNext_STRING; } }else{ sqlite3Fts5ExprFree(pNew); @@ -1844,6 +1818,38 @@ void sqlite3Fts5ParseSetColset( } } +static void fts5ExprAssignXNext(Fts5ExprNode *pNode){ + switch( pNode->eType ){ + case FTS5_STRING: { + Fts5ExprNearset *pNear = pNode->pNear; + if( pNear->nPhrase==1 && pNear->apPhrase[0]->nTerm==1 + && pNear->apPhrase[0]->aTerm[0].pSynonym==0 + ){ + pNode->eType = FTS5_TERM; + pNode->xNext = fts5ExprNodeNext_TERM; + }else{ + pNode->xNext = fts5ExprNodeNext_STRING; + } + break; + }; + + case FTS5_OR: { + pNode->xNext = fts5ExprNodeNext_OR; + break; + }; + + case FTS5_AND: { + pNode->xNext = fts5ExprNodeNext_AND; + break; + }; + + default: assert( pNode->eType==FTS5_NOT ); { + pNode->xNext = fts5ExprNodeNext_NOT; + break; + }; + } +} + static void fts5ExprAddChildren(Fts5ExprNode *p, Fts5ExprNode *pSub){ if( p->eType!=FTS5_NOT && pSub->eType==p->eType ){ int nByte = sizeof(Fts5ExprNode*) * pSub->nChild; @@ -1891,20 +1897,18 @@ Fts5ExprNode *sqlite3Fts5ParseNode( pRet = (Fts5ExprNode*)sqlite3Fts5MallocZero(&pParse->rc, nByte); if( pRet ){ - pRet->xNext = fts5ExprNodeNext_Fallback; pRet->eType = eType; pRet->pNear = pNear; + fts5ExprAssignXNext(pRet); if( eType==FTS5_STRING ){ int iPhrase; for(iPhrase=0; iPhrasenPhrase; iPhrase++){ pNear->apPhrase[iPhrase]->pNode = pRet; } - if( pNear->nPhrase==1 && pNear->apPhrase[0]->nTerm==1 ){ - if( pNear->apPhrase[0]->aTerm[0].pSynonym==0 ){ - pRet->eType = FTS5_TERM; - pRet->xNext = fts5ExprNodeNext_Term; - } - }else if( pParse->pConfig->eDetail!=FTS5_DETAIL_FULL ){ + + if( pParse->pConfig->eDetail!=FTS5_DETAIL_FULL + && (pNear->nPhrase!=1 || pNear->apPhrase[0]->nTerm!=1) + ){ assert( pParse->rc==SQLITE_OK ); pParse->rc = SQLITE_ERROR; assert( pParse->zErr==0 ); @@ -1915,6 +1919,7 @@ Fts5ExprNode *sqlite3Fts5ParseNode( sqlite3_free(pRet); pRet = 0; } + }else{ fts5ExprAddChildren(pRet, pLeft); fts5ExprAddChildren(pRet, pRight); diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index f30c38d91a..0461f3a778 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -518,7 +518,6 @@ struct Fts5Iter { int nSeg; /* Size of aSeg[] array */ int bRev; /* True to iterate in reverse order */ u8 bSkipEmpty; /* True to skip deleted entries */ - u8 bEof; /* True at EOF */ u8 bFiltered; /* True if column-filter already applied */ i64 iSwitchRowid; /* Firstest rowid of other than aFirst[1] */ @@ -2265,9 +2264,6 @@ static void fts5SegIterSeekInit( int bGe = (flags & FTS5INDEX_QUERY_SCAN); int bDlidx = 0; /* True if there is a doclist-index */ - static int nCall = 0; - nCall++; - assert( bGe==0 || (flags & FTS5INDEX_QUERY_DESC)==0 ); assert( pTerm && nTerm ); memset(pIter, 0, sizeof(*pIter)); @@ -2459,7 +2455,7 @@ static void fts5AssertMultiIterSetup(Fts5Index *p, Fts5Iter *pIter){ Fts5SegIter *pFirst = &pIter->aSeg[ pIter->aFirst[1].iFirst ]; int i; - assert( (pFirst->pLeaf==0)==pIter->bEof ); + assert( (pFirst->pLeaf==0)==pIter->base.bEof ); /* Check that pIter->iSwitchRowid is set correctly. */ for(i=0; inSeg; i++){ @@ -2731,7 +2727,7 @@ static int fts5MultiIterAdvanceRowid( */ static void fts5MultiIterSetEof(Fts5Iter *pIter){ Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ]; - pIter->bEof = pSeg->pLeaf==0; + pIter->base.bEof = pSeg->pLeaf==0; pIter->iSwitchRowid = pSeg->iRowid; } @@ -2833,194 +2829,114 @@ static Fts5Iter *fts5MultiIterAlloc( return pNew; } -/* -** Allocate a new Fts5Iter object. -** -** The new object will be used to iterate through data in structure pStruct. -** If iLevel is -ve, then all data in all segments is merged. Or, if iLevel -** is zero or greater, data from the first nSegment segments on level iLevel -** is merged. -** -** The iterator initially points to the first term/rowid entry in the -** iterated data. -*/ -static void fts5MultiIterNew( - Fts5Index *p, /* FTS5 backend to iterate within */ - Fts5Structure *pStruct, /* Structure of specific index */ - int bSkipEmpty, /* True to ignore delete-keys */ - int flags, /* FTS5INDEX_QUERY_XXX flags */ - const u8 *pTerm, int nTerm, /* Term to seek to (or NULL/0) */ - int iLevel, /* Level to iterate (-1 for all) */ - int nSegment, /* Number of segments to merge (iLevel>=0) */ - Fts5Iter **ppOut /* New object */ +static void fts5PoslistCallback( + Fts5Index *p, + void *pContext, + const u8 *pChunk, int nChunk ){ - int nSeg = 0; /* Number of segment-iters in use */ - int iIter = 0; /* */ - int iSeg; /* Used to iterate through segments */ - Fts5Buffer buf = {0,0,0}; /* Buffer used by fts5SegIterSeekInit() */ - Fts5StructureLevel *pLvl; - Fts5Iter *pNew; + assert_nc( nChunk>=0 ); + if( nChunk>0 ){ + fts5BufferSafeAppendBlob((Fts5Buffer*)pContext, pChunk, nChunk); + } +} - assert( (pTerm==0 && nTerm==0) || iLevel<0 ); +typedef struct PoslistCallbackCtx PoslistCallbackCtx; +struct PoslistCallbackCtx { + Fts5Buffer *pBuf; /* Append to this buffer */ + Fts5Colset *pColset; /* Restrict matches to this column */ + int eState; /* See above */ +}; - /* Allocate space for the new multi-seg-iterator. */ - if( p->rc==SQLITE_OK ){ - if( iLevel<0 ){ - assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) ); - nSeg = pStruct->nSegment; - nSeg += (p->pHash ? 1 : 0); - }else{ - nSeg = MIN(pStruct->aLevel[iLevel].nSeg, nSegment); +typedef struct PoslistOffsetsCtx PoslistOffsetsCtx; +struct PoslistOffsetsCtx { + Fts5Buffer *pBuf; /* Append to this buffer */ + Fts5Colset *pColset; /* Restrict matches to this column */ + int iRead; + int iWrite; +}; + +/* +** TODO: Make this more efficient! +*/ +static int fts5IndexColsetTest(Fts5Colset *pColset, int iCol){ + int i; + for(i=0; inCol; i++){ + if( pColset->aiCol[i]==iCol ) return 1; + } + return 0; +} + +static void fts5PoslistOffsetsCallback( + Fts5Index *p, + void *pContext, + const u8 *pChunk, int nChunk +){ + PoslistOffsetsCtx *pCtx = (PoslistOffsetsCtx*)pContext; + assert_nc( nChunk>=0 ); + if( nChunk>0 ){ + int i = 0; + while( iiRead - 2; + pCtx->iRead = iVal; + if( fts5IndexColsetTest(pCtx->pColset, iVal) ){ + fts5BufferSafeAppendVarint(pCtx->pBuf, iVal + 2 - pCtx->iWrite); + pCtx->iWrite = iVal; + } } } - *ppOut = pNew = fts5MultiIterAlloc(p, nSeg); - if( pNew==0 ) return; - pNew->bRev = (0!=(flags & FTS5INDEX_QUERY_DESC)); - pNew->bSkipEmpty = (u8)bSkipEmpty; - pNew->pStruct = pStruct; - fts5StructureRef(pStruct); +} - /* Initialize each of the component segment iterators. */ - if( iLevel<0 ){ - Fts5StructureLevel *pEnd = &pStruct->aLevel[pStruct->nLevel]; - if( p->pHash ){ - /* Add a segment iterator for the current contents of the hash table. */ - Fts5SegIter *pIter = &pNew->aSeg[iIter++]; - fts5SegIterHashInit(p, pTerm, nTerm, flags, pIter); +static void fts5PoslistFilterCallback( + Fts5Index *p, + void *pContext, + const u8 *pChunk, int nChunk +){ + PoslistCallbackCtx *pCtx = (PoslistCallbackCtx*)pContext; + assert_nc( nChunk>=0 ); + if( nChunk>0 ){ + /* Search through to find the first varint with value 1. This is the + ** start of the next columns hits. */ + int i = 0; + int iStart = 0; + + if( pCtx->eState==2 ){ + int iCol; + fts5FastGetVarint32(pChunk, i, iCol); + if( fts5IndexColsetTest(pCtx->pColset, iCol) ){ + pCtx->eState = 1; + fts5BufferSafeAppendVarint(pCtx->pBuf, 1); + }else{ + pCtx->eState = 0; + } } - for(pLvl=&pStruct->aLevel[0]; pLvlnSeg-1; iSeg>=0; iSeg--){ - Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg]; - Fts5SegIter *pIter = &pNew->aSeg[iIter++]; - if( pTerm==0 ){ - fts5SegIterInit(p, pSeg, pIter); + + do { + while( ieState ){ + fts5BufferSafeAppendBlob(pCtx->pBuf, &pChunk[iStart], i-iStart); + } + if( i=nChunk ){ + pCtx->eState = 2; }else{ - fts5SegIterSeekInit(p, &buf, pTerm, nTerm, flags, pSeg, pIter); + fts5FastGetVarint32(pChunk, i, iCol); + pCtx->eState = fts5IndexColsetTest(pCtx->pColset, iCol); + if( pCtx->eState ){ + fts5BufferSafeAppendBlob(pCtx->pBuf, &pChunk[iStart], i-iStart); + iStart = i; + } } } - } - }else{ - pLvl = &pStruct->aLevel[iLevel]; - for(iSeg=nSeg-1; iSeg>=0; iSeg--){ - fts5SegIterInit(p, &pLvl->aSeg[iSeg], &pNew->aSeg[iIter++]); - } + }while( irc==SQLITE_OK ){ - for(iIter=pNew->nSeg-1; iIter>0; iIter--){ - int iEq; - if( (iEq = fts5MultiIterDoCompare(pNew, iIter)) ){ - Fts5SegIter *pSeg = &pNew->aSeg[iEq]; - if( p->rc==SQLITE_OK ) pSeg->xNext(p, pSeg, 0); - fts5MultiIterAdvanced(p, pNew, iEq, iIter); - } - } - fts5MultiIterSetEof(pNew); - fts5AssertMultiIterSetup(p, pNew); - - if( pNew->bSkipEmpty && fts5MultiIterIsEmpty(p, pNew) ){ - fts5MultiIterNext(p, pNew, 0, 0); - } - }else{ - fts5MultiIterFree(p, pNew); - *ppOut = 0; - } - fts5BufferFree(&buf); -} - -/* -** Create an Fts5Iter that iterates through the doclist provided -** as the second argument. -*/ -static void fts5MultiIterNew2( - Fts5Index *p, /* FTS5 backend to iterate within */ - Fts5Data *pData, /* Doclist to iterate through */ - int bDesc, /* True for descending rowid order */ - Fts5Iter **ppOut /* New object */ -){ - Fts5Iter *pNew; - pNew = fts5MultiIterAlloc(p, 2); - if( pNew ){ - Fts5SegIter *pIter = &pNew->aSeg[1]; - - pNew->bFiltered = 1; - pIter->flags = FTS5_SEGITER_ONETERM; - if( pData->szLeaf>0 ){ - pIter->pLeaf = pData; - pIter->iLeafOffset = fts5GetVarint(pData->p, (u64*)&pIter->iRowid); - pIter->iEndofDoclist = pData->nn; - pNew->aFirst[1].iFirst = 1; - if( bDesc ){ - pNew->bRev = 1; - pIter->flags |= FTS5_SEGITER_REVERSE; - fts5SegIterReverseInitPage(p, pIter); - }else{ - fts5SegIterLoadNPos(p, pIter); - } - pData = 0; - }else{ - pNew->bEof = 1; - } - fts5SegIterSetNext(p, pIter); - - *ppOut = pNew; - } - - fts5DataRelease(pData); -} - -/* -** Return true if the iterator is at EOF or if an error has occurred. -** False otherwise. -*/ -static int fts5MultiIterEof(Fts5Index *p, Fts5Iter *pIter){ - assert( p->rc - || (pIter->aSeg[ pIter->aFirst[1].iFirst ].pLeaf==0)==pIter->bEof - ); - return (p->rc || pIter->bEof); -} - -/* -** Return the rowid of the entry that the iterator currently points -** to. If the iterator points to EOF when this function is called the -** results are undefined. -*/ -static i64 fts5MultiIterRowid(Fts5Iter *pIter){ - assert( pIter->aSeg[ pIter->aFirst[1].iFirst ].pLeaf ); - return pIter->aSeg[ pIter->aFirst[1].iFirst ].iRowid; -} - -/* -** Move the iterator to the next entry at or following iMatch. -*/ -static void fts5MultiIterNextFrom( - Fts5Index *p, - Fts5Iter *pIter, - i64 iMatch -){ - while( 1 ){ - i64 iRowid; - fts5MultiIterNext(p, pIter, 1, iMatch); - if( fts5MultiIterEof(p, pIter) ) break; - iRowid = fts5MultiIterRowid(pIter); - if( pIter->bRev==0 && iRowid>=iMatch ) break; - if( pIter->bRev!=0 && iRowid<=iMatch ) break; - } -} - -/* -** Return a pointer to a buffer containing the term associated with the -** entry that the iterator currently points to. -*/ -static const u8 *fts5MultiIterTerm(Fts5Iter *pIter, int *pn){ - Fts5SegIter *p = &pIter->aSeg[ pIter->aFirst[1].iFirst ]; - *pn = p->term.n; - return p->term.p; } static void fts5ChunkIterate( @@ -3064,7 +2980,459 @@ static void fts5ChunkIterate( } } +/* +** Iterator pIter currently points to a valid entry (not EOF). This +** function appends the position list data for the current entry to +** buffer pBuf. It does not make a copy of the position-list size +** field. +*/ +static void fts5SegiterPoslist( + Fts5Index *p, + Fts5SegIter *pSeg, + Fts5Colset *pColset, + Fts5Buffer *pBuf +){ + if( 0==fts5BufferGrow(&p->rc, pBuf, pSeg->nPos) ){ + if( pColset==0 ){ + fts5ChunkIterate(p, pSeg, (void*)pBuf, fts5PoslistCallback); + }else{ + if( p->pConfig->eDetail==FTS5_DETAIL_FULL ){ + PoslistCallbackCtx sCtx; + sCtx.pBuf = pBuf; + sCtx.pColset = pColset; + sCtx.eState = fts5IndexColsetTest(pColset, 0); + assert( sCtx.eState==0 || sCtx.eState==1 ); + fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistFilterCallback); + }else{ + PoslistOffsetsCtx sCtx; + memset(&sCtx, 0, sizeof(sCtx)); + sCtx.pBuf = pBuf; + sCtx.pColset = pColset; + fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistOffsetsCallback); + } + } + } +} +/* +** IN/OUT parameter (*pa) points to a position list n bytes in size. If +** the position list contains entries for column iCol, then (*pa) is set +** to point to the sub-position-list for that column and the number of +** bytes in it returned. Or, if the argument position list does not +** contain any entries for column iCol, return 0. +*/ +static int fts5IndexExtractCol( + const u8 **pa, /* IN/OUT: Pointer to poslist */ + int n, /* IN: Size of poslist in bytes */ + int iCol /* Column to extract from poslist */ +){ + int iCurrent = 0; /* Anything before the first 0x01 is col 0 */ + const u8 *p = *pa; + const u8 *pEnd = &p[n]; /* One byte past end of position list */ + + while( iCol>iCurrent ){ + /* Advance pointer p until it points to pEnd or an 0x01 byte that is + ** not part of a varint. Note that it is not possible for a negative + ** or extremely large varint to occur within an uncorrupted position + ** list. So the last byte of each varint may be assumed to have a clear + ** 0x80 bit. */ + while( *p!=0x01 ){ + while( *p++ & 0x80 ); + if( p>=pEnd ) return 0; + } + *pa = p++; + iCurrent = *p++; + if( iCurrent & 0x80 ){ + p--; + p += fts5GetVarint32(p, iCurrent); + } + } + if( iCol!=iCurrent ) return 0; + + /* Advance pointer p until it points to pEnd or an 0x01 byte that is + ** not part of a varint */ + while( pnCol; i++){ + const u8 *pSub = pPos; + int nSub = fts5IndexExtractCol(&pSub, nPos, pColset->aiCol[i]); + if( nSub ){ + fts5BufferAppendBlob(&rc, pBuf, nSub, pSub); + } + } + return rc; +} + +/* +** xSetOutputs callback used by detail=none tables. +*/ +static void fts5IterSetOutputs_None(Fts5Iter *pIter, Fts5SegIter *pSeg){ + assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_NONE ); + pIter->base.iRowid = pSeg->iRowid; + pIter->base.nData = pSeg->nPos; +} + +/* +** xSetOutputs callback used by detail=full and detail=col tables when no +** column filters are specified. +*/ +static void fts5IterSetOutputs_Nocolset(Fts5Iter *pIter, Fts5SegIter *pSeg){ + pIter->base.iRowid = pSeg->iRowid; + pIter->base.nData = pSeg->nPos; + + assert( pIter->pIndex->pConfig->eDetail!=FTS5_DETAIL_NONE ); + assert( pIter->pColset==0 || pIter->bFiltered ); + + if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf ){ + /* All data is stored on the current page. Populate the output + ** variables to point into the body of the page object. */ + pIter->base.pData = &pSeg->pLeaf->p[pSeg->iLeafOffset]; + }else{ + /* The data is distributed over two or more pages. Copy it into the + ** Fts5Iter.poslist buffer and then set the output pointer to point + ** to this buffer. */ + fts5BufferZero(&pIter->poslist); + fts5SegiterPoslist(pIter->pIndex, pSeg, 0, &pIter->poslist); + pIter->base.pData = pIter->poslist.p; + } +} + +/* +** xSetOutputs callback used by detail=col when there is a column filter +** and there are 100 or more columns. Also called as a fallback from +** fts5IterSetOutputs_Col100 if the column-list spans more than one page. +*/ +static void fts5IterSetOutputs_Col(Fts5Iter *pIter, Fts5SegIter *pSeg){ + fts5BufferZero(&pIter->poslist); + fts5SegiterPoslist(pIter->pIndex, pSeg, pIter->pColset, &pIter->poslist); + pIter->base.iRowid = pSeg->iRowid; + pIter->base.pData = pIter->poslist.p; + pIter->base.nData = pIter->poslist.n; +} + +/* +** xSetOutputs callback used when: +** +** * detail=col, +** * there is a column filter, and +** * the table contains 100 or fewer columns. +** +** The last point is to ensure all column numbers are stored as +** single-byte varints. +*/ +static void fts5IterSetOutputs_Col100(Fts5Iter *pIter, Fts5SegIter *pSeg){ + + assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_COLUMNS ); + assert( pIter->pColset ); + + if( pSeg->iLeafOffset+pSeg->nPos>pSeg->pLeaf->szLeaf ){ + fts5IterSetOutputs_Col(pIter, pSeg); + }else{ + u8 *a = (u8*)&pSeg->pLeaf->p[pSeg->iLeafOffset]; + u8 *pEnd = (u8*)&a[pSeg->nPos]; + int iPrev = 0; + int *aiCol = pIter->pColset->aiCol; + int *aiColEnd = &aiCol[pIter->pColset->nCol]; + + u8 *aOut = pIter->poslist.p; + int iPrevOut = 0; + + pIter->base.iRowid = pSeg->iRowid; + + while( abase.pData = pIter->poslist.p; + pIter->base.nData = aOut - pIter->poslist.p; + } +} + +/* +** xSetOutputs callback used by detail=full when there is a column filter. +*/ +static void fts5IterSetOutputs_Full(Fts5Iter *pIter, Fts5SegIter *pSeg){ + Fts5Colset *pColset = pIter->pColset; + pIter->base.iRowid = pSeg->iRowid; + + assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_FULL ); + assert( pColset ); + + if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf ){ + /* All data is stored on the current page. Populate the output + ** variables to point into the body of the page object. */ + const u8 *a = &pSeg->pLeaf->p[pSeg->iLeafOffset]; + if( pColset->nCol==1 ){ + pIter->base.nData = fts5IndexExtractCol(&a, pSeg->nPos,pColset->aiCol[0]); + pIter->base.pData = a; + }else{ + fts5BufferZero(&pIter->poslist); + fts5IndexExtractColset(pColset, a, pSeg->nPos, &pIter->poslist); + pIter->base.pData = pIter->poslist.p; + pIter->base.nData = pIter->poslist.n; + } + }else{ + /* The data is distributed over two or more pages. Copy it into the + ** Fts5Iter.poslist buffer and then set the output pointer to point + ** to this buffer. */ + fts5BufferZero(&pIter->poslist); + fts5SegiterPoslist(pIter->pIndex, pSeg, pColset, &pIter->poslist); + pIter->base.pData = pIter->poslist.p; + pIter->base.nData = pIter->poslist.n; + } +} + +static void fts5IterSetOutputCb(int *pRc, Fts5Iter *pIter){ + if( *pRc==SQLITE_OK ){ + Fts5Config *pConfig = pIter->pIndex->pConfig; + if( pConfig->eDetail==FTS5_DETAIL_NONE ){ + pIter->xSetOutputs = fts5IterSetOutputs_None; + } + + else if( pIter->pColset==0 || pIter->bFiltered ){ + pIter->xSetOutputs = fts5IterSetOutputs_Nocolset; + } + + else if( pConfig->eDetail==FTS5_DETAIL_FULL ){ + pIter->xSetOutputs = fts5IterSetOutputs_Full; + } + + else{ + assert( pConfig->eDetail==FTS5_DETAIL_COLUMNS ); + if( pConfig->nCol<=100 ){ + pIter->xSetOutputs = fts5IterSetOutputs_Col100; + sqlite3Fts5BufferSize(pRc, &pIter->poslist, pConfig->nCol); + }else{ + pIter->xSetOutputs = fts5IterSetOutputs_Col; + } + } + } +} + + +/* +** Allocate a new Fts5Iter object. +** +** The new object will be used to iterate through data in structure pStruct. +** If iLevel is -ve, then all data in all segments is merged. Or, if iLevel +** is zero or greater, data from the first nSegment segments on level iLevel +** is merged. +** +** The iterator initially points to the first term/rowid entry in the +** iterated data. +*/ +static void fts5MultiIterNew( + Fts5Index *p, /* FTS5 backend to iterate within */ + Fts5Structure *pStruct, /* Structure of specific index */ + int flags, /* FTS5INDEX_QUERY_XXX flags */ + Fts5Colset *pColset, /* Colset to filter on (or NULL) */ + const u8 *pTerm, int nTerm, /* Term to seek to (or NULL/0) */ + int iLevel, /* Level to iterate (-1 for all) */ + int nSegment, /* Number of segments to merge (iLevel>=0) */ + Fts5Iter **ppOut /* New object */ +){ + int nSeg = 0; /* Number of segment-iters in use */ + int iIter = 0; /* */ + int iSeg; /* Used to iterate through segments */ + Fts5Buffer buf = {0,0,0}; /* Buffer used by fts5SegIterSeekInit() */ + Fts5StructureLevel *pLvl; + Fts5Iter *pNew; + + assert( (pTerm==0 && nTerm==0) || iLevel<0 ); + + /* Allocate space for the new multi-seg-iterator. */ + if( p->rc==SQLITE_OK ){ + if( iLevel<0 ){ + assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) ); + nSeg = pStruct->nSegment; + nSeg += (p->pHash ? 1 : 0); + }else{ + nSeg = MIN(pStruct->aLevel[iLevel].nSeg, nSegment); + } + } + *ppOut = pNew = fts5MultiIterAlloc(p, nSeg); + if( pNew==0 ) return; + pNew->bRev = (0!=(flags & FTS5INDEX_QUERY_DESC)); + pNew->bSkipEmpty = (0!=(flags & FTS5INDEX_QUERY_SKIPEMPTY)); + pNew->pStruct = pStruct; + pNew->pColset = pColset; + fts5StructureRef(pStruct); + if( (flags & FTS5INDEX_QUERY_NOOUTPUT)==0 ){ + fts5IterSetOutputCb(&p->rc, pNew); + } + + /* Initialize each of the component segment iterators. */ + if( p->rc==SQLITE_OK ){ + if( iLevel<0 ){ + Fts5StructureLevel *pEnd = &pStruct->aLevel[pStruct->nLevel]; + if( p->pHash ){ + /* Add a segment iterator for the current contents of the hash table. */ + Fts5SegIter *pIter = &pNew->aSeg[iIter++]; + fts5SegIterHashInit(p, pTerm, nTerm, flags, pIter); + } + for(pLvl=&pStruct->aLevel[0]; pLvlnSeg-1; iSeg>=0; iSeg--){ + Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg]; + Fts5SegIter *pIter = &pNew->aSeg[iIter++]; + if( pTerm==0 ){ + fts5SegIterInit(p, pSeg, pIter); + }else{ + fts5SegIterSeekInit(p, &buf, pTerm, nTerm, flags, pSeg, pIter); + } + } + } + }else{ + pLvl = &pStruct->aLevel[iLevel]; + for(iSeg=nSeg-1; iSeg>=0; iSeg--){ + fts5SegIterInit(p, &pLvl->aSeg[iSeg], &pNew->aSeg[iIter++]); + } + } + assert( iIter==nSeg ); + } + + /* If the above was successful, each component iterators now points + ** to the first entry in its segment. In this case initialize the + ** aFirst[] array. Or, if an error has occurred, free the iterator + ** object and set the output variable to NULL. */ + if( p->rc==SQLITE_OK ){ + for(iIter=pNew->nSeg-1; iIter>0; iIter--){ + int iEq; + if( (iEq = fts5MultiIterDoCompare(pNew, iIter)) ){ + Fts5SegIter *pSeg = &pNew->aSeg[iEq]; + if( p->rc==SQLITE_OK ) pSeg->xNext(p, pSeg, 0); + fts5MultiIterAdvanced(p, pNew, iEq, iIter); + } + } + fts5MultiIterSetEof(pNew); + fts5AssertMultiIterSetup(p, pNew); + + if( pNew->bSkipEmpty && fts5MultiIterIsEmpty(p, pNew) ){ + fts5MultiIterNext(p, pNew, 0, 0); + }else if( pNew->base.bEof==0 ){ + Fts5SegIter *pSeg = &pNew->aSeg[pNew->aFirst[1].iFirst]; + pNew->xSetOutputs(pNew, pSeg); + } + + }else{ + fts5MultiIterFree(p, pNew); + *ppOut = 0; + } + fts5BufferFree(&buf); + +} + +/* +** Create an Fts5Iter that iterates through the doclist provided +** as the second argument. +*/ +static void fts5MultiIterNew2( + Fts5Index *p, /* FTS5 backend to iterate within */ + Fts5Data *pData, /* Doclist to iterate through */ + int bDesc, /* True for descending rowid order */ + Fts5Iter **ppOut /* New object */ +){ + Fts5Iter *pNew; + pNew = fts5MultiIterAlloc(p, 2); + if( pNew ){ + Fts5SegIter *pIter = &pNew->aSeg[1]; + + pNew->bFiltered = 1; + pIter->flags = FTS5_SEGITER_ONETERM; + if( pData->szLeaf>0 ){ + pIter->pLeaf = pData; + pIter->iLeafOffset = fts5GetVarint(pData->p, (u64*)&pIter->iRowid); + pIter->iEndofDoclist = pData->nn; + pNew->aFirst[1].iFirst = 1; + if( bDesc ){ + pNew->bRev = 1; + pIter->flags |= FTS5_SEGITER_REVERSE; + fts5SegIterReverseInitPage(p, pIter); + }else{ + fts5SegIterLoadNPos(p, pIter); + } + pData = 0; + }else{ + pNew->base.bEof = 1; + } + fts5SegIterSetNext(p, pIter); + + *ppOut = pNew; + } + + fts5DataRelease(pData); +} + +/* +** Return true if the iterator is at EOF or if an error has occurred. +** False otherwise. +*/ +static int fts5MultiIterEof(Fts5Index *p, Fts5Iter *pIter){ + assert( p->rc + || (pIter->aSeg[ pIter->aFirst[1].iFirst ].pLeaf==0)==pIter->base.bEof + ); + return (p->rc || pIter->base.bEof); +} + +/* +** Return the rowid of the entry that the iterator currently points +** to. If the iterator points to EOF when this function is called the +** results are undefined. +*/ +static i64 fts5MultiIterRowid(Fts5Iter *pIter){ + assert( pIter->aSeg[ pIter->aFirst[1].iFirst ].pLeaf ); + return pIter->aSeg[ pIter->aFirst[1].iFirst ].iRowid; +} + +/* +** Move the iterator to the next entry at or following iMatch. +*/ +static void fts5MultiIterNextFrom( + Fts5Index *p, + Fts5Iter *pIter, + i64 iMatch +){ + while( 1 ){ + i64 iRowid; + fts5MultiIterNext(p, pIter, 1, iMatch); + if( fts5MultiIterEof(p, pIter) ) break; + iRowid = fts5MultiIterRowid(pIter); + if( pIter->bRev==0 && iRowid>=iMatch ) break; + if( pIter->bRev!=0 && iRowid<=iMatch ) break; + } +} + +/* +** Return a pointer to a buffer containing the term associated with the +** entry that the iterator currently points to. +*/ +static const u8 *fts5MultiIterTerm(Fts5Iter *pIter, int *pn){ + Fts5SegIter *p = &pIter->aSeg[ pIter->aFirst[1].iFirst ]; + *pn = p->term.n; + return p->term.p; +} /* ** Allocate a new segment-id for the structure pStruct. The new segment @@ -3685,6 +4053,7 @@ static void fts5IndexMergeLevel( Fts5Buffer term; int bOldest; /* True if the output segment is the oldest */ int eDetail = p->pConfig->eDetail; + const int flags = FTS5INDEX_QUERY_NOOUTPUT; assert( iLvlnLevel ); assert( pLvl->nMerge<=pLvl->nSeg ); @@ -3729,7 +4098,7 @@ static void fts5IndexMergeLevel( bOldest = (pLvlOut->nSeg==1 && pStruct->nLevel==iLvl+2); assert( iLvl>=0 ); - for(fts5MultiIterNew(p, pStruct, 0, 0, 0, 0, iLvl, nInput, &pIter); + for(fts5MultiIterNew(p, pStruct, flags, 0, 0, 0, iLvl, nInput, &pIter); fts5MultiIterEof(p, pIter)==0; fts5MultiIterNext(p, pIter, 0, 0) ){ @@ -4174,281 +4543,28 @@ int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge){ return fts5IndexReturn(p); } -static void fts5PoslistCallback( - Fts5Index *p, - void *pContext, - const u8 *pChunk, int nChunk -){ - assert_nc( nChunk>=0 ); - if( nChunk>0 ){ - fts5BufferSafeAppendBlob((Fts5Buffer*)pContext, pChunk, nChunk); - } -} - -typedef struct PoslistCallbackCtx PoslistCallbackCtx; -struct PoslistCallbackCtx { - Fts5Buffer *pBuf; /* Append to this buffer */ - Fts5Colset *pColset; /* Restrict matches to this column */ - int eState; /* See above */ -}; - -typedef struct PoslistOffsetsCtx PoslistOffsetsCtx; -struct PoslistOffsetsCtx { - Fts5Buffer *pBuf; /* Append to this buffer */ - Fts5Colset *pColset; /* Restrict matches to this column */ - int iRead; - int iWrite; -}; - -/* -** TODO: Make this more efficient! -*/ -static int fts5IndexColsetTest(Fts5Colset *pColset, int iCol){ - int i; - for(i=0; inCol; i++){ - if( pColset->aiCol[i]==iCol ) return 1; - } - return 0; -} - -static void fts5PoslistOffsetsCallback( - Fts5Index *p, - void *pContext, - const u8 *pChunk, int nChunk -){ - PoslistOffsetsCtx *pCtx = (PoslistOffsetsCtx*)pContext; - assert_nc( nChunk>=0 ); - if( nChunk>0 ){ - int i = 0; - while( iiRead - 2; - pCtx->iRead = iVal; - if( fts5IndexColsetTest(pCtx->pColset, iVal) ){ - fts5BufferSafeAppendVarint(pCtx->pBuf, iVal + 2 - pCtx->iWrite); - pCtx->iWrite = iVal; - } - } - } -} - -static void fts5PoslistFilterCallback( - Fts5Index *p, - void *pContext, - const u8 *pChunk, int nChunk -){ - PoslistCallbackCtx *pCtx = (PoslistCallbackCtx*)pContext; - assert_nc( nChunk>=0 ); - if( nChunk>0 ){ - /* Search through to find the first varint with value 1. This is the - ** start of the next columns hits. */ - int i = 0; - int iStart = 0; - - if( pCtx->eState==2 ){ - int iCol; - fts5FastGetVarint32(pChunk, i, iCol); - if( fts5IndexColsetTest(pCtx->pColset, iCol) ){ - pCtx->eState = 1; - fts5BufferSafeAppendVarint(pCtx->pBuf, 1); - }else{ - pCtx->eState = 0; - } - } - - do { - while( ieState ){ - fts5BufferSafeAppendBlob(pCtx->pBuf, &pChunk[iStart], i-iStart); - } - if( i=nChunk ){ - pCtx->eState = 2; - }else{ - fts5FastGetVarint32(pChunk, i, iCol); - pCtx->eState = fts5IndexColsetTest(pCtx->pColset, iCol); - if( pCtx->eState ){ - fts5BufferSafeAppendBlob(pCtx->pBuf, &pChunk[iStart], i-iStart); - iStart = i; - } - } - } - }while( irc, pBuf, pSeg->nPos) ){ - if( pColset==0 ){ - fts5ChunkIterate(p, pSeg, (void*)pBuf, fts5PoslistCallback); - }else{ - if( p->pConfig->eDetail==FTS5_DETAIL_FULL ){ - PoslistCallbackCtx sCtx; - sCtx.pBuf = pBuf; - sCtx.pColset = pColset; - sCtx.eState = fts5IndexColsetTest(pColset, 0); - assert( sCtx.eState==0 || sCtx.eState==1 ); - fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistFilterCallback); - }else{ - PoslistOffsetsCtx sCtx; - memset(&sCtx, 0, sizeof(sCtx)); - sCtx.pBuf = pBuf; - sCtx.pColset = pColset; - fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistOffsetsCallback); - } - } - } -} - -/* -** IN/OUT parameter (*pa) points to a position list n bytes in size. If -** the position list contains entries for column iCol, then (*pa) is set -** to point to the sub-position-list for that column and the number of -** bytes in it returned. Or, if the argument position list does not -** contain any entries for column iCol, return 0. -*/ -static int fts5IndexExtractCol( - const u8 **pa, /* IN/OUT: Pointer to poslist */ - int n, /* IN: Size of poslist in bytes */ - int iCol /* Column to extract from poslist */ -){ - int iCurrent = 0; /* Anything before the first 0x01 is col 0 */ - const u8 *p = *pa; - const u8 *pEnd = &p[n]; /* One byte past end of position list */ - - while( iCol>iCurrent ){ - /* Advance pointer p until it points to pEnd or an 0x01 byte that is - ** not part of a varint. Note that it is not possible for a negative - ** or extremely large varint to occur within an uncorrupted position - ** list. So the last byte of each varint may be assumed to have a clear - ** 0x80 bit. */ - while( *p!=0x01 ){ - while( *p++ & 0x80 ); - if( p>=pEnd ) return 0; - } - *pa = p++; - iCurrent = *p++; - if( iCurrent & 0x80 ){ - p--; - p += fts5GetVarint32(p, iCurrent); - } - } - if( iCol!=iCurrent ) return 0; - - /* Advance pointer p until it points to pEnd or an 0x01 byte that is - ** not part of a varint */ - while( prc, pBuf, iDelta); - return 0; } -/* -** Iterator pMulti currently points to a valid entry (not EOF). This -** function appends the following to buffer pBuf: -** -** * The varint iDelta, and -** * the position list that currently points to, including the size field. -** -** If argument pColset is NULL, then the position list is filtered according -** to pColset before being appended to the buffer. If this means there are -** no entries in the position list, nothing is appended to the buffer (not -** even iDelta). -** -** If an error occurs, an error code is left in p->rc. -*/ -static int fts5AppendPoslist( +static void fts5AppendPoslist( Fts5Index *p, i64 iDelta, Fts5Iter *pMulti, - Fts5Colset *pColset, Fts5Buffer *pBuf ){ - if( p->rc==SQLITE_OK ){ - Fts5SegIter *pSeg = &pMulti->aSeg[ pMulti->aFirst[1].iFirst ]; - assert( fts5MultiIterEof(p, pMulti)==0 ); - assert( pSeg->nPos>0 ); - if( 0==fts5BufferGrow(&p->rc, pBuf, pSeg->nPos+9+9) ){ - if( p->pConfig->eDetail==FTS5_DETAIL_FULL - && pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf - && (pColset==0 || pColset->nCol==1) - ){ - const u8 *pPos = &pSeg->pLeaf->p[pSeg->iLeafOffset]; - int nPos; - if( pColset ){ - nPos = fts5IndexExtractCol(&pPos, pSeg->nPos, pColset->aiCol[0]); - if( nPos==0 ) return 1; - }else{ - nPos = pSeg->nPos; - } - assert( nPos>0 ); - fts5BufferSafeAppendVarint(pBuf, iDelta); - fts5BufferSafeAppendVarint(pBuf, nPos*2); - fts5BufferSafeAppendBlob(pBuf, pPos, nPos); - }else{ - int iSv1; - int iSv2; - int iData; - - /* Append iDelta */ - iSv1 = pBuf->n; - fts5BufferSafeAppendVarint(pBuf, iDelta); - - /* WRITEPOSLISTSIZE */ - iSv2 = pBuf->n; - fts5BufferSafeAppendVarint(pBuf, pSeg->nPos*2); - iData = pBuf->n; - - fts5SegiterPoslist(p, pSeg, pColset, pBuf); - - if( pColset ){ - int nActual = pBuf->n - iData; - if( nActual!=pSeg->nPos ){ - if( nActual==0 ){ - pBuf->n = iSv1; - return 1; - }else{ - int nReq = sqlite3Fts5GetVarintLen((u32)(nActual*2)); - while( iSv2<(iData-nReq) ){ pBuf->p[iSv2++] = 0x80; } - sqlite3Fts5PutVarint(&pBuf->p[iSv2], nActual*2); - } - } - } - } - } + int nData = pMulti->base.nData; + assert( nData>0 ); + if( p->rc==SQLITE_OK && 0==fts5BufferGrow(&p->rc, pBuf, nData+9+9) ){ + fts5BufferSafeAppendVarint(pBuf, iDelta); + fts5BufferSafeAppendVarint(pBuf, nData*2); + fts5BufferSafeAppendBlob(pBuf, pMulti->base.pData, nData); } - - return 0; } @@ -4589,76 +4705,94 @@ static void fts5MergePrefixLists( Fts5Buffer *p2 /* Second list to merge */ ){ if( p2->n ){ - i64 iLastRowid = 0; - Fts5DoclistIter i1; - Fts5DoclistIter i2; - Fts5Buffer out; - Fts5Buffer tmp; - memset(&out, 0, sizeof(out)); - memset(&tmp, 0, sizeof(tmp)); + if( p1->n==0 ){ + fts5BufferSwap(p1, p2); + }else{ + i64 iLastRowid = 0; + Fts5DoclistIter i1; + Fts5DoclistIter i2; + Fts5Buffer out = {0, 0, 0}; + Fts5Buffer tmp = {0, 0, 0}; - sqlite3Fts5BufferSize(&p->rc, &out, p1->n + p2->n); - fts5DoclistIterInit(p1, &i1); - fts5DoclistIterInit(p2, &i2); - while( p->rc==SQLITE_OK && (i1.aPoslist!=0 || i2.aPoslist!=0) ){ - if( i2.aPoslist==0 || (i1.aPoslist && i1.iRowidrc, &out, p1->n + p2->n) ) return; + fts5DoclistIterInit(p1, &i1); + fts5DoclistIterInit(p2, &i2); - Fts5PoslistWriter writer; - memset(&writer, 0, sizeof(writer)); + while( 1 ){ - /* Merge the two position lists. */ - fts5MergeAppendDocid(&out, iLastRowid, i2.iRowid); - fts5BufferZero(&tmp); + if( i1.iRowidrc==SQLITE_OK && (iPos1>=0 || iPos2>=0) ){ - i64 iNew; - if( iPos2<0 || (iPos1>=0 && iPos1=0 || iPos2>=0 ){ + i64 iNew; + if( iPos2<0 || (iPos1>=0 && iPos1rc = sqlite3Fts5PoslistWriterAppend(&tmp, &writer, iNew); + if( p->rc ) goto error_out; } } - if( iNew!=writer.iPrev || tmp.n==0 ){ - p->rc = sqlite3Fts5PoslistWriterAppend(&tmp, &writer, iNew); - } + + /* WRITEPOSLISTSIZE */ + fts5BufferSafeAppendVarint(&out, tmp.n * 2); + fts5BufferSafeAppendBlob(&out, tmp.p, tmp.n); + fts5DoclistIterNext(&i1); + fts5DoclistIterNext(&i2); + if( i1.aPoslist==0 || i2.aPoslist==0 ) break; } - - /* WRITEPOSLISTSIZE */ - fts5BufferSafeAppendVarint(&out, tmp.n * 2); - fts5BufferSafeAppendBlob(&out, tmp.p, tmp.n); - fts5DoclistIterNext(&i1); - fts5DoclistIterNext(&i2); } - } - fts5BufferSet(&p->rc, p1, out.n, out.p); - fts5BufferFree(&tmp); - fts5BufferFree(&out); + if( i1.aPoslist ){ + fts5MergeAppendDocid(&out, iLastRowid, i1.iRowid); + fts5BufferSafeAppendBlob(&out, i1.aPoslist, i1.aEof - i1.aPoslist); + } + else if( i2.aPoslist ){ + fts5MergeAppendDocid(&out, iLastRowid, i2.iRowid); + fts5BufferSafeAppendBlob(&out, i2.aPoslist, i2.aEof - i2.aPoslist); + } + + error_out: + fts5BufferSet(&p->rc, p1, out.n, out.p); + fts5BufferFree(&tmp); + fts5BufferFree(&out); + } } } @@ -4675,7 +4809,7 @@ static void fts5SetupPrefixIter( const int nBuf = 32; void (*xMerge)(Fts5Index*, Fts5Buffer*, Fts5Buffer*); - int (*xAppend)(Fts5Index*, i64, Fts5Iter*, Fts5Colset*, Fts5Buffer*); + void (*xAppend)(Fts5Index*, i64, Fts5Iter*, Fts5Buffer*); if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){ xMerge = fts5MergeRowidLists; xAppend = fts5AppendRowid; @@ -4688,7 +4822,9 @@ static void fts5SetupPrefixIter( pStruct = fts5StructureRead(p); if( aBuf && pStruct ){ - const int flags = FTS5INDEX_QUERY_SCAN; + const int flags = FTS5INDEX_QUERY_SCAN + | FTS5INDEX_QUERY_SKIPEMPTY + | FTS5INDEX_QUERY_NOOUTPUT; int i; i64 iLastRowid = 0; Fts5Iter *p1 = 0; /* Iterator used to gather data from index */ @@ -4697,19 +4833,25 @@ static void fts5SetupPrefixIter( int bNewTerm = 1; memset(&doclist, 0, sizeof(doclist)); - for(fts5MultiIterNew(p, pStruct, 1, flags, pToken, nToken, -1, 0, &p1); + fts5MultiIterNew(p, pStruct, flags, pColset, pToken, nToken, -1, 0, &p1); + fts5IterSetOutputCb(&p->rc, p1); + for( /* no-op */ ; fts5MultiIterEof(p, p1)==0; fts5MultiIterNext2(p, p1, &bNewTerm) ){ - i64 iRowid = fts5MultiIterRowid(p1); - int nTerm; - const u8 *pTerm = fts5MultiIterTerm(p1, &nTerm); + Fts5SegIter *pSeg = &p1->aSeg[ p1->aFirst[1].iFirst ]; + int nTerm = pSeg->term.n; + const u8 *pTerm = pSeg->term.p; + p1->xSetOutputs(p1, pSeg); + assert_nc( memcmp(pToken, pTerm, MIN(nToken, nTerm))<=0 ); if( bNewTerm ){ if( nTerm0 && iRowid<=iLastRowid ){ + if( p1->base.nData==0 ) continue; + + if( p1->base.iRowid<=iLastRowid && doclist.n>0 ){ for(i=0; p->rc==SQLITE_OK && doclist.n; i++){ assert( ibase.iRowid-iLastRowid, p1, &doclist); + iLastRowid = p1->base.iRowid; } for(i=0; inCol; i++){ - const u8 *pSub = pPos; - int nSub = fts5IndexExtractCol(&pSub, nPos, pColset->aiCol[i]); - if( nSub ){ - fts5BufferAppendBlob(&rc, pBuf, nSub, pSub); - } - } - return rc; -} - -/* -** xSetOutputs callback used by detail=none tables. -*/ -static void fts5IterSetOutputs_None(Fts5Iter *pIter, Fts5SegIter *pSeg){ - assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_NONE ); - pIter->base.iRowid = pSeg->iRowid; - pIter->base.nData = pSeg->nPos; -} - -/* -** xSetOutputs callback used by detail=full and detail=col tables when no -** column filters are specified. -*/ -static void fts5IterSetOutputs_Nocolset(Fts5Iter *pIter, Fts5SegIter *pSeg){ - pIter->base.iRowid = pSeg->iRowid; - pIter->base.nData = pSeg->nPos; - - assert( pIter->pIndex->pConfig->eDetail!=FTS5_DETAIL_NONE ); - assert( pIter->pColset==0 || pIter->bFiltered ); - - if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf ){ - /* All data is stored on the current page. Populate the output - ** variables to point into the body of the page object. */ - pIter->base.pData = &pSeg->pLeaf->p[pSeg->iLeafOffset]; - }else{ - /* The data is distributed over two or more pages. Copy it into the - ** Fts5Iter.poslist buffer and then set the output pointer to point - ** to this buffer. */ - fts5BufferZero(&pIter->poslist); - fts5SegiterPoslist(pIter->pIndex, pSeg, 0, &pIter->poslist); - pIter->base.pData = pIter->poslist.p; - } -} - -/* -** xSetOutputs callback used by detail=col when there is a column filter -** and there are 100 or more columns. Also called as a fallback from -** fts5IterSetOutputs_Col100 if the column-list spans more than one page. -*/ -static void fts5IterSetOutputs_Col(Fts5Iter *pIter, Fts5SegIter *pSeg){ - fts5BufferZero(&pIter->poslist); - fts5SegiterPoslist(pIter->pIndex, pSeg, pIter->pColset, &pIter->poslist); - pIter->base.iRowid = pSeg->iRowid; - pIter->base.pData = pIter->poslist.p; - pIter->base.nData = pIter->poslist.n; -} - -/* -** xSetOutputs callback used when: -** -** * detail=col, -** * there is a column filter, and -** * the table contains 100 or fewer columns. -** -** The last point is to ensure all column numbers are stored as -** single-byte varints. -*/ -static void fts5IterSetOutputs_Col100(Fts5Iter *pIter, Fts5SegIter *pSeg){ - - assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_COLUMNS ); - assert( pIter->pColset ); - - if( pSeg->iLeafOffset+pSeg->nPos>pSeg->pLeaf->szLeaf ){ - fts5IterSetOutputs_Col(pIter, pSeg); - }else{ - u8 *a = (u8*)&pSeg->pLeaf->p[pSeg->iLeafOffset]; - u8 *pEnd = (u8*)&a[pSeg->nPos]; - int iPrev = 0; - int *aiCol = pIter->pColset->aiCol; - int *aiColEnd = &aiCol[pIter->pColset->nCol]; - - u8 *aOut = pIter->poslist.p; - int iPrevOut = 0; - - pIter->base.iRowid = pSeg->iRowid; - - while( abase.pData = pIter->poslist.p; - pIter->base.nData = aOut - pIter->poslist.p; - } -} - -/* -** xSetOutputs callback used by detail=full when there is a column filter. -*/ -static void fts5IterSetOutputs_Full(Fts5Iter *pIter, Fts5SegIter *pSeg){ - Fts5Colset *pColset = pIter->pColset; - pIter->base.iRowid = pSeg->iRowid; - - assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_FULL ); - assert( pColset ); - - if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf ){ - /* All data is stored on the current page. Populate the output - ** variables to point into the body of the page object. */ - const u8 *a = &pSeg->pLeaf->p[pSeg->iLeafOffset]; - if( pColset->nCol==1 ){ - pIter->base.nData = fts5IndexExtractCol(&a, pSeg->nPos,pColset->aiCol[0]); - pIter->base.pData = a; - }else{ - fts5BufferZero(&pIter->poslist); - fts5IndexExtractColset(pColset, a, pSeg->nPos, &pIter->poslist); - pIter->base.pData = pIter->poslist.p; - pIter->base.nData = pIter->poslist.n; - } - }else{ - /* The data is distributed over two or more pages. Copy it into the - ** Fts5Iter.poslist buffer and then set the output pointer to point - ** to this buffer. */ - fts5BufferZero(&pIter->poslist); - fts5SegiterPoslist(pIter->pIndex, pSeg, pColset, &pIter->poslist); - pIter->base.pData = pIter->poslist.p; - pIter->base.nData = pIter->poslist.n; - } -} - -static void fts5IterSetOutputCb(int *pRc, Fts5Iter *pIter){ - Fts5Config *pConfig = pIter->pIndex->pConfig; - if( pConfig->eDetail==FTS5_DETAIL_NONE ){ - pIter->xSetOutputs = fts5IterSetOutputs_None; - } - - else if( pIter->pColset==0 || pIter->bFiltered ){ - pIter->xSetOutputs = fts5IterSetOutputs_Nocolset; - } - - else if( pConfig->eDetail==FTS5_DETAIL_FULL ){ - pIter->xSetOutputs = fts5IterSetOutputs_Full; - } - - else{ - assert( pConfig->eDetail==FTS5_DETAIL_COLUMNS ); - if( pConfig->nCol<=100 ){ - pIter->xSetOutputs = fts5IterSetOutputs_Col100; - sqlite3Fts5BufferSize(pRc, &pIter->poslist, pConfig->nCol); - }else{ - pIter->xSetOutputs = fts5IterSetOutputs_Col; - } - } -} - /* ** Open a new iterator to iterate though all rowid that match the ** specified token or token prefix. @@ -5178,7 +5146,9 @@ int sqlite3Fts5IndexQuery( Fts5Structure *pStruct = fts5StructureRead(p); buf.p[0] = (u8)(FTS5_MAIN_PREFIX + iIdx); if( pStruct ){ - fts5MultiIterNew(p, pStruct, 1, flags, buf.p, nToken+1, -1, 0, &pRet); + fts5MultiIterNew(p, pStruct, flags | FTS5INDEX_QUERY_SKIPEMPTY, + pColset, buf.p, nToken+1, -1, 0, &pRet + ); fts5StructureRelease(pStruct); } }else{ @@ -5186,14 +5156,13 @@ int sqlite3Fts5IndexQuery( int bDesc = (flags & FTS5INDEX_QUERY_DESC)!=0; buf.p[0] = FTS5_MAIN_PREFIX; fts5SetupPrefixIter(p, bDesc, buf.p, nToken+1, pColset, &pRet); + fts5IterSetOutputCb(&p->rc, pRet); + if( p->rc==SQLITE_OK ){ + Fts5SegIter *pSeg = &pRet->aSeg[pRet->aFirst[1].iFirst]; + if( p->rc==SQLITE_OK && pSeg->pLeaf ) pRet->xSetOutputs(pRet, pSeg); + } } - if( p->rc==SQLITE_OK ){ - Fts5SegIter *pSeg = &pRet->aSeg[pRet->aFirst[1].iFirst]; - pRet->pColset = pColset; - fts5IterSetOutputCb(&p->rc, pRet); - if( p->rc==SQLITE_OK && pSeg->pLeaf ) pRet->xSetOutputs(pRet, pSeg); - } if( p->rc ){ sqlite3Fts5IterClose(&pRet->base); pRet = 0; @@ -5209,11 +5178,6 @@ int sqlite3Fts5IndexQuery( /* ** Return true if the iterator passed as the only argument is at EOF. */ -int sqlite3Fts5IterEof(Fts5IndexIter *pIter){ - assert( ((Fts5Iter*)pIter)->pIndex->rc==SQLITE_OK ); - return ((Fts5Iter*)pIter)->bEof; -} - /* ** Move to the next matching rowid. */ @@ -5239,7 +5203,7 @@ int sqlite3Fts5IterNextScan(Fts5IndexIter *pIndexIter){ if( pSeg->pLeaf && pSeg->term.p[0]!=FTS5_MAIN_PREFIX ){ fts5DataRelease(pSeg->pLeaf); pSeg->pLeaf = 0; - pIter->bEof = 1; + pIter->base.bEof = 1; } } @@ -5257,13 +5221,6 @@ int sqlite3Fts5IterNextFrom(Fts5IndexIter *pIndexIter, i64 iMatch){ return fts5IndexReturn(pIter->pIndex); } -/* -** Return the current rowid. -*/ -i64 sqlite3Fts5IterRowid(Fts5IndexIter *pIndexIter){ - return fts5MultiIterRowid((Fts5Iter*)pIndexIter); -} - /* ** Return the current term. */ @@ -5450,7 +5407,7 @@ static int fts5QueryCksum( int rc = sqlite3Fts5IndexQuery(p, z, n, flags, 0, &pIter); while( rc==SQLITE_OK && 0==sqlite3Fts5IterEof(pIter) ){ - i64 rowid = sqlite3Fts5IterRowid(pIter); + i64 rowid = pIter->iRowid; if( eDetail==FTS5_DETAIL_NONE ){ cksum ^= sqlite3Fts5IndexEntryCksum(rowid, 0, 0, iIdx, z, n); @@ -5782,6 +5739,7 @@ int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){ u64 cksum3 = 0; /* Checksum based on contents of indexes */ Fts5Buffer term = {0,0,0}; /* Buffer used to hold most recent term */ #endif + const int flags = FTS5INDEX_QUERY_NOOUTPUT; /* Load the FTS index structure */ pStruct = fts5StructureRead(p); @@ -5810,7 +5768,7 @@ int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){ ** same term is performed. cksum3 is calculated based on the entries ** extracted by these queries. */ - for(fts5MultiIterNew(p, pStruct, 0, 0, 0, 0, -1, 0, &pIter); + for(fts5MultiIterNew(p, pStruct, flags, 0, 0, 0, -1, 0, &pIter); fts5MultiIterEof(p, pIter)==0; fts5MultiIterNext(p, pIter, 0, 0) ){ diff --git a/ext/fts5/test/fts5fault1.test b/ext/fts5/test/fts5fault1.test index 429ea39af0..9d63a1175f 100644 --- a/ext/fts5/test/fts5fault1.test +++ b/ext/fts5/test/fts5fault1.test @@ -72,7 +72,7 @@ do_faultsim_test 3 -prep { reset_db do_execsql_test 4.0 { CREATE VIRTUAL TABLE t2 USING fts5(a, b); - INSERT INTO t2 VALUES('m f a jj th q jr ar', 'hj n h h sg j i m'); + INSERT INTO t2 VALUES('m f a jj th q gi ar', 'hj n h h sg j i m'); INSERT INTO t2 VALUES('nr s t g od j kf h', 'sb h aq rg op rb n nl'); INSERT INTO t2 VALUES('do h h pb p p q fr', 'c rj qs or cr a l i'); INSERT INTO t2 VALUES('lk gp t i lq mq qm p', 'h mr g f op ld aj h'); @@ -95,6 +95,7 @@ foreach {tn expr res} { 7 { NEAR(r a, 5) } {9} 8 { m* f* } {1 4 6 8 9 10} 9 { m* + f* } {1 8} + 10 { c NOT p } {5 6 7 10} } { do_faultsim_test 4.$tn -prep { faultsim_restore_and_reopen diff --git a/ext/fts5/test/fts5fault2.test b/ext/fts5/test/fts5fault2.test index ef1df8826a..43c7c7a3dd 100644 --- a/ext/fts5/test/fts5fault2.test +++ b/ext/fts5/test/fts5fault2.test @@ -16,7 +16,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] source $testdir/malloc_common.tcl set testprefix fts5fault2 -# If SQLITE_ENABLE_FTS3 is defined, omit this file. +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { finish_test return diff --git a/ext/fts5/test/fts5fault7.test b/ext/fts5/test/fts5fault7.test index bfe6420615..a35b19ade2 100644 --- a/ext/fts5/test/fts5fault7.test +++ b/ext/fts5/test/fts5fault7.test @@ -99,7 +99,7 @@ do_execsql_test 2.0 { } faultsim_save_and_close -do_faultsim_test 2 -faults oom-* -prep { +do_faultsim_test 2.1 -faults oom-* -prep { faultsim_restore_and_reopen } -body { db eval { UPDATE OR REPLACE xy SET rowid=3 WHERE rowid = 2 } @@ -107,6 +107,13 @@ do_faultsim_test 2 -faults oom-* -prep { faultsim_test_result {0 {}} } +# Test fault-injection when an empty expression is parsed. +# +do_faultsim_test 2.2 -faults oom-* -body { + db eval { SELECT * FROM xy('""') } +} -test { + faultsim_test_result {0 {}} +} finish_test diff --git a/ext/fts5/test/fts5fault8.test b/ext/fts5/test/fts5fault8.test index d93066c6c0..ae5849495b 100644 --- a/ext/fts5/test/fts5fault8.test +++ b/ext/fts5/test/fts5fault8.test @@ -24,32 +24,37 @@ ifcapable !fts5 { foreach_detail_mode $testprefix { -fts5_aux_test_functions db -do_execsql_test 1.0 { - CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%); - INSERT INTO t1 VALUES('a b c d', '1 2 3 4'); - INSERT INTO t1 VALUES('a b a b', NULL); - INSERT INTO t1 VALUES(NULL, '1 2 1 2'); -} - -do_faultsim_test 1 -faults oom-* -body { - execsql { - SELECT rowid, fts5_test_poslist(t1) FROM t1 WHERE t1 MATCH 'b OR 2' + fts5_aux_test_functions db + do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%); + INSERT INTO t1 VALUES('a b c d', '1 2 3 4'); + INSERT INTO t1 VALUES('a b a b', NULL); + INSERT INTO t1 VALUES(NULL, '1 2 1 2'); } -} -test { - faultsim_test_result {0 {1 {0.0.1 1.1.1} 2 {0.0.1 0.0.3} 3 {1.1.1 1.1.3}}} \ - {1 SQLITE_NOMEM} -} - -do_faultsim_test 2 -faults oom-* -body { - execsql { - INSERT INTO t1(t1) VALUES('integrity-check'); + + do_faultsim_test 1 -faults oom-* -body { + execsql { + SELECT rowid, fts5_test_poslist(t1) FROM t1 WHERE t1 MATCH 'b OR 2' + } + } -test { + faultsim_test_result {0 {1 {0.0.1 1.1.1} 2 {0.0.1 0.0.3} 3 {1.1.1 1.1.3}}} \ + {1 SQLITE_NOMEM} + } + + do_faultsim_test 2 -faults oom-* -body { + execsql { INSERT INTO t1(t1) VALUES('integrity-check') } + } -test { + faultsim_test_result {0 {}} {1 SQLITE_NOMEM} } -} -test { - faultsim_test_result {0 {}} {1 SQLITE_NOMEM} -} -} + if {[detail_is_none]==0} { + do_faultsim_test 3 -faults oom-* -body { + execsql { SELECT rowid FROM t1('b:2') } + } -test { + faultsim_test_result {0 {1 3}} {1 SQLITE_NOMEM} + } + } +} ;# foreach_detail_mode... finish_test diff --git a/ext/fts5/test/fts5faultA.test b/ext/fts5/test/fts5faultA.test new file mode 100644 index 0000000000..817ccb4ee8 --- /dev/null +++ b/ext/fts5/test/fts5faultA.test @@ -0,0 +1,64 @@ +# 2016 February 2 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# +# This file is focused on OOM errors. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +source $testdir/malloc_common.tcl +set testprefix fts5faultA + +# If SQLITE_ENABLE_FTS3 is defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +foreach_detail_mode $testprefix { + do_execsql_test 1.0 { + CREATE VIRTUAL TABLE o1 USING fts5(a, detail=%DETAIL%); + INSERT INTO o1(o1, rank) VALUES('pgsz', 32); + + WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<300 ) + INSERT INTO o1 SELECT 'A B C' FROM s; + + INSERT INTO o1 VALUES('A X C'); + + WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<300 ) + INSERT INTO o1 SELECT 'A B C' FROM s; + } + + do_faultsim_test 1 -faults oom* -prep { + sqlite3 db test.db + } -body { + execsql { SELECT rowid FROM o1('a NOT b') } + } -test { + faultsim_test_result {0 301} + } +} + +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE o2 USING fts5(a); + + INSERT INTO o2 VALUES('A B C'); + WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<300 ) + INSERT INTO o2 SELECT group_concat('A B C ') FROM s; +} + +do_faultsim_test 2 -faults oom* -prep { + sqlite3 db test.db +} -body { + execsql { SELECT rowid FROM o2('a+b+c NOT xyz') } +} -test { + faultsim_test_result {0 {1 2}} +} +finish_test + diff --git a/ext/fts5/test/fts5simple3.test b/ext/fts5/test/fts5simple3.test index b84a3e56dd..b6922f84dd 100644 --- a/ext/fts5/test/fts5simple3.test +++ b/ext/fts5/test/fts5simple3.test @@ -39,6 +39,47 @@ do_execsql_test 1.3 { SELECT rowid, fts5_test_collist(t1) FROM t1('b:a'); } {} +#------------------------------------------------------------------------- +# Create detail=col and detail=full tables with 998 columns. +# +foreach_detail_mode $testprefix { + if {[detail_is_none]} continue + + do_test 2.1 { + execsql { DROP TABLE IF EXISTS t2 } + set cols [list] + set vals [list] + for {set i 1} {$i <= 998} {incr i} { + lappend cols "c$i" + lappend vals "'val$i'" + } + execsql "CREATE VIRTUAL TABLE t2 USING fts5(detail=%DETAIL%,[join $cols ,])" + } {} + + do_test 2.2 { + execsql "INSERT INTO t2 VALUES([join $vals ,])" + } {} + + foreach {tn q res} { + 1 { c1:val1 } 1 + 2 { c300:val300 } 1 + 3 { c300:val1 } {} + 4 { c1:val300 } {} + } { + do_execsql_test 2.3.$tn { + SELECT rowid FROM t2($q) + } $res + } +} + +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE x3 USING fts5(one); + INSERT INTO x3 VALUES('a b c'); + INSERT INTO x3 VALUES('c b a'); + INSERT INTO x3 VALUES('o t t'); + SELECT * FROM x3('x OR y OR z'); +} + finish_test diff --git a/ext/fts5/tool/fts5speed.tcl b/ext/fts5/tool/fts5speed.tcl index f0e96e348d..0f38638c26 100644 --- a/ext/fts5/tool/fts5speed.tcl +++ b/ext/fts5/tool/fts5speed.tcl @@ -12,6 +12,10 @@ set Q { {1 "SELECT count(*) FROM t1 WHERE t1 MATCH 'a:t* OR b:t* OR c:t* OR d:t* OR e:t* OR f:t* OR g:t*'"} {1 "SELECT count(*) FROM t1 WHERE t1 MATCH 'a:t*'"} {2 "SELECT count(*) FROM t1 WHERE t1 MATCH 'c:the'"} + + {2 "SELECT count(*) FROM t1 WHERE t1 MATCH 'd:holmes OR e:holmes OR f:holmes OR g:holmes'" } + {2 "SELECT count(*) FROM t1 WHERE t1 MATCH 'd:holmes AND e:holmes AND f:holmes AND g:holmes'" } + {4 "SELECT count(*) FROM t1 WHERE t1 MATCH 'd:holmes NOT e:holmes'" } } proc usage {} { diff --git a/ext/misc/json1.c b/ext/misc/json1.c index 3707516660..3c63b6e946 100644 --- a/ext/misc/json1.c +++ b/ext/misc/json1.c @@ -276,10 +276,33 @@ static void jsonAppendString(JsonString *p, const char *zIn, u32 N){ if( (N+p->nUsed+2 >= p->nAlloc) && jsonGrow(p,N+2)!=0 ) return; p->zBuf[p->nUsed++] = '"'; for(i=0; inUsed+N+3-i > p->nAlloc) && jsonGrow(p,N+3-i)!=0 ) return; p->zBuf[p->nUsed++] = '\\'; + }else if( c<=0x1f ){ + static const char aSpecial[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 'b', 't', 'n', 0, 'f', 'r', 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + assert( sizeof(aSpecial)==32 ); + assert( aSpecial['\b']=='b' ); + assert( aSpecial['\f']=='f' ); + assert( aSpecial['\n']=='n' ); + assert( aSpecial['\r']=='r' ); + assert( aSpecial['\t']=='t' ); + if( aSpecial[c] ){ + c = aSpecial[c]; + goto json_simple_escape; + } + if( (p->nUsed+N+7+i > p->nAlloc) && jsonGrow(p,N+7-i)!=0 ) return; + p->zBuf[p->nUsed++] = '\\'; + p->zBuf[p->nUsed++] = 'u'; + p->zBuf[p->nUsed++] = '0'; + p->zBuf[p->nUsed++] = '0'; + p->zBuf[p->nUsed++] = '0' + (c>>4); + c = "0123456789abcdef"[c&0xf]; } p->zBuf[p->nUsed++] = c; } @@ -320,7 +343,7 @@ static void jsonAppendValue( default: { if( p->bErr==0 ){ sqlite3_result_error(p->pCtx, "JSON cannot hold BLOB values", -1); - p->bErr = 1; + p->bErr = 2; jsonReset(p); } break; @@ -1548,7 +1571,7 @@ static void jsonArrayFinal(sqlite3_context *ctx){ pStr->pCtx = ctx; jsonAppendChar(pStr, ']'); if( pStr->bErr ){ - sqlite3_result_error_nomem(ctx); + if( pStr->bErr==1 ) sqlite3_result_error_nomem(ctx); assert( pStr->bStatic ); }else{ sqlite3_result_text(ctx, pStr->zBuf, pStr->nUsed, @@ -1596,7 +1619,7 @@ static void jsonObjectFinal(sqlite3_context *ctx){ if( pStr ){ jsonAppendChar(pStr, '}'); if( pStr->bErr ){ - sqlite3_result_error_nomem(ctx); + if( pStr->bErr==0 ) sqlite3_result_error_nomem(ctx); assert( pStr->bStatic ); }else{ sqlite3_result_text(ctx, pStr->zBuf, pStr->nUsed, diff --git a/ext/misc/spellfix.c b/ext/misc/spellfix.c index 68bd2afbb2..cac1edcff8 100644 --- a/ext/misc/spellfix.c +++ b/ext/misc/spellfix.c @@ -186,7 +186,7 @@ static const unsigned char className[] = ".ABCDHLRMY9 ?"; ** Return NULL if memory allocation fails. */ static unsigned char *phoneticHash(const unsigned char *zIn, int nIn){ - unsigned char *zOut = sqlite3_malloc( nIn + 1 ); + unsigned char *zOut = sqlite3_malloc64( nIn + 1 ); int i; int nOut = 0; char cPrev = 0x77; @@ -413,7 +413,7 @@ static int editdist1(const char *zA, const char *zB, int *pnMatch){ if( nB<(sizeof(mStack)*4)/(sizeof(mStack[0])*5) ){ m = mStack; }else{ - m = toFree = sqlite3_malloc( (nB+1)*5*sizeof(m[0])/4 ); + m = toFree = sqlite3_malloc64( (nB+1)*5*sizeof(m[0])/4 ); if( m==0 ) return -3; } cx = (char*)&m[nB+1]; @@ -687,7 +687,7 @@ static int editDist3ConfigLoad( if( iCost<0 ) continue; if( pLang==0 || iLang!=iLangPrev ){ EditDist3Lang *pNew; - pNew = sqlite3_realloc(p->a, (p->nLang+1)*sizeof(p->a[0])); + pNew = sqlite3_realloc64(p->a, (p->nLang+1)*sizeof(p->a[0])); if( pNew==0 ){ rc = SQLITE_NOMEM; break; } p->a = pNew; pLang = &p->a[p->nLang]; @@ -709,7 +709,7 @@ static int editDist3ConfigLoad( EditDist3Cost *pCost; int nExtra = nFrom + nTo - 4; if( nExtra<0 ) nExtra = 0; - pCost = sqlite3_malloc( sizeof(*pCost) + nExtra ); + pCost = sqlite3_malloc64( sizeof(*pCost) + nExtra ); if( pCost==0 ){ rc = SQLITE_NOMEM; break; } pCost->nFrom = nFrom; pCost->nTo = nTo; @@ -808,7 +808,7 @@ static EditDist3FromString *editDist3FromStringNew( if( z==0 ) return 0; if( n<0 ) n = (int)strlen(z); - pStr = sqlite3_malloc( sizeof(*pStr) + sizeof(pStr->a[0])*n + n + 1 ); + pStr = sqlite3_malloc64( sizeof(*pStr) + sizeof(pStr->a[0])*n + n + 1 ); if( pStr==0 ) return 0; pStr->a = (EditDist3From*)&pStr[1]; memset(pStr->a, 0, sizeof(pStr->a[0])*n); @@ -833,13 +833,13 @@ static EditDist3FromString *editDist3FromStringNew( if( i+p->nFrom>n ) continue; if( matchFrom(p, z+i, n-i)==0 ) continue; if( p->nTo==0 ){ - apNew = sqlite3_realloc(pFrom->apDel, + apNew = sqlite3_realloc64(pFrom->apDel, sizeof(*apNew)*(pFrom->nDel+1)); if( apNew==0 ) break; pFrom->apDel = apNew; apNew[pFrom->nDel++] = p; }else{ - apNew = sqlite3_realloc(pFrom->apSubst, + apNew = sqlite3_realloc64(pFrom->apSubst, sizeof(*apNew)*(pFrom->nSubst+1)); if( apNew==0 ) break; pFrom->apSubst = apNew; @@ -925,7 +925,7 @@ static int editDist3Core( m = stackSpace; pToFree = 0; }else{ - m = pToFree = sqlite3_malloc( nByte ); + m = pToFree = sqlite3_malloc64( nByte ); if( m==0 ) return -1; /* Out of memory */ } a2 = (EditDist3To*)&m[n]; @@ -940,7 +940,7 @@ static int editDist3Core( if( i2+p->nTo>n2 ) continue; if( matchTo(p, z2+i2, n2-i2)==0 ) continue; a2[i2].nIns++; - apNew = sqlite3_realloc(a2[i2].apIns, sizeof(*apNew)*a2[i2].nIns); + apNew = sqlite3_realloc64(a2[i2].apIns, sizeof(*apNew)*a2[i2].nIns); if( apNew==0 ){ res = -1; /* Out of memory */ goto editDist3Abort; @@ -1118,7 +1118,7 @@ static void editDist3SqlFunc( */ static int editDist3Install(sqlite3 *db){ int rc; - EditDist3Config *pConfig = sqlite3_malloc( sizeof(*pConfig) ); + EditDist3Config *pConfig = sqlite3_malloc64( sizeof(*pConfig) ); if( pConfig==0 ) return SQLITE_NOMEM; memset(pConfig, 0, sizeof(*pConfig)); rc = sqlite3_create_function_v2(db, "editdist3", @@ -1607,7 +1607,7 @@ static const struct { ** should be freed by the caller. */ static unsigned char *transliterate(const unsigned char *zIn, int nIn){ - unsigned char *zOut = sqlite3_malloc( nIn*4 + 1 ); + unsigned char *zOut = sqlite3_malloc64( nIn*4 + 1 ); int c, sz, nOut; if( zOut==0 ) return 0; nOut = 0; @@ -1930,7 +1930,7 @@ static int spellfix1Init( int i; nDbName = (int)strlen(zDbName); - pNew = sqlite3_malloc( sizeof(*pNew) + nDbName + 1); + pNew = sqlite3_malloc64( sizeof(*pNew) + nDbName + 1); if( pNew==0 ){ rc = SQLITE_NOMEM; }else{ @@ -2044,7 +2044,7 @@ static void spellfix1ResetCursor(spellfix1_cursor *pCur){ static void spellfix1ResizeCursor(spellfix1_cursor *pCur, int N){ struct spellfix1_row *aNew; assert( N>=pCur->nRow ); - aNew = sqlite3_realloc(pCur->a, sizeof(pCur->a[0])*N); + aNew = sqlite3_realloc64(pCur->a, sizeof(pCur->a[0])*N); if( aNew==0 && N>0 ){ spellfix1ResetCursor(pCur); sqlite3_free(pCur->a); @@ -2203,7 +2203,7 @@ static int spellfix1BestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ static int spellfix1Open(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ spellfix1_vtab *p = (spellfix1_vtab*)pVTab; spellfix1_cursor *pCur; - pCur = sqlite3_malloc( sizeof(*pCur) ); + pCur = sqlite3_malloc64( sizeof(*pCur) ); if( pCur==0 ) return SQLITE_NOMEM; memset(pCur, 0, sizeof(*pCur)); pCur->pVTab = p; @@ -2417,7 +2417,7 @@ static int spellfix1FilterForMatch( /* Load the cost table if we have not already done so */ if( p->zCostTable!=0 && p->pConfig3==0 ){ - p->pConfig3 = sqlite3_malloc( sizeof(p->pConfig3[0]) ); + p->pConfig3 = sqlite3_malloc64( sizeof(p->pConfig3[0]) ); if( p->pConfig3==0 ) return SQLITE_NOMEM; memset(p->pConfig3, 0, sizeof(p->pConfig3[0])); rc = editDist3ConfigLoad(p->pConfig3, p->db, p->zCostTable); diff --git a/manifest b/manifest index d3554573d4..8ad6e5d2fc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sall\srecent\senhancements\sfrom\strunk. -D 2016-02-02T02:22:30.647 +C Merge\senhancements\sfrom\strunk. +D 2016-02-05T14:15:34.907 F Makefile.in b12720a3957eb741a77a41d3412745b80ac3fa5d F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc c6dfe4708e38482130e3ccc374be6d5ca4aeb26e @@ -13,7 +13,7 @@ F autoconf/INSTALL 83e4a25da9fd053c7b3665eaaaf7919707915903 F autoconf/Makefile.am 1c1657650775960804945dc392e14d9e43c5ed84 F autoconf/Makefile.msc 2ce3b1565a52d7083a1dbf57a1706fd14cc1ee0c F autoconf/README.first 6c4f34fe115ff55d4e8dbfa3cecf04a0188292f7 -F autoconf/README.txt e9757a381e5ce2553dbaa6247bb8ad00eb8d87aa w autoconf/README +F autoconf/README.txt e9757a381e5ce2553dbaa6247bb8ad00eb8d87aa F autoconf/configure.ac 72a5e42beb090b32bca580285dc0ab3c4670adb8 F autoconf/tea/Makefile.in b438a7020446c8a8156e8d97c8914a04833da6fd F autoconf/tea/README 3e9a3c060f29a44344ab50aec506f4db903fb873 @@ -98,13 +98,13 @@ F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0 F ext/fts5/fts5.h ff9c2782e8ed890b0de2f697a8d63971939e70c7 -F ext/fts5/fts5Int.h 6e0f90eb4872654a5b98130dec16965716525c9a +F ext/fts5/fts5Int.h 2095cc38e776f19cc083ca90e00772ea0b204ab3 F ext/fts5/fts5_aux.c b9bcce753ef5b451267b2232f0ca153ddeb3951d F ext/fts5/fts5_buffer.c f6e0c6018ffc8e39fc0b333b5daa8b8d528ae6e4 F ext/fts5/fts5_config.c 0c384ebdd23fd055e2e50a93277b8d59da538238 -F ext/fts5/fts5_expr.c 768d221e592df03b26f46da56aa0a561f00fa4e0 +F ext/fts5/fts5_expr.c ff5c451a6d025909639ac0f0d0af0cc595b50feb F ext/fts5/fts5_hash.c 1b113977296cf4212c6ec667d5e3f2bd18036955 -F ext/fts5/fts5_index.c cd1e4faca8b9adc2d89b367075bf93a7f50c406b +F ext/fts5/fts5_index.c 614c54544f844387ba6728c613a884a1aa151f06 F ext/fts5/fts5_main.c 7e8a5f27d504bc04e3de7f1cba8867f0332aee9d F ext/fts5/fts5_storage.c 2a1f44deae090cd711f02cec0c2af8e660360d24 F ext/fts5/fts5_tcl.c f8731e0508299bd43f1a2eff7dbeaac870768966 @@ -147,15 +147,16 @@ F ext/fts5/test/fts5dlidx.test 13871a14641017ae42f6f1055a8067bafd44cb3d F ext/fts5/test/fts5doclist.test 8edb5b57e5f144030ed74ec00ef6fa4294fed79b F ext/fts5/test/fts5ea.test b01e3a18cdfabbff8104a96a5242a06a68a998a0 F ext/fts5/test/fts5eb.test 021aa80b7ac09b964249aa32ced9ee908703e4aa -F ext/fts5/test/fts5fault1.test 4b39c47ca3544615daa8a2f733b911fa08022c77 -F ext/fts5/test/fts5fault2.test 28c36c843bb39ae855ba79827417ecc37f114341 +F ext/fts5/test/fts5fault1.test e09040d3e17b8c0837101e8c79c8a874c4376fb7 +F ext/fts5/test/fts5fault2.test d8c6c7f916ccbdfc10b2c69530e9dd3bc8313232 F ext/fts5/test/fts5fault3.test d6e9577d4312e331a913c72931bf131704efc8f3 F ext/fts5/test/fts5fault4.test 532b6dacb963016cbf7003196bd87fb366540277 F ext/fts5/test/fts5fault5.test 10c13a783de3f42a21e3e53e123b62ed0c3a1618 F ext/fts5/test/fts5fault6.test 9682664d679643ac6736e90c225526cc84073cda -F ext/fts5/test/fts5fault7.test 01be274bfc8d9bf22451a3bf5892e9399d044f1b -F ext/fts5/test/fts5fault8.test f2d8a2b673a5f72ca1fa0e85bdbfb2041ffd347d +F ext/fts5/test/fts5fault7.test cb14ea3c1f42394f06f2284abc58eecee6ff8080 +F ext/fts5/test/fts5fault8.test 430837fe6dd0511fd3aea52bd602ac02441bcb58 F ext/fts5/test/fts5fault9.test e10e395428a9ea0596ebe752ff7123d16ab78e08 +F ext/fts5/test/fts5faultA.test fa5d59c0ff62b7125cd14eee38ded1c46e15a7ea F ext/fts5/test/fts5full.test 6f6143af0c6700501d9fd597189dfab1555bb741 F ext/fts5/test/fts5hash.test 00668f6fa9b9bffbd7c1be29f408aa2bdade0451 F ext/fts5/test/fts5integrity.test f5e4f8d284385875068ad0f3e894ce43e9de835d @@ -177,7 +178,7 @@ F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17 F ext/fts5/test/fts5rowid.test 16908a99d6efc9ba21081b4f2b86b3fc699839a6 F ext/fts5/test/fts5simple.test 7fcacfa473a37355af2e60096650c87b5ba8f3ba F ext/fts5/test/fts5simple2.test 98377ae1ff7749a42c21fe1a139c1ed312522c46 -F ext/fts5/test/fts5simple3.test e671b36bc4dbd4f5095e66cb04473cba9f680f53 +F ext/fts5/test/fts5simple3.test 8e71733b3d1b0e695011d02c68ebc5ca40b6124e F ext/fts5/test/fts5synonym.test 6475d189c2e20d60795808f83e36bf9318708d48 F ext/fts5/test/fts5synonym2.test aa4c43bd3b691ff80f658cb064f5ab40690e834e F ext/fts5/test/fts5tok1.test beb894c6f3468f10a574302f69ebe4436b0287c7 @@ -190,7 +191,7 @@ F ext/fts5/test/fts5unindexed.test e9539d5b78c677315e7ed8ea911d4fd25437c680 F ext/fts5/test/fts5update.test 57c7012a7919889048947addae10e0613df45529 F ext/fts5/test/fts5version.test 978f59541d8cef7e8591f8be2115ec5ccb863e2e F ext/fts5/test/fts5vocab.test 480d780aa6b699816c5066225fbd86f3a0239477 -F ext/fts5/tool/fts5speed.tcl f9944a9abb9b7685cfbee8101a3dd772ededca66 +F ext/fts5/tool/fts5speed.tcl b0056f91a55b2d1a3684ec05729de92b042e2f85 F ext/fts5/tool/fts5txt2db.tcl 1343745b89ca2a1e975c23f836d0cee410052975 F ext/fts5/tool/loadfts5.tcl 95b03429ee6b138645703c6ca192c3ac96eaf093 F ext/fts5/tool/mkfts5c.tcl d1c2a9ab8e0ec690a52316f33dd9b1d379942f45 @@ -205,14 +206,14 @@ F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f F ext/misc/fuzzer.c 4c84635c71c26cfa7c2e5848cf49fe2d2cfcd767 F ext/misc/ieee754.c f190d0cc5182529acb15babd177781be1ac1718c -F ext/misc/json1.c 7b1155f520d5e8ec1c005d978ac675e8a7f2688a +F ext/misc/json1.c a27cf1eca6583f9b6e18abab5c2a9a82c4540ca9 F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c bcbee3c061b884eccb80e21651daaae8e1e43c63 F ext/misc/regexp.c af92cdaa5058fcec1451e49becc7ba44dba023dc F ext/misc/rot13.c 1ac6f95f99b575907b9b09c81a349114cf9be45a F ext/misc/series.c b8fb7befd85b3a9b4a10e701b30b2b79ca92b6d4 F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52 -F ext/misc/spellfix.c db4cc4b7aa12384e6c19a289a39cd232d355413d +F ext/misc/spellfix.c 525190484b7a9dbc6be646c4842274fff4f27d53 F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512 F ext/misc/vfslog.c fe40fab5c077a40477f7e5eba994309ecac6cc95 F ext/misc/vtshim.c babb0dc2bf116029e3e7c9a618b8a1377045303e @@ -303,37 +304,37 @@ F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a -F src/alter.c 3fe13e97ab38317b092e2f3ae11267b40c9b5cbd -F src/analyze.c 0043d3e501f04297fed2bb50b488bc08d5c39f36 -F src/attach.c 07b3a34a1702dce92a7f1d3888c0c06222b63760 +F src/alter.c e4a5132e6264e002ab87c268108f416df3f1fb10 +F src/analyze.c fbf0e80d83cc893734e872f932f249a056b86e11 +F src/attach.c c16c2648a577fa3def2adfa48c28901376389bc5 F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 2869a76c03eb393ee795416e2387005553df72bc F src/bitvec.c 1a78d450a17c5016710eec900bedfc5729bf9bdf F src/btmutex.c bc87dd3b062cc26edfe79918de2200ccb8d41e73 -F src/btree.c a2a0244ea3f0b3f57f75019c2f229c744ca5d202 +F src/btree.c 0b359bcc2316a57acf12f583253974ad22b4654f F src/btree.h 368ceeb4bd9312dc8df2ffd64b4b7dbcf4db5f8e F src/btreeInt.h c18b7d2a3494695133e4e60ee36061d37f45d9a5 -F src/build.c 33dea2cef04c16a902c55f9d83b1a2065f213979 -F src/callback.c 29ae4faba226c7ebb9aee93016b5ce8a8f071261 +F src/build.c 198eaa849c193f28b802ed135b2483c68ef7a35c +F src/callback.c ed6c2a4a712eb7287ff64e20e3c23265dfb8a7ce F src/complete.c addcd8160b081131005d5bc2d34adf20c1c5c92f F src/ctime.c 60e135af364d777a9ab41c97e5e89cd224da6198 -F src/date.c 997651e3ee6c2818fbf7fcdb7156cef9eb3ece20 +F src/date.c ca17321bc17cca8f40e0843edea4fafff974998e F src/dbstat.c b2ec6793eef97aebb4d171d490a4ffdfa9f2475c -F src/delete.c b6d96db8c7e01ddf1448381c9afd26c7eb4d20d3 -F src/expr.c d10c1cdef5810cdbf73adc9f9b383684230b360a +F src/delete.c eeac28b3d3d88e3541bdf93e91ea7492a7b67842 +F src/expr.c fbf0706199aea23c54efe36b6932d8307c4eb872 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb -F src/fkey.c c66d3e5b35d4d95b5c1e2ee6c12f5df13a7f9ad6 -F src/func.c 42b24923328f65849f52f1659efb2a0907ad78fd +F src/fkey.c 08edad1fce30f761f14b3997e89bad58f9f7f4e0 +F src/func.c 86e55fee35b9577e485f47d9dd5c1d34cd513288 F src/global.c bd5a0af3f30b0c01be6db756c626cd3c33a3d260 F src/hash.c 4263fbc955f26c2e8cdc0cf214bc42435aa4e4f5 F src/hash.h c8f3c31722cf3277d03713909761e152a5b81094 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 -F src/insert.c ca97b03d6eccbe9eb79c4091722660625fb55495 +F src/insert.c 0c96c4f1caa18948484f8f2e229d7af9b8b4e25f F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d -F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e +F src/legacy.c b1b0880fc474abfab89e737b0ecfde0bd7a60902 F src/loadext.c 84996d7d70a605597d79c1f1d7b2012a5fd34f2b -F src/main.c c7d4a1858446db5f611891ddce62572b1fdec040 -F src/malloc.c b67c26c359c13836d370350b3f43d228dff5b360 +F src/main.c 988a73bf14251ab9c4e1ad8743aa651bd713bea2 +F src/malloc.c 55ebb1701ebd39985dbcc497aaecb09192b69682 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 6919bcf12f221868ea066eec27e579fed95ce98b F src/mem2.c f1940d9e91948dd6a908fbb9ce3835c36b5d83c3 @@ -351,28 +352,28 @@ F src/os.c 8fd25588eeba74068d41102d26810e216999b6c8 F src/os.h 3e57a24e2794a94d3cf2342c6d9a884888cd96bf F src/os_common.h abdb9a191a367793268fe553d25bab894e986a0e F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa -F src/os_unix.c 5bb20172d0c9a6afcfa829a88c406970593c848d +F src/os_unix.c 821ed110197175165cf2f50b0930c7ff9a24504c F src/os_win.c ccf29ddded3e41e506b6bd98c1171aa0963b23f2 F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca -F src/pager.c 2916c66aee50f69d9ec56a7619b62d9c6a3bee61 -F src/pager.h 1c2a49143dfba9e69cc8159ef019f472ed8d260b -F src/parse.y 426a91fbbbf7cdde3fd4b8798de7317a8782bec5 +F src/pager.c 67cd2fbab58d0e35fed5f81432856f4f0af9fc6d +F src/pager.h f3eb324a3ff2408b28bab7e81c1c55c13720f865 +F src/parse.y d7bff41d460f2df96fb890f36700e85cb0fc5634 F src/pcache.c 73895411fa6b7bd6f0091212feabbe833b358d23 F src/pcache.h 4d0ccaad264d360981ec5e6a2b596d6e85242545 F src/pcache1.c 72f644dc9e1468c72922eff5904048427b817051 -F src/pragma.c 2ac26ac45eedbed3cc8a9a320ad6d2fc299e69a6 +F src/pragma.c 80ee77226d0008d9188356a6cbbe6010866e1bee F src/pragma.h 64c78a648751b9f4f297276c4eb7507b14b4628c -F src/prepare.c db85f0451ba93ecb3c1e497c279abece5cb5aead -F src/printf.c 98a5cef7fc84577ab8a3098cfa48ecfa5a70b9f8 +F src/prepare.c c12b786713df3e8270c0f85f988c5359d8b4d87c +F src/printf.c 63e6fb12bbe702dd664dc3703776c090383a5a26 F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 F src/resolve.c 9f7ce3a3c087afb7597b7c916c99126ff3f12f0c -F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e -F src/select.c ea6f3b0c279aa37eb3701792d094673a7ad1bf88 +F src/rowset.c 9fe4b3ad7cc00944386bb600233d8f523de07a6e +F src/select.c 57646a44ba9a0bc4aa926ae9c79b8199c246844b F src/shell.c 01e109c27300379b1c35b254cd294cde635f0179 F src/sqlite.h.in 301c01f7c514cbb9ff5110889ea24e1a9715edac F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h dfbe62ffd95b99afe2140d8c35b180d11924072d -F src/sqliteInt.h c44a27bbb24b842e69812f115b74a90909b90f29 +F src/sqliteInt.h f58219ec2bae2458f2ead1c6e187646a3c70594a F src/sqliteLimit.h 216557999cb45f2e3578ed53ebefe228d779cb46 F src/status.c 70912d7be68e9e2dbc4010c93d344af61d4c59ba F src/table.c 51b46b2a62d1b3a959633d593b89bab5e2c9155e @@ -395,7 +396,7 @@ F src/test_config.c a0e8eeb86a7d9393f3bbf7315932c1cccd0e8010 F src/test_demovfs.c 0de72c2c89551629f58486fde5734b7d90758852 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc F src/test_fs.c a61f54247fdb843761d709879c3bcd1989b2050c -F src/test_func.c 0d9c25956152adefee8881c6fadc8354793764d0 +F src/test_func.c 37453d346cfcf118774efd5bf6187f7e6a7e3254 F src/test_hexio.c abfdecb6fa58c354623978efceb088ca18e379cd F src/test_init.c 66b33120ffe9cd853b5a905ec850d51151337b32 F src/test_intarray.c 870124b95ec4c645d4eb84f15efb7133528fb1a5 @@ -425,31 +426,31 @@ F src/test_windirent.c 8f5fada630348558d5745b334702f301da1ffc61 F src/test_windirent.h b12055cab6227f7be10f5c19296f67c60cc5e2a5 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c bbfb74450643cb5372a43ad4f6cffd7e9dfcecb0 -F src/tokenize.c 5606871a377f390af7040ec3c12e0d183512d785 +F src/tokenize.c 214b783d6138e9f9fbb6b225ce9a376db3b03d42 F src/treeview.c dc39ccf04e9331237388b9cb73289c9d87ea050b -F src/trigger.c 72d876b2d0c66604a112362bdae07dae9b104816 -F src/update.c d8d675aa299336ac086ad2039d7e812cd6237db0 +F src/trigger.c e14840ee0c3e549e758ec9bf3e4146e166002280 +F src/update.c 52552d2fe1bb66e7bdaedbd68e7dda1f5705bd44 F src/utf.c 10cc2519e82e3369344d0969ad4b1a333dc86d18 -F src/util.c 72d40df0a52d3f30b462a15f0e094fcbade6dc82 +F src/util.c 49ce0a65306c1c51d61cb5bc214c71cb62452de6 F src/vacuum.c 2ddd5cad2a7b9cef7f9e431b8c7771634c6b1701 -F src/vdbe.c c5244f9750e778a0c5501d6cffa3bfda508f2f2a +F src/vdbe.c 830c927c64e477f86f33d806c585ab1904cf6e4e F src/vdbe.h 19162d5d75d00ee9e634ad630f2a873e1cf76359 F src/vdbeInt.h 2a8796b099a463cf6c32514f8aa138d3b0c36b23 -F src/vdbeapi.c 648ebb95c1a60ced4f0e23ecfb12b43d6aca55ec -F src/vdbeaux.c e41d494842aaa8b6d47d9d40bb58b1aa4bb3ddaa -F src/vdbeblob.c 6ccda01a78b3f9d9a0c75f79e7a1150f3fb0cb39 -F src/vdbemem.c 6c962ee0a77de009dafb6d7f3c7e8bf7b82c47f8 +F src/vdbeapi.c f9bed48c23da6487fbe6648df45f340ccc199c4f +F src/vdbeaux.c 456d0673ddced03335c8657af28eff4d13e54794 +F src/vdbeblob.c c9f2f494b911c6fa34efd9803f0a10807da80f77 +F src/vdbemem.c 36e2779132eaff83c7ed44d76340b8d2caab17cf F src/vdbesort.c ef3c6d1f1a7d44cf67bb2bee59ea3d1fe5bad174 F src/vdbetrace.c f75c5455d8cf389ef86a8bfdfd3177e0e3692484 -F src/vtab.c 320682cca733115b4cbe71320b5c5eeb1074ebde +F src/vtab.c bef51b4f693d82b4b0184457faa8625654534091 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c d21b99fd1458159d0b1ecdccc8ee6ada4fdc4c54 F src/wal.h 2f7c831cf3b071fa548bf2d5cac640846a7ff19c F src/walker.c 0f142b5bd3ed2041fc52d773880748b212e63354 -F src/where.c af9bf5dcec1a0e52726c550924aa91d837166251 +F src/where.c d89fd5cff448ab5c5ca492dd9793b35ffe31ab35 F src/whereInt.h 78b6b4de94db84aecbdc07fe3e38f648eb391e9a -F src/wherecode.c 923f5d04b379b7417bc29f3b86b5eae9d1923d72 -F src/whereexpr.c 197a448b52073aee43eca3a2233fc113369eb2d4 +F src/wherecode.c 791a784bbf8749d560fdb0b990b607bc4f44a38d +F src/whereexpr.c de117970b29471177a6901d60ad83a194671dc03 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -788,7 +789,8 @@ F test/fuzzdata1.db 7ee3227bad0e7ccdeb08a9e6822916777073c664 F test/fuzzdata2.db f03a420d3b822cc82e4f894ca957618fbe9c4973 F test/fuzzdata3.db c6586d3e3cef0fbc18108f9bb649aa77bfc38aba F test/fuzzdata4.db 1882f0055fb63214d8407ddc7aca9b0b1c59af21 -F test/fuzzer1.test d4c52aaf3ef923da293a2653cfab33d02f718a36 +F test/fuzzer1.test 3d4c4b7e547aba5e5511a2991e3e3d07166cfbb8 +F test/fuzzer2.test a85ef814ce071293bce1ad8dffa217cbbaad4c14 F test/fuzzerfault.test 8792cd77fd5bce765b05d0c8e01b9edcf8af8536 F test/genesis.tcl 1e2e2e8e5cc4058549a154ff1892fe5c9de19f98 F test/hexlit.test d7b0a5f41123df1e43985b91b8b2e70f95282d21 @@ -852,7 +854,7 @@ F test/journal3.test ff8af941f9e06161d3db1b46bb9f965ff0e7f307 F test/jrnlmode.test 7864d59cf7f6e552b9b99ba0f38acd167edc10fa F test/jrnlmode2.test 81610545a4e6ed239ea8fa661891893385e23a1d F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa -F test/json101.test f0178422b3a2418f423fd0d3caf3571c8d1b9863 +F test/json101.test ef42283f0b60d8bacbc2243448e7c84988578e52 F test/json102.test bf3fe7a706d30936a76a0f7a0375e1e8e73aff5a F test/json103.test c5f6b85e69de05f6b3195f9f9d5ce9cd179099a0 F test/keyword1.test 37ef6bba5d2ed5b07ecdd6810571de2956599dff @@ -916,9 +918,10 @@ F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test edd0b63e2ee29a256900b0514f6fff27e19e9bb2 F test/misc8.test fc2754d38892f7dac30c22db3616c2764f117d66 F test/misuse.test 3c34719944ba045cc6c188a4852ba04680728912 -F test/mmap1.test 1bfd611b9841eafb44f7d83c0788e146d84a33c9 +F test/mmap1.test 44a5ff1c1bcc7dcf2de50227d1f997e75a8ef1ae F test/mmap2.test 9d6dd9ddb4ad2379f29cc78f38ce1e63ed418022 F test/mmap3.test c92273e16eb8d23c1d55c9815b446bb72ef0512e +F test/mmap4.test 2e2b4e32555b58da15176e6fe750f17c9dcf7f93 F test/mmapfault.test d4c9eff9cd8c2dc14bc43e71e042f175b0a26fe3 F test/multiplex.test efd015ca0b5b4a57dc9535b8feb1273eebeadb60 F test/multiplex2.test 580ca5817c7edbe4cc68fa150609c9473393003a @@ -961,8 +964,8 @@ F test/parser1.test 222b5cbf3e2e659fec1bf7d723488c8b9c94f1d0 F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff -F test/permutations.test 63cb93f915b4056463cef8ad035082e9f1cb524e -F test/pragma.test a44253f911e7d50127d4a08f927f47c861a4c772 +F test/permutations.test 74a48d89ce0d9ca8ad45a8f9f4bf66fa8bee0e34 +F test/pragma.test 507ac7ef2ea5682241ea0ef041799ca70bb5e0bf F test/pragma2.test a9400a7289605280576098b97f5cde3f204075c0 F test/pragma3.test 6f849ccffeee7e496d2f2b5e74152306c0b8757c F test/printf.test b3ff34e73d59124140eaf89f7672e21bc2ca5fcc @@ -993,11 +996,11 @@ F test/rtree.test 0c8d9dd458d6824e59683c19ab2ffa9ef946f798 F test/run-wordcount.sh 891e89c4c2d16e629cd45951d4ed899ad12afc09 F test/savepoint.test c671fdbd34cd3bfe1518a777526ada595180cf8d F test/savepoint2.test 9b8543940572a2f01a18298c3135ad0c9f4f67d7 -F test/savepoint3.test e328085853b14898d78ceea00dfe7db18bb6a9ec F test/savepoint4.test c8f8159ade6d2acd9128be61e1230f1c1edc6cc0 F test/savepoint5.test 0735db177e0ebbaedc39812c8d065075d563c4fd F test/savepoint6.test f41279c5e137139fa5c21485773332c7adb98cd7 F test/savepoint7.test db3db281486c925095f305aad09fe806e5188ff3 +F test/savepointfault.test f044eac64b59f09746c7020ee261734de82bf9b2 w test/savepoint3.test F test/scanstatus.test 5253c219e331318a437f436268e0e82345700285 F test/schema.test 8f7999be894260f151adf15c2c7540f1c6d6a481 F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5 @@ -1054,9 +1057,9 @@ F test/snapshot.test efc6b4edc5d571161835f9dd8552e181ad1f0ac2 F test/snapshot_fault.test 25973aeb1b86a280800e0bcf1eb5ce70e9ef57ab F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 843cd84db9891b2d01b9ab64cef3e9020f98d087 -F test/sort.test 3f492e5b7be1d3f756728d2ff6edf4f6091e84cb -F test/sort2.test 37afbc03f5559f2eb0f18940b55d38dfbb5172ac -F test/sort3.test 6178ade30810ac9166fcdf14b7065e49c0f534e2 +F test/sort.test c2adc635c2564241fefec0b3a68391ef6868fd3b +F test/sort2.test cc23b7c19d684657559e8a55b02f7fcee03851d0 +F test/sort3.test 1d831e95076b92985754a122e92ebc23bdf6f712 F test/sort4.test 5c34d9623a4ae5921d956dfa2b70e77ed0fc6e5c F test/sort5.test a448240a42b49239edc00f85d6d7ac7a1b261e1f F test/sortfault.test d4ccf606a0c77498e2beb542764fd9394acb4d66 @@ -1085,7 +1088,7 @@ F test/substr.test 18f57c4ca8a598805c4d64e304c418734d843c1a F test/subtype1.test 7fe09496352f97053af1437150751be2d0a0cae8 F test/superlock.test 1cde669f68d2dd37d6c9bd35eee1d95491ae3fc2 F test/symlink.test c9ebe7330d228249e447038276bfc8a7b22f4849 -F test/sync.test a34cd43e98b7fb84eabbf38f7ed8f7349b3f3d85 +F test/sync.test 2f607e1821aa3af3c5c53b58835c05e511c95899 F test/syscall.test f59ba4e25f7ba4a4c031026cc2ef8b6e4b4c639c F test/sysfault.test c9f2b0d8d677558f74de750c75e12a5454719d04 F test/tabfunc01.test cc33684f9480fcf1fd5ce287ac28d22971cad1cc @@ -1096,7 +1099,7 @@ F test/tclsqlite.test 7179b4e0bf236ddf0bfa6bfaefa76fbe0a23c28a F test/tempdb.test 19d0f66e2e3eeffd68661a11c83ba5e6ace9128c F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 F test/temptrigger.test 8ec228b0db5d7ebc4ee9b458fc28cb9e7873f5e1 -F test/tester.tcl a4b1c8e78ad88426dc0b2375e0b5348d2b841c88 +F test/tester.tcl d8ef0a5ab95b16fefd7123f38647114c34b0c9b7 F test/thread001.test 9f22fd3525a307ff42a326b6bc7b0465be1745a5 F test/thread002.test e630504f8a06c00bf8bbe68528774dd96aeb2e58 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7 @@ -1314,9 +1317,9 @@ F test/vtabI.test 751b07636700dbdea328e4265b6077ccd6811a3f F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test ea8778d5b0df200adef2ca7c00c3c37d4375f772 -F test/wal.test 65bfc68f3f09dcbc62cee9f794e560428d96cec7 +F test/wal.test 0148c8b3421a25fdb4d9c160e84a681d0646371b F test/wal2.test 1f841d2048080d32f552942e333fd99ce541dada -F test/wal3.test b1d425f68a1f61d12563f0fa1ee6fca7d5afabf4 +F test/wal3.test 5dd734147f1f8f958c5261a1f2775d346d7013ce F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c F test/wal5.test 88b5d9a6a3d1532497ee9f4296f010d66f07e33c F test/wal6.test 4421cd5a2fa99d29cc91ef12fb23bed171ed3a4c @@ -1328,7 +1331,7 @@ F test/wal_common.tcl a98f17fba96206122eff624db0ab13ec377be4fe F test/walbak.test b9f68e39646375c2b877be906babcc15d38b4877 F test/walbig.test f437473a16cfb314867c6b5d1dbcd519e73e3434 F test/walblock.test be48f3a75eff0b4456209f26b3ce186c2015497d -F test/walcksum.test 9afeb96240296c08c72fc524d199c912cfe34daa +F test/walcksum.test bb234a1bb42248b3515d992b719708015c384278 F test/walcrash.test 21038858cc552077b0522f50b0fa87e38139306a F test/walcrash2.test a0edab4e5390f03b99a790de89aad15d6ec70b36 F test/walcrash3.test e426aa58122d20f2b9fbe9a507f9eb8cab85b8af @@ -1338,9 +1341,10 @@ F test/walmode.test 4022fe03ae6e830583672caa101f046438a0473c F test/walnoshm.test 84ca10c544632a756467336b7c3b864d493ee496 F test/waloverwrite.test a0d2ae0783187374c1e6a9571e0916152977cb81 F test/walpersist.test 8c6b7e3ec1ba91b5e4dc4e0921d6d3f87cd356a6 +F test/walprotocol.test 059cb75484a1ecf6357a2c1b3324b8156749221e F test/walro.test 34422d1d95aaff0388f0791ec20edb34e2a3ed57 F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417 -F test/walslow.test e7be6d9888f83aa5d3d3c7c08aa9b5c28b93609a +F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e F test/where.test 9902a3d84e9bc80357a2c54ed0e76c0d6d04a867 F test/where2.test af78c55589cbc82d793449493adba0dc3d659f23 @@ -1350,7 +1354,6 @@ F test/where5.test fdf66f96d29a064b63eb543e28da4dfdccd81ad2 F test/where6.test 5da5a98cec820d488e82708301b96cb8c18a258b F test/where7.test f520bcec2c3d12dc4615623b06b2aec7c2d67e94 F test/where8.test 98eedca0d375fb400b8377269c4b4686582dfb45 -F test/where8m.test da346596e19d54f0aba35ebade032a7c47d79739 F test/where9.test 729c3ba9b47e8f9f1aab96bae7dad2a524f1d1a2 F test/whereA.test 4d253178d135ec46d1671e440cd8f2b916aa6e6b F test/whereB.test 0def95db3bdec220a731c7e4bec5930327c1d8c5 @@ -1363,6 +1366,7 @@ F test/whereH.test e4b07f7a3c2f5d31195cd33710054c78667573b2 F test/whereI.test 1d89199697919d4930be05a71e7fe620f114e622 F test/whereJ.test 55a3221706a7ab706293f17cc8f96da563bf0767 F test/whereK.test f8e3cf26a8513ecc7f514f54df9f0572c046c42b +F test/wherefault.test 1374c3aa198388925246475f84ad4cd5f9528864 w test/where8m.test F test/wherelimit.test 5e9fd41e79bb2b2d588ed999d641d9c965619b31 F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c F test/win32heap.test ea19770974795cff26e11575e12d422dbd16893c @@ -1384,7 +1388,7 @@ F test/zerodamage.test cf6748bad89553cc1632be51a6f54e487e4039ac F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5 F tool/GetTclKit.bat 629d87562e0487c386db630033931d12d62e6372 F tool/addopcodes.tcl 4ca9c3ef196f08da30add5d07ce0c9458dc8c633 -F tool/build-all-msvc.bat 77f85f4268c2711d637c629610d0cf3df5338638 x +F tool/build-all-msvc.bat 31866578036cd1d962628059b0760d407c3ce4d8 x F tool/build-shell.sh 950f47c6174f1eea171319438b93ba67ff5bf367 F tool/cg_anno.tcl 692ce4b8693d59e3a3de77ca97f4139ecfa641b0 x F tool/checkSpacing.c 810e51703529a204fc4e1eb060e9ab663e3c06d2 @@ -1443,7 +1447,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 36cb3d6e274a06a78693ca506feaad0b14bde11d d8b7b1996eefae7768bfcb82d4ff22c69392aa63 -R a6848fc70bf84dd1da28684cee4a5e04 +P f3f9200115caf4b356f90ec97c351d1afbcb9bf6 b082538da774ac77f284fd7e22f9b1b9c2abc343 +R caf89ffed359feebe1a7893788b9d4fd U drh -Z 8c0e589f1ff041b7f5f063df66382fc8 +Z 50b2b3cb434b1b40a7be12f42687aabd diff --git a/manifest.uuid b/manifest.uuid index 1979b8ac77..70420107af 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f3f9200115caf4b356f90ec97c351d1afbcb9bf6 \ No newline at end of file +a533608cb0de3cbc1e28a794aab99864b8c249f4 \ No newline at end of file diff --git a/src/alter.c b/src/alter.c index c8102c55d2..3419059fa2 100644 --- a/src/alter.c +++ b/src/alter.c @@ -695,7 +695,7 @@ void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){ rc = sqlite3ValueFromExpr(db, pDflt, SQLITE_UTF8, SQLITE_AFF_BLOB, &pVal); assert( rc==SQLITE_OK || rc==SQLITE_NOMEM ); if( rc!=SQLITE_OK ){ - db->mallocFailed = 1; + assert( db->mallocFailed = 1 ); return; } if( !pVal ){ @@ -803,7 +803,7 @@ void sqlite3AlterBeginAddColumn(Parse *pParse, SrcList *pSrc){ pNew->aCol = (Column*)sqlite3DbMallocZero(db, sizeof(Column)*nAlloc); pNew->zName = sqlite3MPrintf(db, "sqlite_altertab_%s", pTab->zName); if( !pNew->aCol || !pNew->zName ){ - db->mallocFailed = 1; + assert( db->mallocFailed ); goto exit_begin_add_column; } memcpy(pNew->aCol, pTab->aCol, sizeof(Column)*pNew->nCol); diff --git a/src/analyze.c b/src/analyze.c index 4d777fa9f5..1e026a7530 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -313,7 +313,7 @@ static void sampleClear(sqlite3 *db, Stat4Sample *p){ static void sampleSetRowid(sqlite3 *db, Stat4Sample *p, int n, const u8 *pData){ assert( db!=0 ); if( p->nRowid ) sqlite3DbFree(db, p->u.aRowid); - p->u.aRowid = sqlite3DbMallocRaw(db, n); + p->u.aRowid = sqlite3DbMallocRawNN(db, n); if( p->u.aRowid ){ p->nRowid = n; memcpy(p->u.aRowid, pData, n); @@ -1115,7 +1115,7 @@ static void analyzeOneTable( if( nColTest>0 ){ int endDistinctTest = sqlite3VdbeMakeLabel(v); int *aGotoChng; /* Array of jump instruction addresses */ - aGotoChng = sqlite3DbMallocRaw(db, sizeof(int)*nColTest); + aGotoChng = sqlite3DbMallocRawNN(db, sizeof(int)*nColTest); if( aGotoChng==0 ) continue; /* @@ -1523,7 +1523,7 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ ** the old data with the new instead of allocating a new array. */ if( pIndex->aiRowEst==0 ){ pIndex->aiRowEst = (tRowcnt*)sqlite3MallocZero(sizeof(tRowcnt) * nCol); - if( pIndex->aiRowEst==0 ) pInfo->db->mallocFailed = 1; + if( pIndex->aiRowEst==0 ) sqlite3OomFault(pInfo->db); } aiRowEst = pIndex->aiRowEst; #endif @@ -1670,7 +1670,7 @@ static int loadStatTbl( Index *pPrevIdx = 0; /* Previous index in the loop */ IndexSample *pSample; /* A slot in pIdx->aSample[] */ - assert( db->lookaside.bEnabled==0 ); + assert( db->lookaside.bDisable ); zSql = sqlite3MPrintf(db, zSql1, zDb); if( !zSql ){ return SQLITE_NOMEM; @@ -1784,7 +1784,7 @@ static int loadStatTbl( static int loadStat4(sqlite3 *db, const char *zDb){ int rc = SQLITE_OK; /* Result codes from subroutines */ - assert( db->lookaside.bEnabled==0 ); + assert( db->lookaside.bDisable ); if( sqlite3FindTable(db, "sqlite_stat4", zDb) ){ rc = loadStatTbl(db, 0, "SELECT idx,count(*) FROM %Q.sqlite_stat4 GROUP BY idx", @@ -1866,10 +1866,9 @@ int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ /* Load the statistics from the sqlite_stat4 table. */ #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 if( rc==SQLITE_OK && OptimizationEnabled(db, SQLITE_Stat34) ){ - int lookasideEnabled = db->lookaside.bEnabled; - db->lookaside.bEnabled = 0; + db->lookaside.bDisable++; rc = loadStat4(db, sInfo.zDatabase); - db->lookaside.bEnabled = lookasideEnabled; + db->lookaside.bDisable--; } for(i=sqliteHashFirst(&db->aDb[iDb].pSchema->idxHash);i;i=sqliteHashNext(i)){ Index *pIdx = sqliteHashData(i); @@ -1879,7 +1878,7 @@ int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ #endif if( rc==SQLITE_NOMEM ){ - db->mallocFailed = 1; + sqlite3OomFault(db); } return rc; } diff --git a/src/attach.c b/src/attach.c index fd5cc7c76c..2288ac9b62 100644 --- a/src/attach.c +++ b/src/attach.c @@ -109,7 +109,7 @@ static void attachFunc( ** hash tables. */ if( db->aDb==db->aDbStatic ){ - aNew = sqlite3DbMallocRaw(db, sizeof(db->aDb[0])*3 ); + aNew = sqlite3DbMallocRawNN(db, sizeof(db->aDb[0])*3 ); if( aNew==0 ) return; memcpy(aNew, db->aDb, sizeof(db->aDb[0])*2); }else{ @@ -127,7 +127,7 @@ static void attachFunc( flags = db->openFlags; rc = sqlite3ParseUri(db->pVfs->zName, zFile, &flags, &pVfs, &zPath, &zErr); if( rc!=SQLITE_OK ){ - if( rc==SQLITE_NOMEM ) db->mallocFailed = 1; + if( rc==SQLITE_NOMEM ) sqlite3OomFault(db); sqlite3_result_error(context, zErr, -1); sqlite3_free(zErr); return; @@ -156,7 +156,8 @@ static void attachFunc( sqlite3BtreeSecureDelete(aNew->pBt, sqlite3BtreeSecureDelete(db->aDb[0].pBt,-1) ); #ifndef SQLITE_OMIT_PAGER_PRAGMAS - sqlite3BtreeSetPagerFlags(aNew->pBt, 3 | (db->flags & PAGER_FLAGS_MASK)); + sqlite3BtreeSetPagerFlags(aNew->pBt, + PAGER_SYNCHRONOUS_FULL | (db->flags & PAGER_FLAGS_MASK)); #endif sqlite3BtreeLeave(aNew->pBt); } @@ -229,7 +230,7 @@ static void attachFunc( sqlite3ResetAllSchemasOfConnection(db); db->nDb = iDb; if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ){ - db->mallocFailed = 1; + sqlite3OomFault(db); sqlite3DbFree(db, zErrDyn); zErrDyn = sqlite3MPrintf(db, "out of memory"); }else if( zErrDyn==0 ){ diff --git a/src/btree.c b/src/btree.c index d5acaf3669..b99820ddbc 100644 --- a/src/btree.c +++ b/src/btree.c @@ -2338,7 +2338,6 @@ int sqlite3BtreeOpen( pBt->mutex = sqlite3MutexAlloc(SQLITE_MUTEX_FAST); if( pBt->mutex==0 ){ rc = SQLITE_NOMEM; - db->mallocFailed = 0; goto btree_open_out; } } diff --git a/src/build.c b/src/build.c index 3ca4c99947..476fc28c41 100644 --- a/src/build.c +++ b/src/build.c @@ -78,7 +78,7 @@ void sqlite3TableLock( p->zName = zName; }else{ pToplevel->nTableLock = 0; - pToplevel->db->mallocFailed = 1; + sqlite3OomFault(pToplevel->db); } } @@ -926,7 +926,7 @@ void sqlite3StartTable( pTable = sqlite3DbMallocZero(db, sizeof(Table)); if( pTable==0 ){ - db->mallocFailed = 1; + assert( db->mallocFailed ); pParse->rc = SQLITE_NOMEM; pParse->nErr++; goto begin_table_error; @@ -1555,7 +1555,7 @@ static char *createTableStmt(sqlite3 *db, Table *p){ n += 35 + 6*p->nCol; zStmt = sqlite3DbMallocRaw(0, n); if( zStmt==0 ){ - db->mallocFailed = 1; + sqlite3OomFault(db); return 0; } sqlite3_snprintf(n, zStmt, "CREATE TABLE "); @@ -2038,7 +2038,7 @@ void sqlite3EndTable( pOld = sqlite3HashInsert(&pSchema->tblHash, p->zName, p); if( pOld ){ assert( p==pOld ); /* Malloc must have failed inside HashInsert() */ - db->mallocFailed = 1; + sqlite3OomFault(db); return; } pParse->pNewTable = 0; @@ -2142,7 +2142,6 @@ int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ int n; /* Temporarily holds the number of cursors assigned */ sqlite3 *db = pParse->db; /* Database connection for malloc errors */ sqlite3_xauth xAuth; /* Saved xAuth pointer */ - u8 bEnabledLA; /* Saved db->lookaside.bEnabled state */ assert( pTable ); @@ -2188,18 +2187,18 @@ int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ ** statement that defines the view. */ assert( pTable->pSelect ); - bEnabledLA = db->lookaside.bEnabled; if( pTable->pCheck ){ - db->lookaside.bEnabled = 0; + db->lookaside.bDisable++; sqlite3ColumnsFromExprList(pParse, pTable->pCheck, &pTable->nCol, &pTable->aCol); + db->lookaside.bDisable--; }else{ pSel = sqlite3SelectDup(db, pTable->pSelect, 0); if( pSel ){ n = pParse->nTab; sqlite3SrcListAssignCursors(pParse, pSel->pSrc); pTable->nCol = -1; - db->lookaside.bEnabled = 0; + db->lookaside.bDisable++; #ifndef SQLITE_OMIT_AUTHORIZATION xAuth = db->xAuth; db->xAuth = 0; @@ -2208,6 +2207,7 @@ int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ #else pSelTab = sqlite3ResultSetOfSelect(pParse, pSel); #endif + db->lookaside.bDisable--; pParse->nTab = n; if( pSelTab ){ assert( pTable->aCol==0 ); @@ -2226,7 +2226,6 @@ int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ nErr++; } } - db->lookaside.bEnabled = bEnabledLA; pTable->pSchema->schemaFlags |= DB_UnresetViews; #endif /* SQLITE_OMIT_VIEW */ return nErr; @@ -2692,7 +2691,7 @@ void sqlite3CreateForeignKey( pFKey->zTo, (void *)pFKey ); if( pNextTo==pFKey ){ - db->mallocFailed = 1; + sqlite3OomFault(db); goto fk_end; } if( pNextTo ){ @@ -3274,7 +3273,7 @@ Index *sqlite3CreateIndex( pIndex->zName, pIndex); if( p ){ assert( p==pIndex ); /* Malloc must have failed */ - db->mallocFailed = 1; + sqlite3OomFault(db); goto exit_create_index; } db->flags |= SQLITE_InternChanges; @@ -3703,8 +3702,9 @@ SrcList *sqlite3SrcListAppend( ){ struct SrcList_item *pItem; assert( pDatabase==0 || pTable!=0 ); /* Cannot have C without B */ + assert( db!=0 ); if( pList==0 ){ - pList = sqlite3DbMallocRaw(db, sizeof(SrcList) ); + pList = sqlite3DbMallocRawNN(db, sizeof(SrcList) ); if( pList==0 ) return 0; pList->nAlloc = 1; pList->nSrc = 0; @@ -3992,7 +3992,7 @@ int sqlite3OpenTempDatabase(Parse *pParse){ db->aDb[1].pBt = pBt; assert( db->aDb[1].pSchema ); if( SQLITE_NOMEM==sqlite3BtreeSetPageSize(pBt, db->nextPagesize, -1, 0) ){ - db->mallocFailed = 1; + sqlite3OomFault(db); return 1; } } @@ -4367,10 +4367,9 @@ With *sqlite3WithAdd( }else{ pNew = sqlite3DbMallocZero(db, sizeof(*pWith)); } - assert( zName!=0 || pNew==0 ); - assert( db->mallocFailed==0 || pNew==0 ); + assert( (pNew!=0 && zName!=0) || db->mallocFailed ); - if( pNew==0 ){ + if( db->mallocFailed ){ sqlite3ExprListDelete(db, pArglist); sqlite3SelectDelete(db, pQuery); sqlite3DbFree(db, zName); diff --git a/src/callback.c b/src/callback.c index 2a9f5b802a..2b955fdcdb 100644 --- a/src/callback.c +++ b/src/callback.c @@ -177,7 +177,7 @@ static CollSeq *findCollSeqEntry( */ assert( pDel==0 || pDel==pColl ); if( pDel!=0 ){ - db->mallocFailed = 1; + sqlite3OomFault(db); sqlite3DbFree(db, pDel); pColl = 0; } @@ -465,7 +465,7 @@ Schema *sqlite3SchemaGet(sqlite3 *db, Btree *pBt){ p = (Schema *)sqlite3DbMallocZero(0, sizeof(Schema)); } if( !p ){ - db->mallocFailed = 1; + sqlite3OomFault(db); }else if ( 0==p->file_format ){ sqlite3HashInit(&p->tblHash); sqlite3HashInit(&p->idxHash); diff --git a/src/date.c b/src/date.c index d78e83cf57..0a1d0e0798 100644 --- a/src/date.c +++ b/src/date.c @@ -967,7 +967,7 @@ static void strftimeFunc( sqlite3_result_error_toobig(context); return; }else{ - z = sqlite3DbMallocRaw(db, (int)n); + z = sqlite3DbMallocRawNN(db, (int)n); if( z==0 ){ sqlite3_result_error_nomem(context); return; diff --git a/src/delete.c b/src/delete.c index feda71bf8d..6eae462ba1 100644 --- a/src/delete.c +++ b/src/delete.c @@ -442,7 +442,7 @@ void sqlite3DeleteFrom( ** one, so just keep it in its register(s) and fall through to the ** delete code. */ nKey = nPk; /* OP_Found will use an unpacked key */ - aToOpen = sqlite3DbMallocRaw(db, nIdx+2); + aToOpen = sqlite3DbMallocRawNN(db, nIdx+2); if( aToOpen==0 ){ sqlite3WhereEnd(pWInfo); goto delete_from_cleanup; diff --git a/src/expr.c b/src/expr.c index c91db2834f..8d96ba10cd 100644 --- a/src/expr.c +++ b/src/expr.c @@ -453,6 +453,7 @@ Expr *sqlite3ExprAlloc( int nExtra = 0; int iValue = 0; + assert( db!=0 ); if( pToken ){ if( op!=TK_INTEGER || pToken->z==0 || sqlite3GetInt32(pToken->z, &iValue)==0 ){ @@ -460,7 +461,7 @@ Expr *sqlite3ExprAlloc( assert( iValue>=0 ); } } - pNew = sqlite3DbMallocRaw(db, sizeof(Expr)+nExtra); + pNew = sqlite3DbMallocRawNN(db, sizeof(Expr)+nExtra); if( pNew ){ memset(pNew, 0, sizeof(Expr)); pNew->op = (u8)op; @@ -699,7 +700,10 @@ void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr){ if( x>pParse->nzVar ){ char **a; a = sqlite3DbRealloc(db, pParse->azVar, x*sizeof(a[0])); - if( a==0 ) return; /* Error reported through db->mallocFailed */ + if( a==0 ){ + assert( db->mallocFailed ); /* Error reported through mallocFailed */ + return; + } pParse->azVar = a; memset(&a[pParse->nzVar], 0, (x-pParse->nzVar)*sizeof(a[0])); pParse->nzVar = x; @@ -854,6 +858,7 @@ static int dupedExprSize(Expr *p, int flags){ static Expr *exprDup(sqlite3 *db, Expr *p, int flags, u8 **pzBuffer){ Expr *pNew = 0; /* Value to return */ assert( flags==0 || flags==EXPRDUP_REDUCE ); + assert( db!=0 ); if( p ){ const int isReduced = (flags&EXPRDUP_REDUCE); u8 *zAlloc; @@ -866,7 +871,7 @@ static Expr *exprDup(sqlite3 *db, Expr *p, int flags, u8 **pzBuffer){ zAlloc = *pzBuffer; staticFlag = EP_Static; }else{ - zAlloc = sqlite3DbMallocRaw(db, dupedExprSize(p, flags)); + zAlloc = sqlite3DbMallocRawNN(db, dupedExprSize(p, flags)); } pNew = (Expr *)zAlloc; @@ -989,12 +994,13 @@ ExprList *sqlite3ExprListDup(sqlite3 *db, ExprList *p, int flags){ ExprList *pNew; struct ExprList_item *pItem, *pOldItem; int i; + assert( db!=0 ); if( p==0 ) return 0; - pNew = sqlite3DbMallocRaw(db, sizeof(*pNew) ); + pNew = sqlite3DbMallocRawNN(db, sizeof(*pNew) ); if( pNew==0 ) return 0; pNew->nExpr = i = p->nExpr; if( (flags & EXPRDUP_REDUCE)==0 ) for(i=1; inExpr; i+=i){} - pNew->a = pItem = sqlite3DbMallocRaw(db, i*sizeof(p->a[0]) ); + pNew->a = pItem = sqlite3DbMallocRawNN(db, i*sizeof(p->a[0]) ); if( pItem==0 ){ sqlite3DbFree(db, pNew); return 0; @@ -1025,9 +1031,10 @@ SrcList *sqlite3SrcListDup(sqlite3 *db, SrcList *p, int flags){ SrcList *pNew; int i; int nByte; + assert( db!=0 ); if( p==0 ) return 0; nByte = sizeof(*p) + (p->nSrc>0 ? sizeof(p->a[0]) * (p->nSrc-1) : 0); - pNew = sqlite3DbMallocRaw(db, nByte ); + pNew = sqlite3DbMallocRawNN(db, nByte ); if( pNew==0 ) return 0; pNew->nSrc = pNew->nAlloc = p->nSrc; for(i=0; inSrc; i++){ @@ -1064,11 +1071,12 @@ SrcList *sqlite3SrcListDup(sqlite3 *db, SrcList *p, int flags){ IdList *sqlite3IdListDup(sqlite3 *db, IdList *p){ IdList *pNew; int i; + assert( db!=0 ); if( p==0 ) return 0; - pNew = sqlite3DbMallocRaw(db, sizeof(*pNew) ); + pNew = sqlite3DbMallocRawNN(db, sizeof(*pNew) ); if( pNew==0 ) return 0; pNew->nId = p->nId; - pNew->a = sqlite3DbMallocRaw(db, p->nId*sizeof(p->a[0]) ); + pNew->a = sqlite3DbMallocRawNN(db, p->nId*sizeof(p->a[0]) ); if( pNew->a==0 ){ sqlite3DbFree(db, pNew); return 0; @@ -1086,8 +1094,9 @@ IdList *sqlite3IdListDup(sqlite3 *db, IdList *p){ } Select *sqlite3SelectDup(sqlite3 *db, Select *p, int flags){ Select *pNew, *pPrior; + assert( db!=0 ); if( p==0 ) return 0; - pNew = sqlite3DbMallocRaw(db, sizeof(*p) ); + pNew = sqlite3DbMallocRawNN(db, sizeof(*p) ); if( pNew==0 ) return 0; pNew->pEList = sqlite3ExprListDup(db, p->pEList, flags); pNew->pSrc = sqlite3SrcListDup(db, p->pSrc, flags); @@ -1133,13 +1142,14 @@ ExprList *sqlite3ExprListAppend( Expr *pExpr /* Expression to be appended. Might be NULL */ ){ sqlite3 *db = pParse->db; + assert( db!=0 ); if( pList==0 ){ - pList = sqlite3DbMallocRaw(db, sizeof(ExprList) ); + pList = sqlite3DbMallocRawNN(db, sizeof(ExprList) ); if( pList==0 ){ goto no_mem; } pList->nExpr = 0; - pList->a = sqlite3DbMallocRaw(db, sizeof(pList->a[0])); + pList->a = sqlite3DbMallocRawNN(db, sizeof(pList->a[0])); if( pList->a==0 ) goto no_mem; }else if( (pList->nExpr & (pList->nExpr-1))==0 ){ struct ExprList_item *a; diff --git a/src/fkey.c b/src/fkey.c index 7eb188924d..38fd4f756b 100644 --- a/src/fkey.c +++ b/src/fkey.c @@ -219,7 +219,7 @@ int sqlite3FkLocateIndex( } }else if( paiCol ){ assert( nCol>1 ); - aiCol = (int *)sqlite3DbMallocRaw(pParse->db, nCol*sizeof(int)); + aiCol = (int *)sqlite3DbMallocRawNN(pParse->db, nCol*sizeof(int)); if( !aiCol ) return 1; *paiCol = aiCol; } @@ -1165,7 +1165,6 @@ static Trigger *fkActionTrigger( pTrigger = pFKey->apTrigger[iAction]; if( action!=OE_None && !pTrigger ){ - u8 enableLookaside; /* Copy of db->lookaside.bEnabled */ char const *zFrom; /* Name of child table */ int nFrom; /* Length in bytes of zFrom */ Index *pIdx = 0; /* Parent key index for this FK */ @@ -1274,8 +1273,7 @@ static Trigger *fkActionTrigger( } /* Disable lookaside memory allocation */ - enableLookaside = db->lookaside.bEnabled; - db->lookaside.bEnabled = 0; + db->lookaside.bDisable++; pTrigger = (Trigger *)sqlite3DbMallocZero(db, sizeof(Trigger) + /* struct Trigger */ @@ -1297,7 +1295,7 @@ static Trigger *fkActionTrigger( } /* Re-enable the lookaside buffer, if it was disabled earlier. */ - db->lookaside.bEnabled = enableLookaside; + db->lookaside.bDisable--; sqlite3ExprDelete(db, pWhere); sqlite3ExprDelete(db, pWhen); diff --git a/src/func.c b/src/func.c index 982a8c2caa..d615cf90b1 100644 --- a/src/func.c +++ b/src/func.c @@ -1615,7 +1615,7 @@ void sqlite3RegisterBuiltinFunctions(sqlite3 *db){ int rc = sqlite3_overload_function(db, "MATCH", 2); assert( rc==SQLITE_NOMEM || rc==SQLITE_OK ); if( rc==SQLITE_NOMEM ){ - db->mallocFailed = 1; + sqlite3OomFault(db); } } diff --git a/src/insert.c b/src/insert.c index d89ab4e0d6..f146f4bf36 100644 --- a/src/insert.c +++ b/src/insert.c @@ -83,7 +83,7 @@ const char *sqlite3IndexAffinityStr(sqlite3 *db, Index *pIdx){ Table *pTab = pIdx->pTable; pIdx->zColAff = (char *)sqlite3DbMallocRaw(0, pIdx->nColumn+1); if( !pIdx->zColAff ){ - db->mallocFailed = 1; + sqlite3OomFault(db); return 0; } for(n=0; nnColumn; n++){ @@ -134,7 +134,7 @@ void sqlite3TableAffinity(Vdbe *v, Table *pTab, int iReg){ sqlite3 *db = sqlite3VdbeDb(v); zColAff = (char *)sqlite3DbMallocRaw(0, pTab->nCol+1); if( !zColAff ){ - db->mallocFailed = 1; + sqlite3OomFault(db); return; } @@ -230,7 +230,7 @@ static int autoIncBegin( pInfo = pToplevel->pAinc; while( pInfo && pInfo->pTab!=pTab ){ pInfo = pInfo->pNext; } if( pInfo==0 ){ - pInfo = sqlite3DbMallocRaw(pParse->db, sizeof(*pInfo)); + pInfo = sqlite3DbMallocRawNN(pParse->db, sizeof(*pInfo)); if( pInfo==0 ) return 0; pInfo->pNext = pToplevel->pAinc; pToplevel->pAinc = pInfo; @@ -254,7 +254,6 @@ void sqlite3AutoincrementBegin(Parse *pParse){ sqlite3 *db = pParse->db; /* The database connection */ Db *pDb; /* Database only autoinc table */ int memId; /* Register holding max rowid */ - int addr; /* A VDBE address */ Vdbe *v = pParse->pVdbe; /* VDBE under construction */ /* This routine is never called during trigger-generation. It is @@ -264,33 +263,46 @@ void sqlite3AutoincrementBegin(Parse *pParse){ assert( v ); /* We failed long ago if this is not so */ for(p = pParse->pAinc; p; p = p->pNext){ + static const int iLn = VDBE_OFFSET_LINENO(2); + static const VdbeOpList autoInc[] = { + /* 0 */ {OP_Null, 0, 0, 0}, + /* 1 */ {OP_Rewind, 0, 9, 0}, + /* 2 */ {OP_Column, 0, 0, 0}, + /* 3 */ {OP_Ne, 0, 7, 0}, + /* 4 */ {OP_Rowid, 0, 0, 0}, + /* 5 */ {OP_Column, 0, 1, 0}, + /* 6 */ {OP_Goto, 0, 9, 0}, + /* 7 */ {OP_Next, 0, 2, 0}, + /* 8 */ {OP_Integer, 0, 0, 0}, + /* 9 */ {OP_Close, 0, 0, 0} + }; + VdbeOp *aOp; pDb = &db->aDb[p->iDb]; memId = p->regCtr; assert( sqlite3SchemaMutexHeld(db, 0, pDb->pSchema) ); sqlite3OpenTable(pParse, 0, p->iDb, pDb->pSchema->pSeqTab, OP_OpenRead); - sqlite3VdbeAddOp3(v, OP_Null, 0, memId, memId+1); - addr = sqlite3VdbeCurrentAddr(v); sqlite3VdbeLoadString(v, memId-1, p->pTab->zName); - sqlite3VdbeAddOp2(v, OP_Rewind, 0, addr+9); VdbeCoverage(v); - sqlite3VdbeAddOp3(v, OP_Column, 0, 0, memId); - sqlite3VdbeAddOp3(v, OP_Ne, memId-1, addr+7, memId); VdbeCoverage(v); - sqlite3VdbeChangeP5(v, SQLITE_JUMPIFNULL); - sqlite3VdbeAddOp2(v, OP_Rowid, 0, memId+1); - sqlite3VdbeAddOp3(v, OP_Column, 0, 1, memId); - sqlite3VdbeGoto(v, addr+9); - sqlite3VdbeAddOp2(v, OP_Next, 0, addr+2); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Integer, 0, memId); - sqlite3VdbeAddOp0(v, OP_Close); + aOp = sqlite3VdbeAddOpList(v, ArraySize(autoInc), autoInc, iLn); + if( aOp==0 ) break; + aOp[0].p2 = memId; + aOp[0].p3 = memId+1; + aOp[2].p3 = memId; + aOp[3].p1 = memId-1; + aOp[3].p3 = memId; + aOp[3].p5 = SQLITE_JUMPIFNULL; + aOp[4].p2 = memId+1; + aOp[5].p3 = memId; + aOp[8].p2 = memId; } } /* ** Update the maximum rowid for an autoincrement calculation. ** -** This routine should be called when the top of the stack holds a +** This routine should be called when the regRowid register holds a ** new rowid that is about to be inserted. If that new rowid is ** larger than the maximum rowid in the memId memory cell, then the -** memory cell is updated. The stack is unchanged. +** memory cell is updated. */ static void autoIncStep(Parse *pParse, int memId, int regRowid){ if( memId>0 ){ @@ -305,31 +317,44 @@ static void autoIncStep(Parse *pParse, int memId, int regRowid){ ** table (either directly or through triggers) needs to call this ** routine just before the "exit" code. */ -void sqlite3AutoincrementEnd(Parse *pParse){ +static SQLITE_NOINLINE void autoIncrementEnd(Parse *pParse){ AutoincInfo *p; Vdbe *v = pParse->pVdbe; sqlite3 *db = pParse->db; assert( v ); for(p = pParse->pAinc; p; p = p->pNext){ + static const int iLn = VDBE_OFFSET_LINENO(2); + static const VdbeOpList autoIncEnd[] = { + /* 0 */ {OP_NotNull, 0, 2, 0}, + /* 1 */ {OP_NewRowid, 0, 0, 0}, + /* 2 */ {OP_MakeRecord, 0, 2, 0}, + /* 3 */ {OP_Insert, 0, 0, 0}, + /* 4 */ {OP_Close, 0, 0, 0} + }; + VdbeOp *aOp; Db *pDb = &db->aDb[p->iDb]; - int addr1; int iRec; int memId = p->regCtr; iRec = sqlite3GetTempReg(pParse); assert( sqlite3SchemaMutexHeld(db, 0, pDb->pSchema) ); sqlite3OpenTable(pParse, 0, p->iDb, pDb->pSchema->pSeqTab, OP_OpenWrite); - addr1 = sqlite3VdbeAddOp1(v, OP_NotNull, memId+1); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_NewRowid, 0, memId+1); - sqlite3VdbeJumpHere(v, addr1); - sqlite3VdbeAddOp3(v, OP_MakeRecord, memId-1, 2, iRec); - sqlite3VdbeAddOp3(v, OP_Insert, 0, iRec, memId+1); - sqlite3VdbeChangeP5(v, OPFLAG_APPEND); - sqlite3VdbeAddOp0(v, OP_Close); + aOp = sqlite3VdbeAddOpList(v, ArraySize(autoIncEnd), autoIncEnd, iLn); + if( aOp==0 ) break; + aOp[0].p1 = memId+1; + aOp[1].p2 = memId+1; + aOp[2].p1 = memId-1; + aOp[2].p3 = iRec; + aOp[3].p2 = iRec; + aOp[3].p3 = memId+1; + aOp[3].p5 = OPFLAG_APPEND; sqlite3ReleaseTempReg(pParse, iRec); } } +void sqlite3AutoincrementEnd(Parse *pParse){ + if( pParse->pAinc ) autoIncrementEnd(pParse); +} #else /* ** If SQLITE_OMIT_AUTOINCREMENT is defined, then the three routines @@ -762,7 +787,7 @@ void sqlite3Insert( int nIdx; nIdx = sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, 0, -1, 0, &iDataCur, &iIdxCur); - aRegIdx = sqlite3DbMallocRaw(db, sizeof(int)*(nIdx+1)); + aRegIdx = sqlite3DbMallocRawNN(db, sizeof(int)*(nIdx+1)); if( aRegIdx==0 ){ goto insert_cleanup; } diff --git a/src/legacy.c b/src/legacy.c index a10006e558..1b5e518d43 100644 --- a/src/legacy.c +++ b/src/legacy.c @@ -90,7 +90,7 @@ int sqlite3_exec( for(i=0; imallocFailed = 1; + sqlite3OomFault(db); goto exec_out; } } diff --git a/src/main.c b/src/main.c index 97f40016e2..b5ba280f21 100644 --- a/src/main.c +++ b/src/main.c @@ -698,12 +698,12 @@ static int setupLookaside(sqlite3 *db, void *pBuf, int sz, int cnt){ p = (LookasideSlot*)&((u8*)p)[sz]; } db->lookaside.pEnd = p; - db->lookaside.bEnabled = 1; + db->lookaside.bDisable = 0; db->lookaside.bMalloced = pBuf==0 ?1:0; }else{ db->lookaside.pStart = db; db->lookaside.pEnd = db; - db->lookaside.bEnabled = 0; + db->lookaside.bDisable = 1; db->lookaside.bMalloced = 0; } #endif /* SQLITE_OMIT_LOOKASIDE */ @@ -2229,7 +2229,7 @@ const void *sqlite3_errmsg16(sqlite3 *db){ ** be cleared before returning. Do this directly, instead of via ** sqlite3ApiExit(), to avoid setting the database handle error message. */ - db->mallocFailed = 0; + sqlite3OomClear(db); } sqlite3_mutex_leave(db->mutex); return z; @@ -2867,7 +2867,7 @@ static int openDatabase( db->openFlags = flags; rc = sqlite3ParseUri(zVfs, zFilename, &flags, &db->pVfs, &zOpen, &zErrMsg); if( rc!=SQLITE_OK ){ - if( rc==SQLITE_NOMEM ) db->mallocFailed = 1; + if( rc==SQLITE_NOMEM ) sqlite3OomFault(db); sqlite3ErrorWithMsg(db, rc, zErrMsg ? "%s" : 0, zErrMsg); sqlite3_free(zErrMsg); goto opendb_out; diff --git a/src/malloc.c b/src/malloc.c index c8a04128cf..1ff0a5221d 100644 --- a/src/malloc.c +++ b/src/malloc.c @@ -575,10 +575,24 @@ void *sqlite3MallocZero(u64 n){ ** the mallocFailed flag in the connection pointer. */ void *sqlite3DbMallocZero(sqlite3 *db, u64 n){ - void *p = sqlite3DbMallocRaw(db, n); - if( p ){ - memset(p, 0, (size_t)n); - } + void *p; + testcase( db==0 ); + p = sqlite3DbMallocRaw(db, n); + if( p ) memset(p, 0, (size_t)n); + return p; +} + + +/* Finish the work of sqlite3DbMallocRawNN for the unusual and +** slower case when the allocation cannot be fulfilled using lookaside. +*/ +static SQLITE_NOINLINE void *dbMallocRawFinish(sqlite3 *db, u64 n){ + void *p; + assert( db!=0 ); + p = sqlite3Malloc(n); + if( !p ) sqlite3OomFault(db); + sqlite3MemdebugSetType(p, + (db->lookaside.bDisable==0) ? MEMTYPE_LOOKASIDE : MEMTYPE_HEAP); return p; } @@ -600,67 +614,70 @@ void *sqlite3DbMallocZero(sqlite3 *db, u64 n){ ** ** In other words, if a subsequent malloc (ex: "b") worked, it is assumed ** that all prior mallocs (ex: "a") worked too. +** +** The sqlite3MallocRawNN() variant guarantees that the "db" parameter is +** not a NULL pointer. */ -static SQLITE_NOINLINE void *dbMallocRawFinish(sqlite3 *db, u64 n); void *sqlite3DbMallocRaw(sqlite3 *db, u64 n){ - assert( db==0 || sqlite3_mutex_held(db->mutex) ); - assert( db==0 || db->pnBytesFreed==0 ); + void *p; + if( db ) return sqlite3DbMallocRawNN(db, n); + p = sqlite3Malloc(n); + sqlite3MemdebugSetType(p, MEMTYPE_HEAP); + return p; +} +void *sqlite3DbMallocRawNN(sqlite3 *db, u64 n){ + assert( db!=0 ); + assert( sqlite3_mutex_held(db->mutex) ); + assert( db->pnBytesFreed==0 ); #ifndef SQLITE_OMIT_LOOKASIDE - if( db ){ - LookasideSlot *pBuf; - if( db->mallocFailed ){ - return 0; - } - if( db->lookaside.bEnabled ){ - if( n>db->lookaside.sz ){ - db->lookaside.anStat[1]++; - }else if( (pBuf = db->lookaside.pFree)==0 ){ - db->lookaside.anStat[2]++; - }else{ - db->lookaside.pFree = pBuf->pNext; - db->lookaside.nOut++; - db->lookaside.anStat[0]++; - if( db->lookaside.nOut>db->lookaside.mxOut ){ - db->lookaside.mxOut = db->lookaside.nOut; - } - return (void*)pBuf; + LookasideSlot *pBuf; + if( db->lookaside.bDisable==0 ){ + assert( db->mallocFailed==0 ); + if( n>db->lookaside.sz ){ + db->lookaside.anStat[1]++; + }else if( (pBuf = db->lookaside.pFree)==0 ){ + db->lookaside.anStat[2]++; + }else{ + db->lookaside.pFree = pBuf->pNext; + db->lookaside.nOut++; + db->lookaside.anStat[0]++; + if( db->lookaside.nOut>db->lookaside.mxOut ){ + db->lookaside.mxOut = db->lookaside.nOut; } + return (void*)pBuf; } + }else if( db->mallocFailed ){ + return 0; } #else - if( db && db->mallocFailed ){ + if( db->mallocFailed ){ return 0; } #endif return dbMallocRawFinish(db, n); } -static SQLITE_NOINLINE void *dbMallocRawFinish(sqlite3 *db, u64 n){ - void *p = sqlite3Malloc(n); - if( !p && db ){ - db->mallocFailed = 1; - } - sqlite3MemdebugSetType(p, - (db && db->lookaside.bEnabled) ? MEMTYPE_LOOKASIDE : MEMTYPE_HEAP); - return p; -} + +/* Forward declaration */ +static SQLITE_NOINLINE void *dbReallocFinish(sqlite3 *db, void *p, u64 n); /* ** Resize the block of memory pointed to by p to n bytes. If the ** resize fails, set the mallocFailed flag in the connection object. */ void *sqlite3DbRealloc(sqlite3 *db, void *p, u64 n){ + assert( db!=0 ); + if( p==0 ) return sqlite3DbMallocRawNN(db, n); + assert( sqlite3_mutex_held(db->mutex) ); + if( isLookaside(db,p) && n<=db->lookaside.sz ) return p; + return dbReallocFinish(db, p, n); +} +static SQLITE_NOINLINE void *dbReallocFinish(sqlite3 *db, void *p, u64 n){ void *pNew = 0; assert( db!=0 ); - assert( sqlite3_mutex_held(db->mutex) ); + assert( p!=0 ); if( db->mallocFailed==0 ){ - if( p==0 ){ - return sqlite3DbMallocRaw(db, n); - } if( isLookaside(db, p) ){ - if( n<=db->lookaside.sz ){ - return p; - } - pNew = sqlite3DbMallocRaw(db, n); + pNew = sqlite3DbMallocRawNN(db, n); if( pNew ){ memcpy(pNew, p, db->lookaside.sz); sqlite3DbFree(db, p); @@ -671,10 +688,10 @@ void *sqlite3DbRealloc(sqlite3 *db, void *p, u64 n){ sqlite3MemdebugSetType(p, MEMTYPE_HEAP); pNew = sqlite3_realloc64(p, n); if( !pNew ){ - db->mallocFailed = 1; + sqlite3OomFault(db); } sqlite3MemdebugSetType(pNew, - (db->lookaside.bEnabled ? MEMTYPE_LOOKASIDE : MEMTYPE_HEAP)); + (db->lookaside.bDisable==0 ? MEMTYPE_LOOKASIDE : MEMTYPE_HEAP)); } } return pNew; @@ -716,11 +733,12 @@ char *sqlite3DbStrDup(sqlite3 *db, const char *z){ } char *sqlite3DbStrNDup(sqlite3 *db, const char *z, u64 n){ char *zNew; + assert( db!=0 ); if( z==0 ){ return 0; } assert( (n&0x7fffffff)==n ); - zNew = sqlite3DbMallocRaw(db, n+1); + zNew = sqlite3DbMallocRawNN(db, n+1); if( zNew ){ memcpy(zNew, z, (size_t)n); zNew[n] = 0; @@ -736,11 +754,43 @@ void sqlite3SetString(char **pz, sqlite3 *db, const char *zNew){ *pz = sqlite3DbStrDup(db, zNew); } +/* +** Call this routine to record the fact that an OOM (out-of-memory) error +** has happened. This routine will set db->mallocFailed, and also +** temporarily disable the lookaside memory allocator and interrupt +** any running VDBEs. +*/ +void sqlite3OomFault(sqlite3 *db){ + if( db->mallocFailed==0 && db->bBenignMalloc==0 ){ + db->mallocFailed = 1; + if( db->nVdbeExec>0 ){ + db->u1.isInterrupted = 1; + } + db->lookaside.bDisable++; + } +} + +/* +** This routine reactivates the memory allocator and clears the +** db->mallocFailed flag as necessary. +** +** The memory allocator is not restarted if there are running +** VDBEs. +*/ +void sqlite3OomClear(sqlite3 *db){ + if( db->mallocFailed && db->nVdbeExec==0 ){ + db->mallocFailed = 0; + db->u1.isInterrupted = 0; + assert( db->lookaside.bDisable>0 ); + db->lookaside.bDisable--; + } +} + /* ** Take actions at the end of an API call to indicate an OOM error */ static SQLITE_NOINLINE int apiOomError(sqlite3 *db){ - db->mallocFailed = 0; + sqlite3OomClear(db); sqlite3Error(db, SQLITE_NOMEM); return SQLITE_NOMEM; } diff --git a/src/os_unix.c b/src/os_unix.c index 81422f215d..fe1fc6af19 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -5887,12 +5887,7 @@ static int unixDelete( int fd; rc = osOpenDirectory(zPath, &fd); if( rc==SQLITE_OK ){ -#if OS_VXWORKS - if( fsync(fd)==-1 ) -#else - if( fsync(fd) ) -#endif - { + if( full_fsync(fd,0,0) ){ rc = unixLogError(SQLITE_IOERR_DIR_FSYNC, "fsync", zPath); } robust_close(0, fd, __LINE__); @@ -6936,7 +6931,7 @@ static int proxyTakeConch(unixFile *pFile){ writeSize = PROXY_PATHINDEX + strlen(&writeBuffer[PROXY_PATHINDEX]); robust_ftruncate(conchFile->h, writeSize); rc = unixWrite((sqlite3_file *)conchFile, writeBuffer, writeSize, 0); - fsync(conchFile->h); + full_fsync(conchFile->h,0,0); /* If we created a new conch file (not just updated the contents of a ** valid conch file), try to match the permissions of the database */ diff --git a/src/pager.c b/src/pager.c index 0afbe215c6..5c61968e36 100644 --- a/src/pager.c +++ b/src/pager.c @@ -637,6 +637,7 @@ struct Pager { u8 useJournal; /* Use a rollback journal on this file */ u8 noSync; /* Do not sync the journal if true */ u8 fullSync; /* Do extra syncs of the journal for robustness */ + u8 extraSync; /* sync directory after journal delete */ u8 ckptSyncFlags; /* SYNC_NORMAL or SYNC_FULL for checkpoint */ u8 walSyncFlags; /* SYNC_NORMAL or SYNC_FULL for wal writes */ u8 syncFlags; /* SYNC_NORMAL or SYNC_FULL otherwise */ @@ -1997,8 +1998,7 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){ ); sqlite3OsClose(pPager->jfd); if( bDelete ){ - rc = sqlite3OsDelete(pPager->pVfs, pPager->zJournal, - pPager->fullSync && SQLITE_EXTRA_DURABLE); + rc = sqlite3OsDelete(pPager->pVfs, pPager->zJournal, pPager->extraSync); } } } @@ -3504,9 +3504,15 @@ void sqlite3PagerSetFlags( unsigned pgFlags /* Various flags */ ){ unsigned level = pgFlags & PAGER_SYNCHRONOUS_MASK; - assert( level>=1 && level<=3 ); - pPager->noSync = (level==1 || pPager->tempFile) ?1:0; - pPager->fullSync = (level==3 && !pPager->tempFile) ?1:0; + if( pPager->tempFile ){ + pPager->noSync = 1; + pPager->fullSync = 0; + pPager->extraSync = 0; + }else{ + pPager->noSync = level==PAGER_SYNCHRONOUS_OFF ?1:0; + pPager->fullSync = level>=PAGER_SYNCHRONOUS_FULL ?1:0; + pPager->extraSync = level==PAGER_SYNCHRONOUS_EXTRA ?1:0; + } if( pPager->noSync ){ pPager->syncFlags = 0; pPager->ckptSyncFlags = 0; @@ -4811,11 +4817,17 @@ act_like_temp_file: pPager->noSync = pPager->tempFile; if( pPager->noSync ){ assert( pPager->fullSync==0 ); + assert( pPager->extraSync==0 ); assert( pPager->syncFlags==0 ); assert( pPager->walSyncFlags==0 ); assert( pPager->ckptSyncFlags==0 ); }else{ pPager->fullSync = 1; +#if SQLITE_EXTRA_DURABLE + pPager->extraSync = 1; +#else + pPager->extraSync = 0; +#endif pPager->syncFlags = SQLITE_SYNC_NORMAL; pPager->walSyncFlags = SQLITE_SYNC_NORMAL | WAL_SYNC_TRANSACTIONS; pPager->ckptSyncFlags = SQLITE_SYNC_NORMAL; diff --git a/src/pager.h b/src/pager.h index 3552a876e7..8d9f08108d 100644 --- a/src/pager.h +++ b/src/pager.h @@ -90,11 +90,12 @@ typedef struct PgHdr DbPage; #define PAGER_SYNCHRONOUS_OFF 0x01 /* PRAGMA synchronous=OFF */ #define PAGER_SYNCHRONOUS_NORMAL 0x02 /* PRAGMA synchronous=NORMAL */ #define PAGER_SYNCHRONOUS_FULL 0x03 /* PRAGMA synchronous=FULL */ -#define PAGER_SYNCHRONOUS_MASK 0x03 /* Mask for three values above */ -#define PAGER_FULLFSYNC 0x04 /* PRAGMA fullfsync=ON */ -#define PAGER_CKPT_FULLFSYNC 0x08 /* PRAGMA checkpoint_fullfsync=ON */ -#define PAGER_CACHESPILL 0x10 /* PRAGMA cache_spill=ON */ -#define PAGER_FLAGS_MASK 0x1c /* All above except SYNCHRONOUS */ +#define PAGER_SYNCHRONOUS_EXTRA 0x04 /* PRAGMA synchronous=EXTRA */ +#define PAGER_SYNCHRONOUS_MASK 0x07 /* Mask for four values above */ +#define PAGER_FULLFSYNC 0x08 /* PRAGMA fullfsync=ON */ +#define PAGER_CKPT_FULLFSYNC 0x10 /* PRAGMA checkpoint_fullfsync=ON */ +#define PAGER_CACHESPILL 0x20 /* PRAGMA cache_spill=ON */ +#define PAGER_FLAGS_MASK 0x38 /* All above except SYNCHRONOUS */ /* ** The remainder of this file contains the declarations of the functions diff --git a/src/parse.y b/src/parse.y index 846c2504b7..0bfe4e473a 100644 --- a/src/parse.y +++ b/src/parse.y @@ -106,6 +106,15 @@ struct TrigEvent { int a; IdList * b; }; */ struct AttachKey { int type; Token key; }; +/* +** Disable lookaside memory allocation for objects that might be +** shared across database connections. +*/ +static void disableLookaside(Parse *pParse){ + pParse->disableLookaside++; + pParse->db->lookaside.bDisable++; +} + } // end %include // Input is a single SQL command @@ -156,7 +165,7 @@ create_table ::= createkw temp(T) TABLE ifnotexists(E) nm(Y) dbnm(Z). { sqlite3StartTable(pParse,&Y,&Z,T,0,0,E); } createkw(A) ::= CREATE(X). { - pParse->db->lookaside.bEnabled = 0; + disableLookaside(pParse); A = X; } %type ifnotexists {int} @@ -1507,7 +1516,7 @@ cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt column(Y). { sqlite3AlterFinishAddColumn(pParse, &Y); } add_column_fullname ::= fullname(X). { - pParse->db->lookaside.bEnabled = 0; + disableLookaside(pParse); sqlite3AlterBeginAddColumn(pParse, X); } kwcolumn_opt ::= . diff --git a/src/pragma.c b/src/pragma.c index 1d827d7b70..c34d5421c2 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -32,8 +32,8 @@ /* ** Interpret the given string as a safety level. Return 0 for OFF, -** 1 for ON or NORMAL and 2 for FULL. Return 1 for an empty or -** unrecognized string argument. The FULL option is disallowed +** 1 for ON or NORMAL, 2 for FULL, and 3 for EXTRA. Return 1 for an empty or +** unrecognized string argument. The FULL and EXTRA option is disallowed ** if the omitFull parameter it 1. ** ** Note that the values returned are one less that the values that @@ -42,18 +42,21 @@ ** and older scripts may have used numbers 0 for OFF and 1 for ON. */ static u8 getSafetyLevel(const char *z, int omitFull, u8 dflt){ - /* 123456789 123456789 */ - static const char zText[] = "onoffalseyestruefull"; - static const u8 iOffset[] = {0, 1, 2, 4, 9, 12, 16}; - static const u8 iLength[] = {2, 2, 3, 5, 3, 4, 4}; - static const u8 iValue[] = {1, 0, 0, 0, 1, 1, 2}; + /* 123456789 123456789 123 */ + static const char zText[] = "onoffalseyestruextrafull"; + static const u8 iOffset[] = {0, 1, 2, 4, 9, 12, 15, 20}; + static const u8 iLength[] = {2, 2, 3, 5, 3, 4, 5, 4}; + static const u8 iValue[] = {1, 0, 0, 0, 1, 1, 3, 2}; + /* on no off false yes true extra full */ int i, n; if( sqlite3Isdigit(*z) ){ return (u8)sqlite3Atoi(z); } n = sqlite3Strlen30(z); - for(i=0; inextPagesize = sqlite3Atoi(zRight); if( SQLITE_NOMEM==sqlite3BtreeSetPageSize(pBt, db->nextPagesize,-1,0) ){ - db->mallocFailed = 1; + sqlite3OomFault(db); } } break; @@ -972,7 +975,7 @@ void sqlite3Pragma( /* ** PRAGMA [schema.]synchronous - ** PRAGMA [schema.]synchronous=OFF|ON|NORMAL|FULL + ** PRAGMA [schema.]synchronous=OFF|ON|NORMAL|FULL|EXTRA ** ** Return or set the local value of the synchronous flag. Changing ** the local value does not make changes to the disk file and the @@ -1599,16 +1602,15 @@ void sqlite3Pragma( static const int iLn = VDBE_OFFSET_LINENO(2); static const VdbeOpList endCode[] = { { OP_AddImm, 1, 0, 0}, /* 0 */ - { OP_If, 1, 0, 0}, /* 1 */ + { OP_If, 1, 4, 0}, /* 1 */ { OP_String8, 0, 3, 0}, /* 2 */ - { OP_ResultRow, 3, 1, 0}, + { OP_ResultRow, 3, 1, 0}, /* 3 */ }; VdbeOp *aOp; aOp = sqlite3VdbeAddOpList(v, ArraySize(endCode), endCode, iLn); if( aOp ){ aOp[0].p2 = -mxErr; - aOp[1].p2 = sqlite3VdbeCurrentAddr(v); aOp[2].p4type = P4_STATIC; aOp[2].p4.z = "ok"; } diff --git a/src/prepare.c b/src/prepare.c index f8ad54665a..f74aa52e0b 100644 --- a/src/prepare.c +++ b/src/prepare.c @@ -28,11 +28,10 @@ static void corruptSchema( if( !db->mallocFailed && (db->flags & SQLITE_RecoveryMode)==0 ){ char *z; if( zObj==0 ) zObj = "?"; - z = sqlite3_mprintf("malformed database schema (%s)", zObj); - if( z && zExtra ) z = sqlite3_mprintf("%z - %s", z, zExtra); + z = sqlite3MPrintf(db, "malformed database schema (%s)", zObj); + if( zExtra ) z = sqlite3MPrintf(db, "%z - %s", z, zExtra); sqlite3DbFree(db, *pData->pzErrMsg); *pData->pzErrMsg = z; - if( z==0 ) db->mallocFailed = 1; } pData->rc = db->mallocFailed ? SQLITE_NOMEM : SQLITE_CORRUPT_BKPT; } @@ -91,7 +90,7 @@ int sqlite3InitCallback(void *pInit, int argc, char **argv, char **NotUsed){ }else{ pData->rc = rc; if( rc==SQLITE_NOMEM ){ - db->mallocFailed = 1; + sqlite3OomFault(db); }else if( rc!=SQLITE_INTERRUPT && (rc&0xFF)!=SQLITE_LOCKED ){ corruptSchema(pData, argv[0], sqlite3_errmsg(db)); } @@ -336,7 +335,7 @@ initone_error_out: error_out: if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ){ - db->mallocFailed = 1; + sqlite3OomFault(db); } return rc; } @@ -434,7 +433,7 @@ static void schemaIsValid(Parse *pParse){ if( !sqlite3BtreeIsInReadTrans(pBt) ){ rc = sqlite3BtreeBeginTrans(pBt, 0); if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ){ - db->mallocFailed = 1; + sqlite3OomFault(db); } if( rc!=SQLITE_OK ) return; openedTransaction = 1; @@ -497,6 +496,11 @@ void sqlite3ParserReset(Parse *pParse){ sqlite3 *db = pParse->db; sqlite3DbFree(db, pParse->aLabel); sqlite3ExprListDelete(db, pParse->pConstExpr); + if( db ){ + assert( db->lookaside.bDisable >= pParse->disableLookaside ); + db->lookaside.bDisable -= pParse->disableLookaside; + } + pParse->disableLookaside = 0; } } @@ -592,9 +596,6 @@ static int sqlite3Prepare( } assert( 0==pParse->nQueryLoop ); - if( db->mallocFailed ){ - pParse->rc = SQLITE_NOMEM; - } if( pParse->rc==SQLITE_DONE ) pParse->rc = SQLITE_OK; if( pParse->checkSchema ){ schemaIsValid(pParse); @@ -716,7 +717,7 @@ int sqlite3Reprepare(Vdbe *p){ rc = sqlite3LockAndPrepare(db, zSql, -1, 0, p, &pNew, 0); if( rc ){ if( rc==SQLITE_NOMEM ){ - db->mallocFailed = 1; + sqlite3OomFault(db); } assert( pNew==0 ); return rc; diff --git a/src/printf.c b/src/printf.c index 946161f64c..238997f37d 100644 --- a/src/printf.c +++ b/src/printf.c @@ -929,7 +929,7 @@ char *sqlite3VMPrintf(sqlite3 *db, const char *zFormat, va_list ap){ sqlite3VXPrintf(&acc, zFormat, ap); z = sqlite3StrAccumFinish(&acc); if( acc.accError==STRACCUM_NOMEM ){ - db->mallocFailed = 1; + sqlite3OomFault(db); } return z; } diff --git a/src/rowset.c b/src/rowset.c index ff5593892a..c2e73ed72e 100644 --- a/src/rowset.c +++ b/src/rowset.c @@ -181,7 +181,7 @@ static struct RowSetEntry *rowSetEntryAlloc(RowSet *p){ assert( p!=0 ); if( p->nFresh==0 ){ struct RowSetChunk *pNew; - pNew = sqlite3DbMallocRaw(p->db, sizeof(*pNew)); + pNew = sqlite3DbMallocRawNN(p->db, sizeof(*pNew)); if( pNew==0 ){ return 0; } diff --git a/src/select.c b/src/select.c index 891b123543..ea3e920b9f 100644 --- a/src/select.c +++ b/src/select.c @@ -112,7 +112,7 @@ Select *sqlite3SelectNew( Select *pNew; Select standin; sqlite3 *db = pParse->db; - pNew = sqlite3DbMallocRaw(db, sizeof(*pNew) ); + pNew = sqlite3DbMallocRawNN(db, sizeof(*pNew) ); if( pNew==0 ){ assert( db->mallocFailed ); pNew = &standin; @@ -1016,7 +1016,7 @@ KeyInfo *sqlite3KeyInfoAlloc(sqlite3 *db, int N, int X){ p->nRef = 1; memset(&p[1], 0, nExtra); }else{ - db->mallocFailed = 1; + sqlite3OomFault(db); } return p; } @@ -1677,7 +1677,7 @@ int sqlite3ColumnsFromExprList( pCol->zName = zName; sqlite3ColumnPropertiesFromName(0, pCol); if( zName && sqlite3HashInsert(&ht, zName, pCol)==pCol ){ - db->mallocFailed = 1; + sqlite3OomFault(db); } } sqlite3HashClear(&ht); @@ -1764,7 +1764,7 @@ Table *sqlite3ResultSetOfSelect(Parse *pParse, Select *pSelect){ } /* The sqlite3ResultSetOfSelect() is only used n contexts where lookaside ** is disabled */ - assert( db->lookaside.bEnabled==0 ); + assert( db->lookaside.bDisable ); pTab->nRef = 1; pTab->zName = 0; pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); @@ -2870,7 +2870,7 @@ static int multiSelectOrderBy( ** to the right and the left are evaluated, they use the correct ** collation. */ - aPermute = sqlite3DbMallocRaw(db, sizeof(int)*(nOrderBy + 1)); + aPermute = sqlite3DbMallocRawNN(db, sizeof(int)*(nOrderBy + 1)); if( aPermute ){ struct ExprList_item *pItem; aPermute[0] = nOrderBy; @@ -5562,7 +5562,8 @@ int sqlite3Select( if( flag ){ pMinMax = sqlite3ExprListDup(db, pMinMax, 0); pDel = pMinMax; - if( pMinMax && !db->mallocFailed ){ + assert( db->mallocFailed || pMinMax!=0 ); + if( !db->mallocFailed ){ pMinMax->a[0].sortOrder = flag!=WHERE_ORDERBY_MIN ?1:0; pMinMax->a[0].pExpr->op = TK_COLUMN; } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 3da16465cf..245bd0a129 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1095,8 +1095,8 @@ struct Schema { ** lookaside allocations are not used to construct the schema objects. */ struct Lookaside { + u32 bDisable; /* Only operate the lookaside when zero */ u16 sz; /* Size of each buffer in bytes */ - u8 bEnabled; /* False to disable new lookaside allocations */ u8 bMalloced; /* True if pStart obtained from sqlite3_malloc() */ int nOut; /* Number of buffers currently checked out */ int mxOut; /* Highwater mark for nOut */ @@ -1179,6 +1179,7 @@ struct sqlite3 { u8 autoCommit; /* The auto-commit flag. */ u8 temp_store; /* 1: file 2: memory 0: default */ u8 mallocFailed; /* True if we have seen a malloc failure */ + u8 bBenignMalloc; /* Do not require OOMs if true */ u8 dfltLockMode; /* Default locking-mode for attached dbs */ signed char nextAutovac; /* Autovac setting after VACUUM if >=0 */ u8 suppressErr; /* Do not issue error messages if true */ @@ -1294,10 +1295,10 @@ struct sqlite3 { */ #define SQLITE_VdbeTrace 0x00000001 /* True to trace VDBE execution */ #define SQLITE_InternChanges 0x00000002 /* Uncommitted Hash table changes */ -#define SQLITE_FullFSync 0x00000004 /* Use full fsync on the backend */ -#define SQLITE_CkptFullFSync 0x00000008 /* Use full fsync for checkpoint */ -#define SQLITE_CacheSpill 0x00000010 /* OK to spill pager cache */ -#define SQLITE_FullColNames 0x00000020 /* Show full column names on SELECT */ +#define SQLITE_FullColNames 0x00000004 /* Show full column names on SELECT */ +#define SQLITE_FullFSync 0x00000008 /* Use full fsync on the backend */ +#define SQLITE_CkptFullFSync 0x00000010 /* Use full fsync for checkpoint */ +#define SQLITE_CacheSpill 0x00000020 /* OK to spill pager cache */ #define SQLITE_ShortColNames 0x00000040 /* Show short columns names */ #define SQLITE_CountRows 0x00000080 /* Count rows changed by INSERT, */ /* DELETE, or UPDATE and return */ @@ -2649,7 +2650,7 @@ struct SelectDest { ** tables, the following information is attached to the Table.u.autoInc.p ** pointer of each autoincrement table to record some side information that ** the code generator needs. We have to keep per-table autoincrement -** information in case inserts are down within triggers. Triggers do not +** information in case inserts are done within triggers. Triggers do not ** normally coordinate their activities, but we do need to coordinate the ** loading and saving of autoincrement information. */ @@ -2741,6 +2742,7 @@ struct Parse { u8 mayAbort; /* True if statement may throw an ABORT exception */ u8 hasCompound; /* Need to invoke convertCompoundSelectToSubquery() */ u8 okConstFactor; /* OK to factor out constants */ + u8 disableLookaside; /* Number of times lookaside has been disabled */ int aTempReg[8]; /* Holding area for temporary registers */ int nRangeReg; /* Size of the temporary register block */ int iRangeReg; /* First register in temporary register block */ @@ -3237,6 +3239,7 @@ void *sqlite3Malloc(u64); void *sqlite3MallocZero(u64); void *sqlite3DbMallocZero(sqlite3*, u64); void *sqlite3DbMallocRaw(sqlite3*, u64); +void *sqlite3DbMallocRawNN(sqlite3*, u64); char *sqlite3DbStrDup(sqlite3*,const char*); char *sqlite3DbStrNDup(sqlite3*,const char*, u64); void *sqlite3Realloc(void*, u64); @@ -3789,6 +3792,8 @@ int sqlite3CreateFunc(sqlite3 *, const char *, int, int, void *, void (*)(sqlite3_context*,int,sqlite3_value **), void (*)(sqlite3_context*), FuncDestructor *pDestructor ); +void sqlite3OomFault(sqlite3*); +void sqlite3OomClear(sqlite3*); int sqlite3ApiExit(sqlite3 *db, int); int sqlite3OpenTempDatabase(Parse *); diff --git a/src/test_func.c b/src/test_func.c index 63cf18e3f7..579177b70c 100644 --- a/src/test_func.c +++ b/src/test_func.c @@ -166,9 +166,7 @@ static void test_agg_errmsg16_final(sqlite3_context *ctx){ const void *z; sqlite3 * db = sqlite3_context_db_handle(ctx); sqlite3_aggregate_context(ctx, 2048); - sqlite3BeginBenignMalloc(); z = sqlite3_errmsg16(db); - sqlite3EndBenignMalloc(); sqlite3_result_text16(ctx, z, -1, SQLITE_TRANSIENT); #endif } diff --git a/src/tokenize.c b/src/tokenize.c index b85e35dc10..5bee3d5a84 100644 --- a/src/tokenize.c +++ b/src/tokenize.c @@ -390,7 +390,6 @@ int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){ void *pEngine; /* The LEMON-generated LALR(1) parser */ int tokenType; /* type of the next token */ int lastTokenParsed = -1; /* type of the previous token */ - u8 enableLookaside; /* Saved value of db->lookaside.bEnabled */ sqlite3 *db = pParse->db; /* The database connection */ int mxSqlLen; /* Max length of an SQL string */ @@ -406,7 +405,7 @@ int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){ /* sqlite3ParserTrace(stdout, "parser: "); */ pEngine = sqlite3ParserAlloc(sqlite3Malloc); if( pEngine==0 ){ - db->mallocFailed = 1; + sqlite3OomFault(db); return SQLITE_NOMEM; } assert( pParse->pNewTable==0 ); @@ -414,8 +413,6 @@ int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){ assert( pParse->nVar==0 ); assert( pParse->nzVar==0 ); assert( pParse->azVar==0 ); - enableLookaside = db->lookaside.bEnabled; - if( db->lookaside.pStart ) db->lookaside.bEnabled = 1; while( zSql[i]!=0 ){ assert( i>=0 ); pParse->sLastToken.z = &zSql[i]; @@ -428,7 +425,6 @@ int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){ if( tokenType>=TK_SPACE ){ assert( tokenType==TK_SPACE || tokenType==TK_ILLEGAL ); if( db->u1.isInterrupted ){ - sqlite3ErrorMsg(pParse, "interrupt"); pParse->rc = SQLITE_INTERRUPT; break; } @@ -463,7 +459,6 @@ int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){ sqlite3_mutex_leave(sqlite3MallocMutex()); #endif /* YYDEBUG */ sqlite3ParserFree(pEngine, sqlite3_free); - db->lookaside.bEnabled = enableLookaside; if( db->mallocFailed ){ pParse->rc = SQLITE_NOMEM; } diff --git a/src/trigger.c b/src/trigger.c index 4bfb55af69..72c31eb2a3 100644 --- a/src/trigger.c +++ b/src/trigger.c @@ -323,7 +323,7 @@ void sqlite3FinishTrigger( assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); pTrig = sqlite3HashInsert(pHash, zName, pTrig); if( pTrig ){ - db->mallocFailed = 1; + sqlite3OomFault(db); }else if( pLink->pSchema==pLink->pTabSchema ){ Table *pTab; pTab = sqlite3HashFind(&pLink->pTabSchema->tblHash, pLink->table); diff --git a/src/update.c b/src/update.c index d0f958a295..ecb66d23ea 100644 --- a/src/update.c +++ b/src/update.c @@ -197,7 +197,7 @@ void sqlite3Update( /* Allocate space for aXRef[], aRegIdx[], and aToOpen[]. ** Initialize aXRef[] and aToOpen[] to their default values. */ - aXRef = sqlite3DbMallocRaw(db, sizeof(int) * (pTab->nCol+nIdx) + nIdx+2 ); + aXRef = sqlite3DbMallocRawNN(db, sizeof(int) * (pTab->nCol+nIdx) + nIdx+2 ); if( aXRef==0 ) goto update_cleanup; aRegIdx = aXRef+pTab->nCol; aToOpen = (u8*)(aRegIdx+nIdx); diff --git a/src/util.c b/src/util.c index 96f7c7f3dc..37b585b2ea 100644 --- a/src/util.c +++ b/src/util.c @@ -1150,7 +1150,7 @@ void *sqlite3HexToBlob(sqlite3 *db, const char *z, int n){ char *zBlob; int i; - zBlob = (char *)sqlite3DbMallocRaw(db, n/2 + 1); + zBlob = (char *)sqlite3DbMallocRawNN(db, n/2 + 1); n--; if( zBlob ){ for(i=0; ipc]; rc==SQLITE_OK; pOp++){ assert( pOp>=aOp && pOp<&aOp[p->nOp]); - if( db->mallocFailed ) goto no_mem; #ifdef VDBE_PROFILE start = sqlite3Hwtime(); #endif @@ -1637,7 +1636,7 @@ case OP_Function0: { assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) ); assert( n==0 || (pOp->p2>0 && pOp->p2+n<=(p->nMem-p->nCursor)+1) ); assert( pOp->p3p2 || pOp->p3>=pOp->p2+n ); - pCtx = sqlite3DbMallocRaw(db, sizeof(*pCtx) + (n-1)*sizeof(sqlite3_value*)); + pCtx = sqlite3DbMallocRawNN(db, sizeof(*pCtx) + (n-1)*sizeof(sqlite3_value*)); if( pCtx==0 ) goto no_mem; pCtx->pOut = 0; pCtx->pFunc = pOp->p4.pFunc; @@ -2881,7 +2880,7 @@ case OP_Savepoint: { #endif /* Create a new savepoint structure. */ - pNew = sqlite3DbMallocRaw(db, sizeof(Savepoint)+nName+1); + pNew = sqlite3DbMallocRawNN(db, sizeof(Savepoint)+nName+1); if( pNew ){ pNew->zName = (char *)&pNew[1]; memcpy(pNew->zName, zName, nName+1); @@ -5020,8 +5019,8 @@ case OP_IdxDelete: { ** occur, no unnecessary I/O happens. ** ** P4 may be an array of integers (type P4_INTARRAY) containing -** one entry for each column in the P3 table. If array entry a[i] -** is non-zero, then reading column (a[i]-1) from cursor P3 is +** one entry for each column in the P3 table. If array entry a(i) +** is non-zero, then reading column a(i)-1 from cursor P3 is ** equivalent to performing the deferred seek and then reading column i ** from P1. This information is stored in P3 and used to redirect ** reads against P3 over to P1, thus possibly avoiding the need to @@ -5481,7 +5480,7 @@ case OP_IntegrityCk: { assert( p->bIsReader ); nRoot = pOp->p2; assert( nRoot>0 ); - aRoot = sqlite3DbMallocRaw(db, sizeof(int)*(nRoot+1) ); + aRoot = sqlite3DbMallocRawNN(db, sizeof(int)*(nRoot+1) ); if( aRoot==0 ) goto no_mem; assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) ); pnErr = &aMem[pOp->p3]; @@ -5978,7 +5977,7 @@ case OP_AggStep0: { assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) ); assert( n==0 || (pOp->p2>0 && pOp->p2+n<=(p->nMem-p->nCursor)+1) ); assert( pOp->p3p2 || pOp->p3>=pOp->p2+n ); - pCtx = sqlite3DbMallocRaw(db, sizeof(*pCtx) + (n-1)*sizeof(sqlite3_value*)); + pCtx = sqlite3DbMallocRawNN(db, sizeof(*pCtx) + (n-1)*sizeof(sqlite3_value*)); if( pCtx==0 ) goto no_mem; pCtx->pMem = 0; pCtx->pFunc = pOp->p4.pFunc; @@ -6845,7 +6844,7 @@ vdbe_error_halt: sqlite3_log(rc, "statement aborts at %d: [%s] %s", (int)(pOp - aOp), p->zSql, p->zErrMsg); sqlite3VdbeHalt(p); - if( rc==SQLITE_IOERR_NOMEM ) db->mallocFailed = 1; + if( rc==SQLITE_IOERR_NOMEM ) sqlite3OomFault(db); rc = SQLITE_ERROR; if( resetSchemaOnFault>0 ){ sqlite3ResetOneSchema(db, resetSchemaOnFault-1); @@ -6875,7 +6874,7 @@ too_big: /* Jump to here if a malloc() fails. */ no_mem: - db->mallocFailed = 1; + sqlite3OomFault(db); sqlite3VdbeError(p, "out of memory"); rc = SQLITE_NOMEM; goto vdbe_error_halt; @@ -6896,7 +6895,7 @@ abort_due_to_error: */ abort_due_to_interrupt: assert( db->u1.isInterrupted ); - rc = SQLITE_INTERRUPT; + rc = db->mallocFailed ? SQLITE_NOMEM : SQLITE_INTERRUPT; p->rc = rc; sqlite3VdbeError(p, "%s", sqlite3ErrStr(rc)); goto vdbe_error_halt; diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 2743caa358..be70c67196 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -473,7 +473,7 @@ void sqlite3_result_error_nomem(sqlite3_context *pCtx){ sqlite3VdbeMemSetNull(pCtx->pOut); pCtx->isError = SQLITE_NOMEM; pCtx->fErrorOrAux = 1; - pCtx->pOut->db->mallocFailed = 1; + sqlite3OomFault(pCtx->pOut->db); } /* @@ -1101,7 +1101,7 @@ static const void *columnName( ** is the case, clear the mallocFailed flag and return NULL. */ if( db->mallocFailed ){ - db->mallocFailed = 0; + sqlite3OomClear(db); ret = 0; } sqlite3_mutex_leave(db->mutex); diff --git a/src/vdbeaux.c b/src/vdbeaux.c index df0d7fd887..12b917bbca 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -290,7 +290,7 @@ int sqlite3VdbeAddOp4Dup8( const u8 *zP4, /* The P4 operand */ int p4type /* P4 operand type */ ){ - char *p4copy = sqlite3DbMallocRaw(sqlite3VdbeDb(p), 8); + char *p4copy = sqlite3DbMallocRawNN(sqlite3VdbeDb(p), 8); if( p4copy ) memcpy(p4copy, zP4, 8); return sqlite3VdbeAddOp4(p, op, p1, p2, p3, p4copy, p4type); } @@ -647,6 +647,9 @@ VdbeOp *sqlite3VdbeTakeOpArray(Vdbe *p, int *pnOp, int *pnMaxArg){ /* ** Add a whole list of operations to the operation stack. Return a ** pointer to the first operation inserted. +** +** Non-zero P2 arguments to jump instructions are automatically adjusted +** so that the jump target is relative to the first operation inserted. */ VdbeOp *sqlite3VdbeAddOpList( Vdbe *p, /* Add opcodes to the prepared statement */ @@ -667,6 +670,9 @@ VdbeOp *sqlite3VdbeAddOpList( pOut->p1 = aOp->p1; pOut->p2 = aOp->p2; assert( aOp->p2>=0 ); + if( (sqlite3OpcodeProperty[aOp->opcode] & OPFLG_JUMP)!=0 && aOp->p2>0 ){ + pOut->p2 += p->nOp; + } pOut->p3 = aOp->p3; pOut->p4type = P4_NOTUSED; pOut->p4.p = 0; @@ -1418,7 +1424,6 @@ static void releaseMemArray(Mem *p, int N){ if( p && N ){ Mem *pEnd = &p[N]; sqlite3 *db = p->db; - u8 malloc_failed = db->mallocFailed; if( db->pnBytesFreed ){ do{ if( p->szMalloc ) sqlite3DbFree(db, p->zMalloc); @@ -1454,7 +1459,6 @@ static void releaseMemArray(Mem *p, int N){ p->flags = MEM_Undefined; }while( (++p)mallocFailed = malloc_failed; } } @@ -1515,7 +1519,7 @@ int sqlite3VdbeList( if( p->rc==SQLITE_NOMEM ){ /* This happens if a malloc() inside a call to sqlite3_column_text() or ** sqlite3_column_text16() failed. */ - db->mallocFailed = 1; + sqlite3OomFault(db); return SQLITE_ERROR; } @@ -2508,7 +2512,7 @@ int sqlite3VdbeHalt(Vdbe *p){ ** one, or the complete transaction if there is no statement transaction. */ - if( p->db->mallocFailed ){ + if( db->mallocFailed ){ p->rc = SQLITE_NOMEM; } if( p->aOnceFlag ) memset(p->aOnceFlag, 0, p->nOnceFlag); @@ -2669,7 +2673,7 @@ int sqlite3VdbeHalt(Vdbe *p){ } p->magic = VDBE_MAGIC_HALT; checkActiveVdbeCnt(db); - if( p->db->mallocFailed ){ + if( db->mallocFailed ){ p->rc = SQLITE_NOMEM; } @@ -2706,12 +2710,12 @@ int sqlite3VdbeTransferError(Vdbe *p){ sqlite3 *db = p->db; int rc = p->rc; if( p->zErrMsg ){ - u8 mallocFailed = db->mallocFailed; + db->bBenignMalloc++; sqlite3BeginBenignMalloc(); if( db->pErr==0 ) db->pErr = sqlite3ValueNew(db); sqlite3ValueSetStr(db->pErr, -1, p->zErrMsg, SQLITE_UTF8, SQLITE_TRANSIENT); sqlite3EndBenignMalloc(); - db->mallocFailed = mallocFailed; + db->bBenignMalloc--; db->errCode = rc; }else{ sqlite3Error(db, rc); diff --git a/src/vdbeblob.c b/src/vdbeblob.c index 7d343d742c..f2b3ffef2b 100644 --- a/src/vdbeblob.c +++ b/src/vdbeblob.c @@ -253,19 +253,17 @@ int sqlite3_blob_open( ** which closes the b-tree cursor and (possibly) commits the ** transaction. */ - static const int iLn = VDBE_OFFSET_LINENO(4); + static const int iLn = VDBE_OFFSET_LINENO(2); static const VdbeOpList openBlob[] = { - /* addr/ofst */ - /* {OP_Transaction, 0, 0, 0}, // 0/ inserted separately */ - {OP_TableLock, 0, 0, 0}, /* 1/0: Acquire a read or write lock */ - {OP_OpenRead, 0, 0, 0}, /* 2/1: Open a cursor */ - {OP_Variable, 1, 1, 0}, /* 3/2: Move ?1 into reg[1] */ - {OP_NotExists, 0, 8, 1}, /* 4/3: Seek the cursor */ - {OP_Column, 0, 0, 1}, /* 5/4 */ - {OP_ResultRow, 1, 0, 0}, /* 6/5 */ - {OP_Goto, 0, 3, 0}, /* 7/6 */ - {OP_Close, 0, 0, 0}, /* 8/7 */ - {OP_Halt, 0, 0, 0}, /* 9/8 */ + {OP_TableLock, 0, 0, 0}, /* 0: Acquire a read or write lock */ + {OP_OpenRead, 0, 0, 0}, /* 1: Open a cursor */ + {OP_Variable, 1, 1, 0}, /* 2: Move ?1 into reg[1] */ + {OP_NotExists, 0, 7, 1}, /* 3: Seek the cursor */ + {OP_Column, 0, 0, 1}, /* 4 */ + {OP_ResultRow, 1, 0, 0}, /* 5 */ + {OP_Goto, 0, 2, 0}, /* 6 */ + {OP_Close, 0, 0, 0}, /* 7 */ + {OP_Halt, 0, 0, 0}, /* 8 */ }; Vdbe *v = (Vdbe *)pBlob->pStmt; int iDb = sqlite3SchemaToIndex(db, pTab->pSchema); diff --git a/src/vdbemem.c b/src/vdbemem.c index cfa314476a..83b8feb34f 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -116,6 +116,7 @@ int sqlite3VdbeChangeEncoding(Mem *pMem, int desiredEnc){ SQLITE_NOINLINE int sqlite3VdbeMemGrow(Mem *pMem, int n, int bPreserve){ assert( sqlite3VdbeCheckMemInvariants(pMem) ); assert( (pMem->flags&MEM_RowSet)==0 ); + testcase( pMem->db==0 ); /* If the bPreserve flag is set to true, then the memory cell must already ** contain a valid string or blob value. */ @@ -719,7 +720,7 @@ void sqlite3VdbeMemSetRowSet(Mem *pMem){ assert( db!=0 ); assert( (pMem->flags & MEM_RowSet)==0 ); sqlite3VdbeMemRelease(pMem); - pMem->zMalloc = sqlite3DbMallocRaw(db, 64); + pMem->zMalloc = sqlite3DbMallocRawNN(db, 64); if( db->mallocFailed ){ pMem->flags = MEM_Null; pMem->szMalloc = 0; @@ -1377,7 +1378,7 @@ static int valueFromExpr( return rc; no_mem: - db->mallocFailed = 1; + sqlite3OomFault(db); sqlite3DbFree(db, zVal); assert( *ppVal==0 ); #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 @@ -1436,7 +1437,7 @@ static void recordFunc( db = sqlite3_context_db_handle(context); nRet = 1 + nSerial + nVal; - aRet = sqlite3DbMallocRaw(db, nRet); + aRet = sqlite3DbMallocRawNN(db, nRet); if( aRet==0 ){ sqlite3_result_error_nomem(context); }else{ diff --git a/src/vtab.c b/src/vtab.c index ea79cd8cef..e8794e687d 100644 --- a/src/vtab.c +++ b/src/vtab.c @@ -49,7 +49,7 @@ static int createModule( rc = SQLITE_MISUSE_BKPT; }else{ Module *pMod; - pMod = (Module *)sqlite3DbMallocRaw(db, sizeof(Module) + nName + 1); + pMod = (Module *)sqlite3DbMallocRawNN(db, sizeof(Module) + nName + 1); if( pMod ){ Module *pDel; char *zCopy = (char *)(&pMod[1]); @@ -62,7 +62,7 @@ static int createModule( pDel = (Module *)sqlite3HashInsert(&db->aModule,zCopy,(void*)pMod); assert( pDel==0 || pDel==pMod ); if( pDel ){ - db->mallocFailed = 1; + sqlite3OomFault(db); sqlite3DbFree(db, pDel); } } @@ -439,7 +439,7 @@ void sqlite3VtabFinishParse(Parse *pParse, Token *pEnd){ assert( sqlite3SchemaMutexHeld(db, 0, pSchema) ); pOld = sqlite3HashInsert(&pSchema->tblHash, zName, pTab); if( pOld ){ - db->mallocFailed = 1; + sqlite3OomFault(db); assert( pTab==pOld ); /* Malloc must have failed inside HashInsert() */ return; } @@ -530,7 +530,7 @@ static int vtabCallConstructor( db->pVtabCtx = &sCtx; rc = xConstruct(db, pMod->pAux, nArg, azArg, &pVTable->pVtab, &zErr); db->pVtabCtx = sCtx.pPrior; - if( rc==SQLITE_NOMEM ) db->mallocFailed = 1; + if( rc==SQLITE_NOMEM ) sqlite3OomFault(db); assert( sCtx.pTab==pTab ); if( SQLITE_OK!=rc ){ @@ -1088,7 +1088,7 @@ void sqlite3VtabMakeWritable(Parse *pParse, Table *pTab){ pToplevel->apVtabLock = apVtabLock; pToplevel->apVtabLock[pToplevel->nVtabLock++] = pTab; }else{ - pToplevel->db->mallocFailed = 1; + sqlite3OomFault(pToplevel->db); } } diff --git a/src/where.c b/src/where.c index 8c8dfbb7b6..1bee2ca6fe 100644 --- a/src/where.c +++ b/src/where.c @@ -943,7 +943,7 @@ static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){ if( rc!=SQLITE_OK ){ if( rc==SQLITE_NOMEM ){ - pParse->db->mallocFailed = 1; + sqlite3OomFault(pParse->db); }else if( !pVtab->zErrMsg ){ sqlite3ErrorMsg(pParse, "%s", sqlite3ErrStr(rc)); }else{ @@ -1735,7 +1735,7 @@ static int whereLoopResize(sqlite3 *db, WhereLoop *p, int n){ WhereTerm **paNew; if( p->nLSlot>=n ) return SQLITE_OK; n = (n+7)&~7; - paNew = sqlite3DbMallocRaw(db, sizeof(p->aLTerm[0])*n); + paNew = sqlite3DbMallocRawNN(db, sizeof(p->aLTerm[0])*n); if( paNew==0 ) return SQLITE_NOMEM; memcpy(paNew, p->aLTerm, sizeof(p->aLTerm[0])*p->nLSlot); if( p->aLTerm!=p->aLTermSpace ) sqlite3DbFree(db, p->aLTerm); @@ -2032,7 +2032,7 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ #endif if( p==0 ){ /* Allocate a new WhereLoop to add to the end of the list */ - *ppPrev = p = sqlite3DbMallocRaw(db, sizeof(WhereLoop)); + *ppPrev = p = sqlite3DbMallocRawNN(db, sizeof(WhereLoop)); if( p==0 ) return SQLITE_NOMEM; whereLoopInit(p); p->pNextLoop = 0; @@ -3529,7 +3529,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ /* Allocate and initialize space for aTo, aFrom and aSortCost[] */ nSpace = (sizeof(WherePath)+sizeof(WhereLoop*)*nLoop)*mxChoice*2; nSpace += sizeof(LogEst) * nOrderBy; - pSpace = sqlite3DbMallocRaw(db, nSpace); + pSpace = sqlite3DbMallocRawNN(db, nSpace); if( pSpace==0 ) return SQLITE_NOMEM; aTo = (WherePath*)pSpace; aFrom = aTo+mxChoice; diff --git a/src/wherecode.c b/src/wherecode.c index f5f45da397..bb48e5dc70 100644 --- a/src/wherecode.c +++ b/src/wherecode.c @@ -495,9 +495,7 @@ static int codeAllEqualityTerms( pParse->nMem += nReg; zAff = sqlite3DbStrDup(pParse->db,sqlite3IndexAffinityStr(pParse->db,pIdx)); - if( !zAff ){ - pParse->db->mallocFailed = 1; - } + assert( zAff!=0 || pParse->db->mallocFailed ); if( nSkip ){ int iIdxCur = pLevel->iIdxCur; diff --git a/src/whereexpr.c b/src/whereexpr.c index e0b8bac1b7..c84d2f230a 100644 --- a/src/whereexpr.c +++ b/src/whereexpr.c @@ -64,7 +64,7 @@ static int whereClauseInsert(WhereClause *pWC, Expr *p, u16 wtFlags){ if( pWC->nTerm>=pWC->nSlot ){ WhereTerm *pOld = pWC->a; sqlite3 *db = pWC->pWInfo->pParse->db; - pWC->a = sqlite3DbMallocRaw(db, sizeof(pWC->a[0])*pWC->nSlot*2 ); + pWC->a = sqlite3DbMallocRawNN(db, sizeof(pWC->a[0])*pWC->nSlot*2 ); if( pWC->a==0 ){ if( wtFlags & TERM_DYNAMIC ){ sqlite3ExprDelete(db, p); @@ -549,7 +549,7 @@ static void exprAnalyzeOrTerm( WhereAndInfo *pAndInfo; assert( (pOrTerm->wtFlags & (TERM_ANDINFO|TERM_ORINFO))==0 ); chngToIN = 0; - pAndInfo = sqlite3DbMallocRaw(db, sizeof(*pAndInfo)); + pAndInfo = sqlite3DbMallocRawNN(db, sizeof(*pAndInfo)); if( pAndInfo ){ WhereClause *pAndWC; WhereTerm *pAndTerm; @@ -563,7 +563,6 @@ static void exprAnalyzeOrTerm( sqlite3WhereSplit(pAndWC, pOrTerm->pExpr, TK_AND); sqlite3WhereExprAnalyze(pSrc, pAndWC); pAndWC->pOuter = pWC; - testcase( db->mallocFailed ); if( !db->mallocFailed ){ for(j=0, pAndTerm=pAndWC->a; jnTerm; j++, pAndTerm++){ assert( pAndTerm->pExpr ); diff --git a/test/fuzzer1.test b/test/fuzzer1.test index 4ee5730807..5e361e9df0 100644 --- a/test/fuzzer1.test +++ b/test/fuzzer1.test @@ -23,7 +23,6 @@ ifcapable !vtab { } set ::testprefix fuzzer1 - load_static_extension db fuzzer # Check configuration errors. @@ -1648,51 +1647,6 @@ do_catchsql_test 5.5.4 { CREATE VIRTUAL TABLE x USING fuzzer('fuzzer [x] rules table'); } {1 {fuzzer: ruleset must be between 0 and 2147483647}} -#------------------------------------------------------------------------- -# This test uses a fuzzer table with many rules. There is one rule to -# map each possible two character string, where characters are lower-case -# letters used in the English language, to all other possible two character -# strings. In total, (26^4)-(26^2) mappings (the subtracted term represents -# the no-op mappings discarded automatically by the fuzzer). -# -# -do_execsql_test 6.1.1 { - DROP TABLE IF EXISTS x1; - DROP TABLE IF EXISTS x1_rules; - CREATE TABLE x1_rules(ruleset, cFrom, cTo, cost); -} -puts "This test is slow - perhaps around 7 seconds on an average pc" -do_test 6.1.2 { - set LETTERS {a b c d e f g h i j k l m n o p q r s t u v w x y z} - set cost 1 - db transaction { - foreach c1 $LETTERS { - foreach c2 $LETTERS { - foreach c3 $LETTERS { - foreach c4 $LETTERS { - db eval {INSERT INTO x1_rules VALUES(0, $c1||$c2, $c3||$c4, $cost)} - set cost [expr ($cost%1000) + 1] - } - } - } - } - db eval {UPDATE x1_rules SET cost = 20 WHERE cost<20 AND cFrom!='xx'} - } -} {} - -do_execsql_test 6.2 { - SELECT count(*) FROM x1_rules WHERE cTo!=cFrom; -} [expr 26*26*26*26 - 26*26] - -do_execsql_test 6.2.1 { - CREATE VIRTUAL TABLE x1 USING fuzzer(x1_rules); - SELECT word FROM x1 WHERE word MATCH 'xx' LIMIT 10; -} {xx hw hx hy hz ia ib ic id ie} -do_execsql_test 6.2.2 { - SELECT cTo FROM x1_rules WHERE cFrom='xx' - ORDER BY cost asc, rowid asc LIMIT 9; -} {hw hx hy hz ia ib ic id ie} - #------------------------------------------------------------------------- # Test using different types of quotes with CREATE VIRTUAL TABLE # arguments. diff --git a/test/fuzzer2.test b/test/fuzzer2.test new file mode 100644 index 0000000000..44ee9e312c --- /dev/null +++ b/test/fuzzer2.test @@ -0,0 +1,72 @@ +# 2016 February 4 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# The focus of the tests is the word-fuzzer virtual table. The tests +# in this file are slower than those in fuzzer1.test. So this file does +# not run as part of veryquick.test etc. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +ifcapable !vtab { + finish_test + return +} + +set ::testprefix fuzzer2 +load_static_extension db fuzzer + +#------------------------------------------------------------------------- +# This test uses a fuzzer table with many rules. There is one rule to +# map each possible two character string, where characters are lower-case +# letters used in the English language, to all other possible two character +# strings. In total, (26^4)-(26^2) mappings (the subtracted term represents +# the no-op mappings discarded automatically by the fuzzer). +# +# +do_execsql_test 1.1.1 { + DROP TABLE IF EXISTS x1; + DROP TABLE IF EXISTS x1_rules; + CREATE TABLE x1_rules(ruleset, cFrom, cTo, cost); +} +puts "This test is slow - perhaps around 7 seconds on an average pc" +do_test 1.1.2 { + set LETTERS {a b c d e f g h i j k l m n o p q r s t u v w x y z} + set cost 1 + db transaction { + foreach c1 $LETTERS { + foreach c2 $LETTERS { + foreach c3 $LETTERS { + foreach c4 $LETTERS { + db eval {INSERT INTO x1_rules VALUES(0, $c1||$c2, $c3||$c4, $cost)} + set cost [expr ($cost%1000) + 1] + } + } + } + } + db eval {UPDATE x1_rules SET cost = 20 WHERE cost<20 AND cFrom!='xx'} + } +} {} + +do_execsql_test 1.2 { + SELECT count(*) FROM x1_rules WHERE cTo!=cFrom; +} [expr 26*26*26*26 - 26*26] + +do_execsql_test 1.2.1 { + CREATE VIRTUAL TABLE x1 USING fuzzer(x1_rules); + SELECT word FROM x1 WHERE word MATCH 'xx' LIMIT 10; +} {xx hw hx hy hz ia ib ic id ie} +do_execsql_test 1.2.2 { + SELECT cTo FROM x1_rules WHERE cFrom='xx' + ORDER BY cost asc, rowid asc LIMIT 9; +} {hw hx hy hz ia ib ic id ie} + +finish_test diff --git a/test/json101.test b/test/json101.test index 8c62c8b16c..9b780a379b 100644 --- a/test/json101.test +++ b/test/json101.test @@ -342,4 +342,19 @@ foreach {tn isvalid ws} { $isvalid } +# Ticket https://www.sqlite.org/src/info/ad2559db380abf8e +# Control characters must be escaped in JSON strings. +# +do_execsql_test json-8.1 { + DROP TABLE IF EXISTS t8; + CREATE TABLE t8(a,b); + INSERT INTO t8(a) VALUES('abc' || char(1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35) || 'xyz'); + UPDATE t8 SET b=json_array(a); + SELECT b FROM t8; +} {{["abc\u0001\u0002\u0003\u0004\u0005\u0006\u0007\b\t\n\u000b\f\r\u000e\u000f\u0010\u0011\u0012\u0013\u0014\u0015\u0016\u0017\u0018\u0019\u001a\u001b\u001c\u001d\u001e\u001f !\"#xyz"]}} +do_execsql_test json-8.2 { + SELECT a=json_extract(b,'$[0]') FROM t8; +} {1} + + finish_test diff --git a/test/mmap1.test b/test/mmap1.test index 18aec9f8fa..199a058754 100644 --- a/test/mmap1.test +++ b/test/mmap1.test @@ -28,6 +28,10 @@ proc nRead {db} { return $stats(read) } +# Return a Tcl script that registers a user-defined scalar function +# named rblob() with database handle $dbname. The function returns a +# sequence of pseudo-random blobs based on seed value $seed. +# proc register_rblob_code {dbname seed} { return [subst -nocommands { set ::rcnt $seed @@ -40,6 +44,7 @@ proc register_rblob_code {dbname seed} { }] } + # For cases 1.1 and 1.4, the number of pages read using xRead() is 4 on # unix and 9 on windows. The difference is that windows only ever maps # an integer number of OS pages (i.e. creates mappings that are a multiple @@ -269,65 +274,5 @@ do_test 5.5 { sqlite3_finalize $::STMT } SQLITE_OK -#------------------------------------------------------------------------- -# Test various mmap_size settings. -# -foreach {tn1 mmap1 mmap2} { - 1 6144 167773 - 2 18432 140399 - 3 43008 401302 - 4 92160 253899 - 5 190464 2 - 6 387072 752431 - 7 780288 291143 - 8 1566720 594306 - 9 3139584 829137 - 10 6285312 793963 - 11 12576768 1015590 -} { - do_multiclient_test tn { - sql1 { - CREATE TABLE t1(a PRIMARY KEY); - CREATE TABLE t2(x); - INSERT INTO t2 VALUES(''); - } - - code1 [register_rblob_code db 0] - code2 [register_rblob_code db2 444] - - sql1 "PRAGMA mmap_size = $mmap1" - sql2 "PRAGMA mmap_size = $mmap2" - - do_test $tn1.$tn { - for {set i 1} {$i <= 100} {incr i} { - if {$i % 2} { - set c1 sql1 - set c2 sql2 - } else { - set c1 sql2 - set c2 sql1 - } - - $c1 { - INSERT INTO t1 VALUES( rblob(5000) ); - UPDATE t2 SET x = (SELECT md5sum(a) FROM t1); - } - - set res [$c2 { - SELECT count(*) FROM t1; - SELECT x == (SELECT md5sum(a) FROM t1) FROM t2; - PRAGMA integrity_check; - }] - if {$res != [list $i 1 ok]} { - do_test $tn1.$tn.$i { - set ::res - } [list $i 1 ok] - } - } - set res 1 - } {1} - } -} - finish_test diff --git a/test/mmap4.test b/test/mmap4.test new file mode 100644 index 0000000000..97015489cc --- /dev/null +++ b/test/mmap4.test @@ -0,0 +1,104 @@ +# 2016 February 04 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file tests the effect of the mmap() or mremap() system calls +# returning an error on the library. +# +# If either mmap() or mremap() fails, SQLite should log an error +# message, then continue accessing the database using read() and +# write() exclusively. +# +set testdir [file dirname $argv0] +source $testdir/tester.tcl +ifcapable !mmap { + finish_test + return +} +source $testdir/lock_common.tcl +set testprefix mmap4 + +# Return a Tcl script that registers a user-defined scalar function +# named rblob() with database handle $dbname. The function returns a +# sequence of pseudo-random blobs based on seed value $seed. +# +proc register_rblob_code {dbname seed} { + return [subst -nocommands { + set ::rcnt $seed + proc rblob {n} { + set ::rcnt [expr (([set ::rcnt] << 3) + [set ::rcnt] + 456) & 0xFFFFFFFF] + set str [format %.8x [expr [set ::rcnt] ^ 0xbdf20da3]] + string range [string repeat [set str] [expr [set n]/4]] 1 [set n] + } + $dbname func rblob rblob + }] +} + +#------------------------------------------------------------------------- +# Test various mmap_size settings. +# +foreach {tn1 mmap1 mmap2} { + 1 6144 167773 + 2 18432 140399 + 3 43008 401302 + 4 92160 253899 + 5 190464 2 + 6 387072 752431 + 7 780288 291143 + 8 1566720 594306 + 9 3139584 829137 + 10 6285312 793963 + 11 12576768 1015590 +} { + do_multiclient_test tn { + sql1 { + CREATE TABLE t1(a PRIMARY KEY); + CREATE TABLE t2(x); + INSERT INTO t2 VALUES(''); + } + + code1 [register_rblob_code db 0] + code2 [register_rblob_code db2 444] + + sql1 "PRAGMA mmap_size = $mmap1" + sql2 "PRAGMA mmap_size = $mmap2" + + do_test $tn1.$tn { + for {set i 1} {$i <= 100} {incr i} { + if {$i % 2} { + set c1 sql1 + set c2 sql2 + } else { + set c1 sql2 + set c2 sql1 + } + + $c1 { + INSERT INTO t1 VALUES( rblob(5000) ); + UPDATE t2 SET x = (SELECT md5sum(a) FROM t1); + } + + set res [$c2 { + SELECT count(*) FROM t1; + SELECT x == (SELECT md5sum(a) FROM t1) FROM t2; + PRAGMA integrity_check; + }] + if {$res != [list $i 1 ok]} { + do_test $tn1.$tn.$i { + set ::res + } [list $i 1 ok] + } + } + set res 1 + } {1} + } +} + +finish_test diff --git a/test/permutations.test b/test/permutations.test index 5e7b00eff1..f6c0399d85 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -117,7 +117,14 @@ set allquicktests [test_set $alltests -exclude { vtab_err.test walslow.test walcrash.test walcrash3.test walthread.test rtree3.test indexfault.test securedel2.test sort3.test sort4.test fts4growth.test fts4growth2.test - bigsort.test rbu.test + bigsort.test rbu.test walprotocol.test mmap4.test fuzzer2.test + walcrash2.test e_fkey.test backup.test + + fts4merge.test fts4merge2.test fts4merge4.test fts4check.test + fts3cov.test fts3snippet.test fts3corrupt2.test fts3an.test + fts3defer.test fts4langid.test fts3sort.test fts5unicode.test + + rtree4.test }] if {[info exists ::env(QUICKTEST_INCLUDE)]} { set allquicktests [concat $allquicktests $::env(QUICKTEST_INCLUDE)] @@ -154,7 +161,7 @@ test_suite "veryquick" -prefix "" -description { This test suite is the same as the "quick" tests, except that some files that test malloc and IO errors are omitted. } -files [ - test_set $allquicktests -exclude *malloc* *ioerr* *fault* *bigfile* + test_set $allquicktests -exclude *malloc* *ioerr* *fault* *bigfile* *_err* ] test_suite "extraquick" -prefix "" -description { @@ -162,7 +169,7 @@ test_suite "extraquick" -prefix "" -description { This test suite is the same as the "veryquick" tests, except that slower tests are omitted. } -files [ - test_set $allquicktests -exclude *malloc* *ioerr* *fault* *bigfile* \ + test_set $allquicktests -exclude *malloc* *ioerr* *fault* *bigfile* *_err* \ wal3.test fts4merge* sort2.test mmap1.test walcrash* \ percentile.test where8m.test walcksum.test savepoint3.test \ fuzzer1.test fuzzer3.test fts3expr3.test @@ -180,7 +187,7 @@ test_suite "valgrind" -prefix "" -description { Run the "veryquick" test suite with a couple of multi-process tests (that fail under valgrind) omitted. } -files [ - test_set $allquicktests -exclude *malloc* *ioerr* *fault* wal.test \ + test_set $allquicktests -exclude *malloc* *ioerr* *fault* *_err* wal.test \ shell*.test crash8.test atof1.test selectG.test \ tkt-fc62af4523.test numindex1.test ] -initialize { @@ -193,7 +200,8 @@ test_suite "valgrind-nolookaside" -prefix "" -description { Run the "veryquick" test suite with a couple of multi-process tests (that fail under valgrind) omitted. } -files [ - test_set $allquicktests -exclude *malloc* *ioerr* *fault* wal.test atof1.test + test_set $allquicktests -exclude *malloc* *ioerr* *fault* *_err* \ + wal.test atof1.test ] -initialize { set ::G(valgrind) 1 catch {db close} @@ -270,7 +278,7 @@ test_suite "nofaultsim" -prefix "" -description { This test suite is the same as the "quick" tests, except that some files that test malloc and IO errors are omitted. } -files [ - test_set $allquicktests -exclude *malloc* *ioerr* *fault* + test_set $allquicktests -exclude *malloc* *ioerr* *fault* *_err* ] -initialize { catch {db close} sqlite3_shutdown diff --git a/test/pragma.test b/test/pragma.test index 246a7cd854..befa5cf548 100644 --- a/test/pragma.test +++ b/test/pragma.test @@ -186,7 +186,15 @@ do_test pragma-1.10 { PRAGMA synchronous; } } {123 123 1} -do_test pragma-1.11 { +do_test pragma-1.11.1 { + execsql { + PRAGMA synchronous=EXTRA; + PRAGMA cache_size; + PRAGMA default_cache_size; + PRAGMA synchronous; + } +} {123 123 3} +do_test pragma-1.11.2 { execsql { PRAGMA synchronous=FULL; PRAGMA cache_size; @@ -224,14 +232,20 @@ do_test pragma-1.14.1 { PRAGMA synchronous=4; PRAGMA synchronous; } -} {0} +} {4} do_test pragma-1.14.2 { execsql { PRAGMA synchronous=3; PRAGMA synchronous; } -} {0} +} {3} do_test pragma-1.14.3 { + execsql { + PRAGMA synchronous=8; + PRAGMA synchronous; + } +} {0} +do_test pragma-1.14.4 { execsql { PRAGMA synchronous=10; PRAGMA synchronous; diff --git a/test/savepoint3.test b/test/savepointfault.test similarity index 94% rename from test/savepoint3.test rename to test/savepointfault.test index 0dcb0aceba..0329f8caa2 100644 --- a/test/savepoint3.test +++ b/test/savepointfault.test @@ -9,14 +9,15 @@ # #*********************************************************************** # -# $Id: savepoint3.test,v 1.5 2009/06/05 17:09:12 drh Exp $ set testdir [file dirname $argv0] source $testdir/tester.tcl source $testdir/malloc_common.tcl -do_malloc_test savepoint3-1 -sqlprep { +set testprefix savepointfault + +do_malloc_test 1 -sqlprep { CREATE TABLE t1(a, b, c); INSERT INTO t1 VALUES(1, 2, 3); } -sqlbody { @@ -28,7 +29,7 @@ do_malloc_test savepoint3-1 -sqlprep { RELEASE one; } -do_malloc_test savepoint3-2 -sqlprep { +do_malloc_test 2 -sqlprep { PRAGMA cache_size = 10; CREATE TABLE t1(a, b, c); INSERT INTO t1 VALUES(randstr(400,400), randstr(400,400), randstr(400,400)); @@ -59,7 +60,7 @@ do_malloc_test savepoint3-2 -sqlprep { RELEASE one; } -do_ioerr_test savepoint3.3 -sqlprep { +do_ioerr_test 3 -sqlprep { CREATE TABLE t1(a, b, c); INSERT INTO t1 VALUES(1, randstr(1000,1000), randstr(1000,1000)); INSERT INTO t1 VALUES(2, randstr(1000,1000), randstr(1000,1000)); @@ -79,7 +80,7 @@ do_ioerr_test savepoint3.3 -sqlprep { # The following test does a really big savepoint rollback. One involving # more than 4000 pages. The idea is to get a specific sqlite3BitvecSet() # operation in pagerPlaybackSavepoint() to fail. -#do_malloc_test savepoint3-4 -sqlprep { +#do_malloc_test 4 -sqlprep { # BEGIN; # CREATE TABLE t1(a, b); # CREATE INDEX i1 ON t1(a); @@ -107,7 +108,7 @@ do_ioerr_test savepoint3.3 -sqlprep { # Cause a specific malloc in savepoint rollback code to fail. # -do_malloc_test savepoint3-4 -start 7 -sqlprep { +do_malloc_test 4 -start 7 -sqlprep { PRAGMA auto_vacuum = incremental; PRAGMA cache_size = 1000; diff --git a/test/sort.test b/test/sort.test index 21b11d6c88..d73ecea480 100644 --- a/test/sort.test +++ b/test/sort.test @@ -491,52 +491,6 @@ do_execsql_test sort-13.3 { SELECT a, b FROM t10 ORDER BY a; } [db eval {SELECT a, b FROM t10 ORDER BY a, b}] -#------------------------------------------------------------------------- -# Sort some large ( > 4KiB) records. -# -proc cksum {x} { - set i1 1 - set i2 2 - binary scan $x c* L - foreach {a b} $L { - set i1 [expr (($i2<<3) + $a) & 0x7FFFFFFF] - set i2 [expr (($i1<<3) + $b) & 0x7FFFFFFF] - } - list $i1 $i2 -} -db func cksum cksum - -do_execsql_test sort-14.0 { - PRAGMA cache_size = 5; - CREATE TABLE t11(a, b); - INSERT INTO t11 VALUES(randomblob(5000), NULL); - INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --2 - INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --3 - INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --4 - INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --5 - INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --6 - INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --7 - INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --8 - INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --9 - UPDATE t11 SET b = cksum(a); -} - -foreach {tn mmap_limit} { - 1 0 - 2 1000000 -} { - do_test sort-14.$tn { - sqlite3_test_control SQLITE_TESTCTRL_SORTER_MMAP db $mmap_limit - set prev "" - db eval { SELECT * FROM t11 ORDER BY b } { - if {$b != [cksum $a]} {error "checksum failed"} - if {[string compare $b $prev] < 0} {error "sort failed"} - set prev $b - } - set {} {} - } {} -} - #------------------------------------------------------------------------- # foreach {tn mmap_limit nWorker tmpstore coremutex fakeheap softheaplimit} { diff --git a/test/sort2.test b/test/sort2.test index 890025a5d0..f686654d53 100644 --- a/test/sort2.test +++ b/test/sort2.test @@ -31,7 +31,6 @@ foreach {tn script} { catch { db eval {PRAGMA threads=7} } } } { - eval $script do_execsql_test $tn.1 { @@ -67,17 +66,22 @@ foreach {tn script} { do_execsql_test $tn.2.4 { PRAGMA integrity_check } {ok} - do_execsql_test $tn.3 { - PRAGMA cache_size = 5; - WITH r(x,y) AS ( - SELECT 1, randomblob(100) - UNION ALL - SELECT x+1, randomblob(100) FROM r - LIMIT 1000000 - ) - SELECT count(x), length(y) FROM r GROUP BY (x%5) - } { - 200000 100 200000 100 200000 100 200000 100 200000 100 + # Because it uses so much data, this test can take 12-13 seconds even on + # a modern workstation. So it is omitted from "veryquick" and other + # permutations.test tests. + if {[isquick]==0} { + do_execsql_test $tn.3 { + PRAGMA cache_size = 5; + WITH r(x,y) AS ( + SELECT 1, randomblob(100) + UNION ALL + SELECT x+1, randomblob(100) FROM r + LIMIT 1000000 + ) + SELECT count(x), length(y) FROM r GROUP BY (x%5) + } { + 200000 100 200000 100 200000 100 200000 100 200000 100 + } } } diff --git a/test/sort3.test b/test/sort3.test index 80d8bbca3f..3aa6512a4b 100644 --- a/test/sort3.test +++ b/test/sort3.test @@ -19,6 +19,53 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl set testprefix sort3 +#------------------------------------------------------------------------- +# Sort some large ( > 4KiB) records. +# +proc cksum {x} { + set i1 1 + set i2 2 + binary scan $x c* L + foreach {a b} $L { + set i1 [expr (($i2<<3) + $a) & 0x7FFFFFFF] + set i2 [expr (($i1<<3) + $b) & 0x7FFFFFFF] + } + list $i1 $i2 +} +db func cksum cksum + +do_execsql_test 1.0 { + PRAGMA cache_size = 5; + CREATE TABLE t11(a, b); + INSERT INTO t11 VALUES(randomblob(5000), NULL); + INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --2 + INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --3 + INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --4 + INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --5 + INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --6 + INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --7 + INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --8 + INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --9 + UPDATE t11 SET b = cksum(a); +} + +foreach {tn mmap_limit} { + 1 0 + 2 1000000 +} { + do_test 1.$tn { + sqlite3_test_control SQLITE_TESTCTRL_SORTER_MMAP db $mmap_limit + set prev "" + db eval { SELECT * FROM t11 ORDER BY b } { + if {$b != [cksum $a]} {error "checksum failed"} + if {[string compare $b $prev] < 0} {error "sort failed"} + set prev $b + } + set {} {} + } {} +} + + # Sort roughly 20MB of data. Once with a mmap limit of 5MB and once without. # foreach {itest limit} { @@ -26,7 +73,7 @@ foreach {itest limit} { 2 0x7FFFFFFF } { sqlite3_test_control SQLITE_TESTCTRL_SORTER_MMAP db $limit - do_execsql_test 1.$itest { + do_execsql_test 2.$itest { WITH r(x,y) AS ( SELECT 1, randomblob(1000) UNION ALL @@ -46,7 +93,7 @@ foreach {itest limit} { # Sort more than 2GB of data. At one point this was causing a problem. # This test might take one minute or more to run. # -do_execsql_test 2 { +do_execsql_test 3 { PRAGMA cache_size = 20000; WITH r(x,y) AS ( SELECT 1, randomblob(1000) diff --git a/test/sync.test b/test/sync.test index 14252b5c08..2ee636a266 100644 --- a/test/sync.test +++ b/test/sync.test @@ -13,7 +13,6 @@ # This file implements tests to verify that fsync is disabled when # pragma synchronous=off even for multi-database commits. # -# $Id: sync.test,v 1.6 2007/10/09 08:29:33 danielk1977 Exp $ set testdir [file dirname $argv0] source $testdir/tester.tcl @@ -33,7 +32,7 @@ proc cond_incr_sync_count {adj} { global sqlite_sync_count if {$::tcl_platform(platform) == "windows"} { incr sqlite_sync_count $adj - } { + } else { ifcapable !dirsync { incr sqlite_sync_count $adj } @@ -64,9 +63,9 @@ ifcapable pager_pragmas { INSERT INTO t2 VALUES(3,4); COMMIT; } - cond_incr_sync_count 3 + cond_incr_sync_count 4 set sqlite_sync_count - } 8 + } 9 } do_test sync-1.3 { set sqlite_sync_count 0 @@ -78,9 +77,9 @@ do_test sync-1.3 { INSERT INTO t2 VALUES(5,6); COMMIT; } - cond_incr_sync_count 3 + cond_incr_sync_count 4 set sqlite_sync_count -} 10 +} 11 ifcapable pager_pragmas { do_test sync-1.4 { set sqlite_sync_count 0 diff --git a/test/tester.tcl b/test/tester.tcl index f520080192..720e15979f 100644 --- a/test/tester.tcl +++ b/test/tester.tcl @@ -1911,6 +1911,12 @@ proc presql {} { set presql } +proc isquick {} { + set ret 0 + catch {set ret $::G(isquick)} + set ret +} + #------------------------------------------------------------------------- # proc slave_test_script {script} { diff --git a/test/wal.test b/test/wal.test index abd3a3ce49..92b65e66ba 100644 --- a/test/wal.test +++ b/test/wal.test @@ -824,91 +824,6 @@ do_test wal-12.6 { db2 close db close -#------------------------------------------------------------------------- -# Test large log summaries. -# -# In this case "large" usually means a log file that requires a wal-index -# mapping larger than 64KB (the default initial allocation). A 64KB wal-index -# is large enough for a log file that contains approximately 13100 frames. -# So the following tests create logs containing at least this many frames. -# -# wal-13.1.*: This test case creates a very large log file within the -# file-system (around 200MB). The log file does not contain -# any valid frames. Test that the database file can still be -# opened and queried, and that the invalid log file causes no -# problems. -# -# wal-13.2.*: Test that a process may create a large log file and query -# the database (including the log file that it itself created). -# -# wal-13.3.*: Test that if a very large log file is created, and then a -# second connection is opened on the database file, it is possible -# to query the database (and the very large log) using the -# second connection. -# -# wal-13.4.*: Same test as wal-13.3.*. Except in this case the second -# connection is opened by an external process. -# -do_test wal-13.1.1 { - list [file exists test.db] [file exists test.db-wal] -} {1 0} -do_test wal-13.1.2 { - set fd [open test.db-wal w] - seek $fd [expr 200*1024*1024] - puts $fd "" - close $fd - sqlite3 db test.db - execsql { SELECT * FROM t2 } -} {B 2} -do_test wal-13.1.3 { - db close - file exists test.db-wal -} {0} - -do_test wal-13.2.1 { - sqlite3 db test.db - execsql { SELECT count(*) FROM t2 } -} {1} -do_test wal-13.2.2 { - db function blob blob - for {set i 0} {$i < 16} {incr i} { - execsql { INSERT INTO t2 SELECT blob(400), blob(400) FROM t2 } - } - execsql { SELECT count(*) FROM t2 } -} [expr int(pow(2, 16))] -do_test wal-13.2.3 { - expr [file size test.db-wal] > [wal_file_size 33000 1024] -} 1 - -do_multiclient_test tn { - incr tn 2 - - do_test wal-13.$tn.0 { - sql1 { - PRAGMA journal_mode = WAL; - CREATE TABLE t1(x); - INSERT INTO t1 SELECT randomblob(800); - } - sql1 { SELECT count(*) FROM t1 } - } {1} - - for {set ii 1} {$ii<16} {incr ii} { - do_test wal-13.$tn.$ii.a { - sql2 { INSERT INTO t1 SELECT randomblob(800) FROM t1 } - sql2 { SELECT count(*) FROM t1 } - } [expr (1<<$ii)] - do_test wal-13.$tn.$ii.b { - sql1 { SELECT count(*) FROM t1 } - } [expr (1<<$ii)] - do_test wal-13.$tn.$ii.c { - sql1 { SELECT count(*) FROM t1 } - } [expr (1<<$ii)] - do_test wal-13.$tn.$ii.d { - sql1 { PRAGMA integrity_check } - } {ok} - } -} - #------------------------------------------------------------------------- # Check a fun corruption case has been fixed. # diff --git a/test/wal3.test b/test/wal3.test index 9ee8a99458..da3d318773 100644 --- a/test/wal3.test +++ b/test/wal3.test @@ -61,7 +61,7 @@ do_test wal3-1.0 { PRAGMA cache_size = 10; } set x [wal_frame_count test.db-wal 1024] - if {$::G(perm:name)=="memsubsys1"} { + if {[permutation]=="memsubsys1"} { if {$x==4251 || $x==4290} {set x 4056} } set x @@ -238,67 +238,6 @@ foreach {tn syncmode synccount} { T delete } -#------------------------------------------------------------------------- -# When recovering the contents of a WAL file, a process obtains the WRITER -# lock, then locks all other bytes before commencing recovery. If it fails -# to lock all other bytes (because some other process is holding a read -# lock) it should retry up to 100 times. Then return SQLITE_PROTOCOL to the -# caller. Test this (test case wal3-4.3). -# -# Also test the effect of hitting an SQLITE_BUSY while attempting to obtain -# the WRITER lock (should be the same). Test case wal3-4.4. -# -proc lock_callback {method filename handle lock} { - lappend ::locks $lock -} -do_test wal3-4.1 { - testvfs T - T filter xShmLock - T script lock_callback - set ::locks [list] - sqlite3 db test.db -vfs T - execsql { SELECT * FROM x } - lrange $::locks 0 3 -} [list {0 1 lock exclusive} {1 7 lock exclusive} \ - {1 7 unlock exclusive} {0 1 unlock exclusive} \ -] -do_test wal3-4.2 { - db close - set ::locks [list] - sqlite3 db test.db -vfs T - execsql { SELECT * FROM x } - lrange $::locks 0 3 -} [list {0 1 lock exclusive} {1 7 lock exclusive} \ - {1 7 unlock exclusive} {0 1 unlock exclusive} \ -] -proc lock_callback {method filename handle lock} { - if {$lock == "1 7 lock exclusive"} { return SQLITE_BUSY } - return SQLITE_OK -} -puts "# Warning: This next test case causes SQLite to call xSleep(1) 100 times." -puts "# Normally this equates to a 100ms delay, but if SQLite is built on unix" -puts "# without HAVE_USLEEP defined, it may be 100 seconds." -do_test wal3-4.3 { - db close - set ::locks [list] - sqlite3 db test.db -vfs T - catchsql { SELECT * FROM x } -} {1 {locking protocol}} - -puts "# Warning: Same again!" -proc lock_callback {method filename handle lock} { - if {$lock == "0 1 lock exclusive"} { return SQLITE_BUSY } - return SQLITE_OK -} -do_test wal3-4.4 { - db close - set ::locks [list] - sqlite3 db test.db -vfs T - catchsql { SELECT * FROM x } -} {1 {locking protocol}} -db close -T delete - #------------------------------------------------------------------------- # Only one client may run recovery at a time. Test this mechanism. @@ -617,86 +556,6 @@ db close db2 close T delete -#------------------------------------------------------------------------- -# -do_test wal3-8.1 { - forcedelete test.db test.db-journal test.db wal - sqlite3 db test.db - sqlite3 db2 test.db - execsql { - PRAGMA auto_vacuum = off; - PRAGMA journal_mode = WAL; - CREATE TABLE b(c); - INSERT INTO b VALUES('Tehran'); - INSERT INTO b VALUES('Qom'); - INSERT INTO b VALUES('Markazi'); - PRAGMA wal_checkpoint; - } -} {wal 0 5 5} -do_test wal3-8.2 { - execsql { SELECT * FROM b } -} {Tehran Qom Markazi} -do_test wal3-8.3 { - db eval { SELECT * FROM b } { - db eval { INSERT INTO b VALUES('Qazvin') } - set r [db2 eval { SELECT * FROM b }] - break - } - set r -} {Tehran Qom Markazi Qazvin} -do_test wal3-8.4 { - execsql { - INSERT INTO b VALUES('Gilan'); - INSERT INTO b VALUES('Ardabil'); - } -} {} -db2 close - -faultsim_save_and_close -testvfs T -default 1 -faultsim_restore_and_reopen -T filter xShmLock -T script lock_callback - -proc lock_callback {method file handle spec} { - if {$spec == "1 7 unlock exclusive"} { - T filter {} - set ::r [catchsql { SELECT * FROM b } db2] - } -} -sqlite3 db test.db -sqlite3 db2 test.db -do_test wal3-8.5 { - execsql { SELECT * FROM b } -} {Tehran Qom Markazi Qazvin Gilan Ardabil} -do_test wal3-8.6 { - set ::r -} {1 {locking protocol}} - -db close -db2 close - -faultsim_restore_and_reopen -sqlite3 db2 test.db -T filter xShmLock -T script lock_callback -proc lock_callback {method file handle spec} { - if {$spec == "1 7 unlock exclusive"} { - T filter {} - set ::r [catchsql { SELECT * FROM b } db2] - } -} -unset ::r -do_test wal3-8.5 { - execsql { SELECT * FROM b } -} {Tehran Qom Markazi Qazvin Gilan Ardabil} -do_test wal3-8.6 { - set ::r -} {1 {locking protocol}} - -db close -db2 close -T delete #------------------------------------------------------------------------- # When a connection opens a read-lock on the database, it searches for diff --git a/test/walcksum.test b/test/walcksum.test index 3005a758ac..f3fc427115 100644 --- a/test/walcksum.test +++ b/test/walcksum.test @@ -334,59 +334,5 @@ do_test walcksum-2.1 { catch { db close } catch { db2 close } -#------------------------------------------------------------------------- -# Test case walcksum-3.* tests that the checksum calculation detects single -# byte changes to frame or frame-header data and considers the frame -# invalid as a result. -# -do_test walcksum-3.1 { - forcedelete test.db test.db-wal test.db-journal - sqlite3 db test.db - - execsql { - PRAGMA synchronous = NORMAL; - PRAGMA page_size = 1024; - CREATE TABLE t1(a, b); - INSERT INTO t1 VALUES(1, randomblob(300)); - INSERT INTO t1 VALUES(2, randomblob(300)); - PRAGMA journal_mode = WAL; - INSERT INTO t1 VALUES(3, randomblob(300)); - } - - file size test.db-wal -} [wal_file_size 1 1024] -do_test walcksum-3.2 { - forcecopy test.db-wal test2.db-wal - forcecopy test.db test2.db - sqlite3 db2 test2.db - execsql { SELECT a FROM t1 } db2 -} {1 2 3} -db2 close -forcecopy test.db test2.db - - -foreach incr {1 2 3 20 40 60 80 100 120 140 160 180 200 220 240 253 254 255} { - do_test walcksum-3.3.$incr { - set FAIL 0 - for {set iOff 0} {$iOff < [wal_file_size 1 1024]} {incr iOff} { - - forcecopy test.db-wal test2.db-wal - set fd [open test2.db-wal r+] - fconfigure $fd -encoding binary - fconfigure $fd -translation binary - - seek $fd $iOff - binary scan [read $fd 1] c x - seek $fd $iOff - puts -nonewline $fd [binary format c [expr {($x+$incr)&0xFF}]] - close $fd - - sqlite3 db2 test2.db - if { [execsql { SELECT a FROM t1 } db2] != "1 2" } {set FAIL 1} - db2 close - } - set FAIL - } {0} -} finish_test diff --git a/test/walprotocol.test b/test/walprotocol.test new file mode 100644 index 0000000000..d658de9d7c --- /dev/null +++ b/test/walprotocol.test @@ -0,0 +1,177 @@ +# 2016 February 4 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the operation of the library in +# "PRAGMA journal_mode=WAL" mode. +# +# More specifically, it tests "locking protocol" errors - errors that +# may be caused if one or more SQLite clients does not follow the expected +# locking protocol when accessing a wal-mode database. These tests take +# quite a while to run. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +source $testdir/wal_common.tcl +ifcapable !wal {finish_test ; return } + +set testprefix walprotocol + +#------------------------------------------------------------------------- +# When recovering the contents of a WAL file, a process obtains the WRITER +# lock, then locks all other bytes before commencing recovery. If it fails +# to lock all other bytes (because some other process is holding a read +# lock) it should retry up to 100 times. Then return SQLITE_PROTOCOL to the +# caller. Test this (test case 1.3). +# +# Also test the effect of hitting an SQLITE_BUSY while attempting to obtain +# the WRITER lock (should be the same). Test case 1.4. +# +do_execsql_test 1.0 { + PRAGMA journal_mode = wal; + CREATE TABLE x(y); + INSERT INTO x VALUES('z'); +} {wal} + +proc lock_callback {method filename handle lock} { + lappend ::locks $lock +} +do_test 1.1 { + testvfs T + T filter xShmLock + T script lock_callback + set ::locks [list] + sqlite3 db test.db -vfs T + execsql { SELECT * FROM x } + lrange $::locks 0 3 +} [list {0 1 lock exclusive} {1 7 lock exclusive} \ + {1 7 unlock exclusive} {0 1 unlock exclusive} \ +] +do_test 1.2 { + db close + set ::locks [list] + sqlite3 db test.db -vfs T + execsql { SELECT * FROM x } + lrange $::locks 0 3 +} [list {0 1 lock exclusive} {1 7 lock exclusive} \ + {1 7 unlock exclusive} {0 1 unlock exclusive} \ +] +proc lock_callback {method filename handle lock} { + if {$lock == "1 7 lock exclusive"} { return SQLITE_BUSY } + return SQLITE_OK +} +puts "# Warning: This next test case causes SQLite to call xSlee(1) 100 times." +puts "# Normally this equates to a delay of roughly 10 seconds, but if SQLite" +puts "# is built on unix without HAVE_USLEEP defined, it may be much longer." +do_test 1.3 { + db close + set ::locks [list] + sqlite3 db test.db -vfs T + catchsql { SELECT * FROM x } +} {1 {locking protocol}} + +puts "# Warning: Same again!" +proc lock_callback {method filename handle lock} { + if {$lock == "0 1 lock exclusive"} { return SQLITE_BUSY } + return SQLITE_OK +} +do_test 1.4 { + db close + set ::locks [list] + sqlite3 db test.db -vfs T + catchsql { SELECT * FROM x } +} {1 {locking protocol}} +db close +T delete + +#------------------------------------------------------------------------- +# +do_test 2.1 { + forcedelete test.db test.db-journal test.db wal + sqlite3 db test.db + sqlite3 db2 test.db + execsql { + PRAGMA auto_vacuum = off; + PRAGMA journal_mode = WAL; + CREATE TABLE b(c); + INSERT INTO b VALUES('Tehran'); + INSERT INTO b VALUES('Qom'); + INSERT INTO b VALUES('Markazi'); + PRAGMA wal_checkpoint; + } +} {wal 0 5 5} +do_test 2.2 { + execsql { SELECT * FROM b } +} {Tehran Qom Markazi} +do_test 2.3 { + db eval { SELECT * FROM b } { + db eval { INSERT INTO b VALUES('Qazvin') } + set r [db2 eval { SELECT * FROM b }] + break + } + set r +} {Tehran Qom Markazi Qazvin} +do_test 2.4 { + execsql { + INSERT INTO b VALUES('Gilan'); + INSERT INTO b VALUES('Ardabil'); + } +} {} +db2 close + +faultsim_save_and_close +testvfs T -default 1 +faultsim_restore_and_reopen +T filter xShmLock +T script lock_callback + +proc lock_callback {method file handle spec} { + if {$spec == "1 7 unlock exclusive"} { + T filter {} + set ::r [catchsql { SELECT * FROM b } db2] + } +} +sqlite3 db test.db +sqlite3 db2 test.db +do_test 2.5 { + execsql { SELECT * FROM b } +} {Tehran Qom Markazi Qazvin Gilan Ardabil} +do_test 2.6 { + set ::r +} {1 {locking protocol}} + +db close +db2 close + +faultsim_restore_and_reopen +sqlite3 db2 test.db +T filter xShmLock +T script lock_callback +proc lock_callback {method file handle spec} { + if {$spec == "1 7 unlock exclusive"} { + T filter {} + set ::r [catchsql { SELECT * FROM b } db2] + } +} +unset ::r +do_test 2.7 { + execsql { SELECT * FROM b } +} {Tehran Qom Markazi Qazvin Gilan Ardabil} +do_test 2.8 { + set ::r +} {1 {locking protocol}} + +db close +db2 close +T delete + +finish_test diff --git a/test/walslow.test b/test/walslow.test index 83f292281a..2a52a225d8 100644 --- a/test/walslow.test +++ b/test/walslow.test @@ -16,9 +16,13 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +source $testdir/wal_common.tcl +source $testdir/lock_common.tcl ifcapable !wal {finish_test ; return } +set testprefix walslow + proc reopen_db {} { catch { db close } forcedelete test.db test.db-wal @@ -69,5 +73,159 @@ for {set seed 1} {$seed<10} {incr seed} { } } +#------------------------------------------------------------------------- +# Test case walslow-3.* tests that the checksum calculation detects single +# byte changes to frame or frame-header data and considers the frame +# invalid as a result. +# +reset_db +do_test 3.1 { + + execsql { + PRAGMA synchronous = NORMAL; + PRAGMA page_size = 1024; + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, randomblob(300)); + INSERT INTO t1 VALUES(2, randomblob(300)); + PRAGMA journal_mode = WAL; + INSERT INTO t1 VALUES(3, randomblob(300)); + } + + file size test.db-wal +} [wal_file_size 1 1024] +do_test 3.2 { + forcecopy test.db-wal test2.db-wal + forcecopy test.db test2.db + sqlite3 db2 test2.db + execsql { SELECT a FROM t1 } db2 +} {1 2 3} +db2 close +forcecopy test.db test2.db + +foreach incr {1 2 3 20 40 60 80 100 120 140 160 180 200 220 240 253 254 255} { + do_test 3.3.$incr { + set FAIL 0 + for {set iOff 0} {$iOff < [wal_file_size 1 1024]} {incr iOff} { + + forcecopy test.db-wal test2.db-wal + set fd [open test2.db-wal r+] + fconfigure $fd -encoding binary + fconfigure $fd -translation binary + + seek $fd $iOff + binary scan [read $fd 1] c x + seek $fd $iOff + puts -nonewline $fd [binary format c [expr {($x+$incr)&0xFF}]] + close $fd + + sqlite3 db2 test2.db + if { [execsql { SELECT a FROM t1 } db2] != "1 2" } {set FAIL 1} + db2 close + } + set FAIL + } {0} +} + + +#------------------------------------------------------------------------- +# Test large log summaries. +# +# In this case "large" usually means a log file that requires a wal-index +# mapping larger than 64KB (the default initial allocation). A 64KB wal-index +# is large enough for a log file that contains approximately 13100 frames. +# So the following tests create logs containing at least this many frames. +# +# 4.1.*: This test case creates a very large log file within the +# file-system (around 200MB). The log file does not contain +# any valid frames. Test that the database file can still be +# opened and queried, and that the invalid log file causes no +# problems. +# +# 4.2.*: Test that a process may create a large log file and query +# the database (including the log file that it itself created). +# +# 4.3.*: Test that if a very large log file is created, and then a +# second connection is opened on the database file, it is possible +# to query the database (and the very large log) using the +# second connection. +# +# 4.4.*: Same test as wal-13.3.*. Except in this case the second +# connection is opened by an external process. +# +set ::blobcnt 0 +proc blob {nByte} { + incr ::blobcnt + return [string range [string repeat "${::blobcnt}x" $nByte] 1 $nByte] +} + +reset_db +do_execsql_test 4.1 { + PRAGMA journal_mode = wal; + CREATE TABLE t1(x, y); + INSERT INTO "t1" VALUES('A',0); + CREATE TABLE t2(x, y); + INSERT INTO "t2" VALUES('B',2); +} {wal} +db close + +do_test 4.1.1 { + list [file exists test.db] [file exists test.db-wal] +} {1 0} +do_test 4.1.2 { + set fd [open test.db-wal w] + seek $fd [expr 200*1024*1024] + puts $fd "" + close $fd + sqlite3 db test.db + execsql { SELECT * FROM t2 } +} {B 2} +do_test 4.1.3 { + db close + file exists test.db-wal +} {0} + +do_test 4.2.1 { + sqlite3 db test.db + execsql { SELECT count(*) FROM t2 } +} {1} +do_test 4.2.2 { + db function blob blob + for {set i 0} {$i < 16} {incr i} { + execsql { INSERT INTO t2 SELECT blob(400), blob(400) FROM t2 } + } + execsql { SELECT count(*) FROM t2 } +} [expr int(pow(2, 16))] +do_test 4.2.3 { + expr [file size test.db-wal] > [wal_file_size 33000 1024] +} 1 + +do_multiclient_test tn { + incr tn 2 + + do_test 4.$tn.0 { + sql1 { + PRAGMA journal_mode = WAL; + CREATE TABLE t1(x); + INSERT INTO t1 SELECT randomblob(800); + } + sql1 { SELECT count(*) FROM t1 } + } {1} + + for {set ii 1} {$ii<16} {incr ii} { + do_test 4.$tn.$ii.a { + sql2 { INSERT INTO t1 SELECT randomblob(800) FROM t1 } + sql2 { SELECT count(*) FROM t1 } + } [expr (1<<$ii)] + do_test 4.$tn.$ii.b { + sql1 { SELECT count(*) FROM t1 } + } [expr (1<<$ii)] + do_test 4.$tn.$ii.c { + sql1 { SELECT count(*) FROM t1 } + } [expr (1<<$ii)] + do_test 4.$tn.$ii.d { + sql1 { PRAGMA integrity_check } + } {ok} + } +} finish_test diff --git a/test/where8m.test b/test/wherefault.test similarity index 84% rename from test/where8m.test rename to test/wherefault.test index 4fce3db7c3..60330a92dd 100644 --- a/test/where8m.test +++ b/test/wherefault.test @@ -9,17 +9,19 @@ # #*********************************************************************** # This file implements regression tests for SQLite library. The focus -# is testing of where.c. More specifically, the focus is the optimization -# of WHERE clauses that feature the OR operator. +# is testing of where.c. More specifically, the focus is on handling OOM +# errors within the code that optimizes WHERE clauses that feature the +# OR operator. # -# $Id: where8m.test,v 1.3 2009/06/05 17:09:12 drh Exp $ set testdir [file dirname $argv0] source $testdir/tester.tcl source $testdir/malloc_common.tcl -do_malloc_test where8m-1 -sqlprep { +set testprefix wherefault + +do_malloc_test 1 -sqlprep { CREATE TABLE t1(a, b, c); CREATE INDEX i1 ON t1(a); CREATE INDEX i2 ON t1(b); @@ -37,7 +39,7 @@ do_malloc_test where8m-1 -sqlprep { a BETWEEN 1 AND 3 AND b < 5 AND b > 2 AND c = 4; } -do_malloc_test where8m-2 -tclprep { +do_malloc_test 2 -tclprep { db eval { BEGIN; CREATE TABLE t1(a, b, c); diff --git a/tool/build-all-msvc.bat b/tool/build-all-msvc.bat index 2dd01004d5..84528e37c2 100755 --- a/tool/build-all-msvc.bat +++ b/tool/build-all-msvc.bat @@ -729,12 +729,16 @@ GOTO no_errors GOTO :EOF :fn_UnsetVariable + SETLOCAL SET VALUE=%1 IF DEFINED VALUE ( - SET %VALUE%= SET VALUE= - CALL :fn_ResetErrorLevel + ENDLOCAL + SET %VALUE%= + ) ELSE ( + ENDLOCAL ) + CALL :fn_ResetErrorLevel GOTO :EOF :fn_AppendVariable