diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 754f3b5cb7..41791cb3cb 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -468,6 +468,7 @@ static int fts3DisconnectMethod(sqlite3_vtab *pVtab){ sqlite3_free(p->zSegmentsTbl); sqlite3_free(p->zReadExprlist); sqlite3_free(p->zWriteExprlist); + sqlite3_free(p->zContentTbl); /* Invoke the tokenizer destructor to free the tokenizer. */ p->pTokenizer->pModule->xDestroy(p->pTokenizer); @@ -507,16 +508,19 @@ static void fts3DbExec( ** The xDestroy() virtual table method. */ static int fts3DestroyMethod(sqlite3_vtab *pVtab){ - int rc = SQLITE_OK; /* Return code */ Fts3Table *p = (Fts3Table *)pVtab; - sqlite3 *db = p->db; + int rc = SQLITE_OK; /* Return code */ + const char *zDb = p->zDb; /* Name of database (e.g. "main", "temp") */ + sqlite3 *db = p->db; /* Database handle */ /* Drop the shadow tables */ - fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_content'", p->zDb, p->zName); - fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segments'", p->zDb,p->zName); - fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segdir'", p->zDb, p->zName); - fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_docsize'", p->zDb, p->zName); - fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_stat'", p->zDb, p->zName); + if( p->zContentTbl==0 ){ + fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_content'", zDb, p->zName); + } + fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segments'", zDb,p->zName); + fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segdir'", zDb, p->zName); + fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_docsize'", zDb, p->zName); + fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_stat'", zDb, p->zName); /* If everything has worked, invoke fts3DisconnectMethod() to free the ** memory associated with the Fts3Table structure and return SQLITE_OK. @@ -578,23 +582,27 @@ static void fts3DeclareVtab(int *pRc, Fts3Table *p){ static int fts3CreateTables(Fts3Table *p){ int rc = SQLITE_OK; /* Return code */ int i; /* Iterator variable */ - char *zContentCols; /* Columns of %_content table */ sqlite3 *db = p->db; /* The database connection */ - /* Create a list of user columns for the content table */ - zContentCols = sqlite3_mprintf("docid INTEGER PRIMARY KEY"); - for(i=0; zContentCols && inColumn; i++){ - char *z = p->azColumn[i]; - zContentCols = sqlite3_mprintf("%z, 'c%d%q'", zContentCols, i, z); - } - if( zContentCols==0 ) rc = SQLITE_NOMEM; + if( p->zContentTbl==0 ){ + char *zContentCols; /* Columns of %_content table */ + + /* Create a list of user columns for the content table */ + zContentCols = sqlite3_mprintf("docid INTEGER PRIMARY KEY"); + for(i=0; zContentCols && inColumn; i++){ + char *z = p->azColumn[i]; + zContentCols = sqlite3_mprintf("%z, 'c%d%q'", zContentCols, i, z); + } + if( zContentCols==0 ) rc = SQLITE_NOMEM; + + /* Create the content table */ + fts3DbExec(&rc, db, + "CREATE TABLE %Q.'%q_content'(%s)", + p->zDb, p->zName, zContentCols + ); + sqlite3_free(zContentCols); + } - /* Create the content table */ - fts3DbExec(&rc, db, - "CREATE TABLE %Q.'%q_content'(%s)", - p->zDb, p->zName, zContentCols - ); - sqlite3_free(zContentCols); /* Create other tables */ fts3DbExec(&rc, db, "CREATE TABLE %Q.'%q_segments'(blockid INTEGER PRIMARY KEY, block BLOB);", @@ -745,8 +753,8 @@ static char *fts3QuoteId(char const *zInput){ } /* -** Return a list of comma separated SQL expressions that could be used -** in a SELECT statement such as the following: +** Return a list of comma separated SQL expressions and a FROM clause that +** could be used in a SELECT statement such as the following: ** ** SELECT FROM %_content AS x ... ** @@ -757,7 +765,7 @@ static char *fts3QuoteId(char const *zInput){ ** table has the three user-defined columns "a", "b", and "c", the following ** string is returned: ** -** "docid, unzip(x.'a'), unzip(x.'b'), unzip(x.'c')" +** "docid, unzip(x.'a'), unzip(x.'b'), unzip(x.'c') FROM %_content AS x" ** ** The pointer returned points to a buffer allocated by sqlite3_malloc(). It ** is the responsibility of the caller to eventually free it. @@ -773,16 +781,28 @@ static char *fts3ReadExprList(Fts3Table *p, const char *zFunc, int *pRc){ char *zFunction; int i; - if( !zFunc ){ - zFunction = ""; + if( p->zContentTbl==0 ){ + if( !zFunc ){ + zFunction = ""; + }else{ + zFree = zFunction = fts3QuoteId(zFunc); + } + fts3Appendf(pRc, &zRet, "docid"); + for(i=0; inColumn; i++){ + fts3Appendf(pRc, &zRet, ",%s(x.'c%d%q')", zFunction, i, p->azColumn[i]); + } + sqlite3_free(zFree); }else{ - zFree = zFunction = fts3QuoteId(zFunc); + fts3Appendf(pRc, &zRet, "rowid"); + for(i=0; inColumn; i++){ + fts3Appendf(pRc, &zRet, ", x.'%q'", p->azColumn[i]); + } } - fts3Appendf(pRc, &zRet, "docid"); - for(i=0; inColumn; i++){ - fts3Appendf(pRc, &zRet, ",%s(x.'c%d%q')", zFunction, i, p->azColumn[i]); - } - sqlite3_free(zFree); + fts3Appendf(pRc, &zRet, "FROM '%q'.'%q%s' AS x", + p->zDb, + (p->zContentTbl ? p->zContentTbl : p->zName), + (p->zContentTbl ? "" : "_content") + ); return zRet; } @@ -906,6 +926,91 @@ static int fts3PrefixParameter( return SQLITE_OK; } +/* +** This function is called when initializing an FTS4 table that uses the +** content=xxx option. It determines the number of and names of the columns +** of the new FTS4 table. +** +** The third argument passed to this function is the value passed to the +** config=xxx option (i.e. "xxx"). This function queries the database for +** a table of that name. If found, the output variables are populated +** as follows: +** +** *pnCol: Set to the number of columns table xxx has, +** +** *pnStr: Set to the total amount of space required to store a copy +** of each columns name, including the nul-terminator. +** +** *pazCol: Set to point to an array of *pnCol strings. Each string is +** the name of the corresponding column in table xxx. The array +** and its contents are allocated using a single allocation. It +** is the responsibility of the caller to free this allocation +** by eventually passing the *pazCol value to sqlite3_free(). +** +** If the table cannot be found, an error code is returned and the output +** variables are undefined. Or, if an OOM is encountered, SQLITE_NOMEM is +** returned (and the output variables are undefined). +*/ +static int fts3ContentColumns( + sqlite3 *db, /* Database handle */ + const char *zDb, /* Name of db (i.e. "main", "temp" etc.) */ + const char *zTbl, /* Name of content table */ + const char ***pazCol, /* OUT: Malloc'd array of column names */ + int *pnCol, /* OUT: Size of array *pazCol */ + int *pnStr /* OUT: Bytes of string content */ +){ + int rc = SQLITE_OK; /* Return code */ + char *zSql; /* "SELECT *" statement on zTbl */ + sqlite3_stmt *pStmt = 0; /* Compiled version of zSql */ + + zSql = sqlite3_mprintf("SELECT * FROM %Q.%Q", zDb, zTbl); + if( !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); + } + sqlite3_free(zSql); + + if( rc==SQLITE_OK ){ + const char **azCol; /* Output array */ + int nStr = 0; /* Size of all column names (incl. 0x00) */ + int nCol; /* Number of table columns */ + int i; /* Used to iterate through columns */ + + /* Loop through the returned columns. Set nStr to the number of bytes of + ** space required to store a copy of each column name, including the + ** nul-terminator byte. */ + nCol = sqlite3_column_count(pStmt); + for(i=0; i MATCHINFO */ - { "prefix", 6, 0 }, /* 1 -> PREFIX */ - { "compress", 8, 0 }, /* 2 -> COMPRESS */ - { "uncompress", 10, 0 }, /* 3 -> UNCOMPRESS */ - { "order", 5, 0 } /* 4 -> ORDER */ + { "matchinfo", 9 }, /* 0 -> MATCHINFO */ + { "prefix", 6 }, /* 1 -> PREFIX */ + { "compress", 8 }, /* 2 -> COMPRESS */ + { "uncompress", 10 }, /* 3 -> UNCOMPRESS */ + { "order", 5 }, /* 4 -> ORDER */ + { "content", 7 } /* 5 -> CONTENT */ }; int iOpt; @@ -1045,13 +1151,20 @@ static int fts3InitVtab( case 4: /* ORDER */ if( (strlen(zVal)!=3 || sqlite3_strnicmp(zVal, "asc", 3)) - && (strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "desc", 3)) + && (strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "desc", 4)) ){ *pzErr = sqlite3_mprintf("unrecognized order: %s", zVal); rc = SQLITE_ERROR; } bDescIdx = (zVal[0]=='d' || zVal[0]=='D'); break; + + default: /* CONTENT */ + assert( iOpt==5 ); + sqlite3_free(zUncompress); + zContent = zVal; + zVal = 0; + break; } } sqlite3_free(zVal); @@ -1064,6 +1177,26 @@ static int fts3InitVtab( aCol[nCol++] = z; } } + + /* If a content=xxx option was specified, the following: + ** + ** 1. Ignore any compress= and uncompress= options. + ** + ** 2. If no column names were specified as part of the CREATE VIRTUAL + ** TABLE statement, use all columns from the content table. + */ + if( rc==SQLITE_OK && zContent ){ + sqlite3_free(zCompress); + sqlite3_free(zUncompress); + zCompress = 0; + zUncompress = 0; + if( nCol==0 ){ + sqlite3_free(aCol); + aCol = 0; + rc = fts3ContentColumns(db, argv[1], zContent, &aCol, &nCol, &nString); + } + assert( rc!=SQLITE_OK || nCol>0 ); + } if( rc!=SQLITE_OK ) goto fts3_init_out; if( nCol==0 ){ @@ -1108,6 +1241,8 @@ static int fts3InitVtab( p->bHasDocsize = (isFts4 && bNoDocsize==0); p->bHasStat = isFts4; p->bDescIdx = bDescIdx; + p->zContentTbl = zContent; + zContent = 0; TESTONLY( p->inTransaction = -1 ); TESTONLY( p->mxSavepoint = -1 ); @@ -1169,6 +1304,7 @@ fts3_init_out: sqlite3_free(aIndex); sqlite3_free(zCompress); sqlite3_free(zUncompress); + sqlite3_free(zContent); sqlite3_free((void *)aCol); if( rc!=SQLITE_OK ){ if( p ){ @@ -1320,35 +1456,64 @@ static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){ return SQLITE_OK; } +/* +** If pCsr->pStmt has not been prepared (i.e. if pCsr->pStmt==0), then +** compose and prepare an SQL statement of the form: +** +** "SELECT FROM %_content WHERE rowid = ?" +** +** (or the equivalent for a content=xxx table) and set pCsr->pStmt to +** it. If an error occurs, return an SQLite error code. +** +** Otherwise, set *ppStmt to point to pCsr->pStmt and return SQLITE_OK. +*/ +static int fts3CursorSeekStmt(Fts3Cursor *pCsr, sqlite3_stmt **ppStmt){ + int rc = SQLITE_OK; + if( pCsr->pStmt==0 ){ + Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; + char *zSql; + zSql = sqlite3_mprintf("SELECT %s WHERE rowid = ?", p->zReadExprlist); + if( !zSql ) return SQLITE_NOMEM; + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); + sqlite3_free(zSql); + } + *ppStmt = pCsr->pStmt; + return rc; +} + /* ** Position the pCsr->pStmt statement so that it is on the row ** of the %_content table that contains the last match. Return ** SQLITE_OK on success. */ static int fts3CursorSeek(sqlite3_context *pContext, Fts3Cursor *pCsr){ + int rc = SQLITE_OK; if( pCsr->isRequireSeek ){ - sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId); - pCsr->isRequireSeek = 0; - if( SQLITE_ROW==sqlite3_step(pCsr->pStmt) ){ - return SQLITE_OK; - }else{ - int rc = sqlite3_reset(pCsr->pStmt); - if( rc==SQLITE_OK ){ - /* If no row was found and no error has occured, then the %_content - ** table is missing a row that is present in the full-text index. - ** The data structures are corrupt. - */ - rc = FTS_CORRUPT_VTAB; + sqlite3_stmt *pStmt = 0; + + rc = fts3CursorSeekStmt(pCsr, &pStmt); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId); + pCsr->isRequireSeek = 0; + if( SQLITE_ROW==sqlite3_step(pCsr->pStmt) ){ + return SQLITE_OK; + }else{ + rc = sqlite3_reset(pCsr->pStmt); + if( rc==SQLITE_OK && ((Fts3Table *)pCsr->base.pVtab)->zContentTbl==0 ){ + /* If no row was found and no error has occured, then the %_content + ** table is missing a row that is present in the full-text index. + ** The data structures are corrupt. */ + rc = FTS_CORRUPT_VTAB; + pCsr->isEof = 1; + } } - pCsr->isEof = 1; - if( pContext ){ - sqlite3_result_error_code(pContext, rc); - } - return rc; } - }else{ - return SQLITE_OK; } + + if( rc!=SQLITE_OK && pContext ){ + sqlite3_result_error_code(pContext, rc); + } + return rc; } /* @@ -1788,7 +1953,7 @@ static int fts3PoslistPhraseMerge( char **pp1, /* IN/OUT: Left input list */ char **pp2 /* IN/OUT: Right input list */ ){ - char *p = (pp ? *pp : 0); + char *p = *pp; char *p1 = *pp1; char *p2 = *pp2; int iCol1 = 0; @@ -1814,7 +1979,7 @@ static int fts3PoslistPhraseMerge( sqlite3_int64 iPos1 = 0; sqlite3_int64 iPos2 = 0; - if( pp && iCol1 ){ + if( iCol1 ){ *p++ = POS_COLUMN; p += sqlite3Fts3PutVarint(p, iCol1); } @@ -1829,13 +1994,6 @@ static int fts3PoslistPhraseMerge( || (isExact==0 && iPos2>iPos1 && iPos2<=iPos1+nToken) ){ sqlite3_int64 iSave; - if( !pp ){ - fts3PoslistCopy(0, &p2); - fts3PoslistCopy(0, &p1); - *pp1 = p1; - *pp2 = p2; - return 1; - } iSave = isSaveLeft ? iPos1 : iPos2; fts3PutDeltaVarint(&p, &iPrev, iSave+2); iPrev -= 2; pSave = 0; @@ -1887,7 +2045,7 @@ static int fts3PoslistPhraseMerge( fts3PoslistCopy(0, &p1); *pp1 = p1; *pp2 = p2; - if( !pp || *pp==p ){ + if( *pp==p ){ return 0; } *p++ = 0x00; @@ -2190,6 +2348,56 @@ static void fts3DoclistPhraseMerge( *pnRight = p - aOut; } +/* +** Argument pList points to a position list nList bytes in size. This +** function checks to see if the position list contains any entries for +** a token in position 0 (of any column). If so, it writes argument iDelta +** to the output buffer pOut, followed by a position list consisting only +** of the entries from pList at position 0, and terminated by an 0x00 byte. +** The value returned is the number of bytes written to pOut (if any). +*/ +int sqlite3Fts3FirstFilter( + sqlite3_int64 iDelta, /* Varint that may be written to pOut */ + char *pList, /* Position list (no 0x00 term) */ + int nList, /* Size of pList in bytes */ + char *pOut /* Write output here */ +){ + int nOut = 0; + int bWritten = 0; /* True once iDelta has been written */ + char *p = pList; + char *pEnd = &pList[nList]; + + if( *p!=0x01 ){ + if( *p==0x02 ){ + nOut += sqlite3Fts3PutVarint(&pOut[nOut], iDelta); + pOut[nOut++] = 0x02; + bWritten = 1; + } + fts3ColumnlistCopy(0, &p); + } + + while( pisPrefix ? FTS3_SEGMENT_PREFIX : 0) + | (pTok->bFirst ? FTS3_SEGMENT_FIRST : 0) | (iColumnnColumn ? FTS3_SEGMENT_COLUMN_FILTER : 0); filter.iCol = iColumn; filter.zTerm = pTok->z; @@ -2686,8 +2895,8 @@ static int fts3FilterMethod( return SQLITE_NOMEM; } - rc = sqlite3Fts3ExprParse(p->pTokenizer, p->azColumn, p->nColumn, - iCol, zQuery, -1, &pCsr->pExpr + rc = sqlite3Fts3ExprParse(p->pTokenizer, p->azColumn, p->bHasStat, + p->nColumn, iCol, zQuery, -1, &pCsr->pExpr ); if( rc!=SQLITE_OK ){ if( rc==SQLITE_ERROR ){ @@ -2714,23 +2923,24 @@ static int fts3FilterMethod( ** row by docid. */ if( idxNum==FTS3_FULLSCAN_SEARCH ){ - const char *zSort = (pCsr->bDesc ? "DESC" : "ASC"); - const char *zTmpl = "SELECT %s FROM %Q.'%q_content' AS x ORDER BY docid %s"; - zSql = sqlite3_mprintf(zTmpl, p->zReadExprlist, p->zDb, p->zName, zSort); - }else{ - const char *zTmpl = "SELECT %s FROM %Q.'%q_content' AS x WHERE docid = ?"; - zSql = sqlite3_mprintf(zTmpl, p->zReadExprlist, p->zDb, p->zName); + zSql = sqlite3_mprintf( + "SELECT %s ORDER BY rowid %s", + p->zReadExprlist, (pCsr->bDesc ? "DESC" : "ASC") + ); + if( zSql ){ + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); + sqlite3_free(zSql); + }else{ + rc = SQLITE_NOMEM; + } + }else if( idxNum==FTS3_DOCID_SEARCH ){ + rc = fts3CursorSeekStmt(pCsr, &pCsr->pStmt); + if( rc==SQLITE_OK ){ + rc = sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]); + } } - if( !zSql ) return SQLITE_NOMEM; - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); - sqlite3_free(zSql); if( rc!=SQLITE_OK ) return rc; - if( idxNum==FTS3_DOCID_SEARCH ){ - rc = sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]); - if( rc!=SQLITE_OK ) return rc; - } - return fts3NextMethod(pCursor); } @@ -2782,7 +2992,7 @@ static int fts3ColumnMethod( sqlite3_result_blob(pContext, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT); }else{ rc = fts3CursorSeek(0, pCsr); - if( rc==SQLITE_OK ){ + if( rc==SQLITE_OK && sqlite3_data_count(pCsr->pStmt)>(iCol+1) ){ sqlite3_result_value(pContext, sqlite3_column_value(pCsr->pStmt, iCol+1)); } } @@ -3075,15 +3285,22 @@ static int fts3RenameMethod( sqlite3 *db = p->db; /* Database connection */ int rc; /* Return Code */ + /* As it happens, the pending terms table is always empty here. This is + ** because an "ALTER TABLE RENAME TABLE" statement inside a transaction + ** always opens a savepoint transaction. And the xSavepoint() method + ** flushes the pending terms table. But leave the (no-op) call to + ** PendingTermsFlush() in in case that changes. + */ + assert( p->nPendingData==0 ); rc = sqlite3Fts3PendingTermsFlush(p); - if( rc!=SQLITE_OK ){ - return rc; + + if( p->zContentTbl==0 ){ + fts3DbExec(&rc, db, + "ALTER TABLE %Q.'%q_content' RENAME TO '%q_content';", + p->zDb, p->zName, zName + ); } - fts3DbExec(&rc, db, - "ALTER TABLE %Q.'%q_content' RENAME TO '%q_content';", - p->zDb, p->zName, zName - ); if( p->bHasDocsize ){ fts3DbExec(&rc, db, "ALTER TABLE %Q.'%q_docsize' RENAME TO '%q_docsize';", @@ -3442,21 +3659,20 @@ static int fts3EvalPhraseLoad( */ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ int iToken; /* Used to iterate through phrase tokens */ - int rc = SQLITE_OK; /* Return code */ char *aPoslist = 0; /* Position list for deferred tokens */ int nPoslist = 0; /* Number of bytes in aPoslist */ int iPrev = -1; /* Token number of previous deferred token */ assert( pPhrase->doclist.bFreeList==0 ); - for(iToken=0; rc==SQLITE_OK && iTokennToken; iToken++){ + for(iToken=0; iTokennToken; iToken++){ Fts3PhraseToken *pToken = &pPhrase->aToken[iToken]; Fts3DeferredToken *pDeferred = pToken->pDeferred; if( pDeferred ){ char *pList; int nList; - rc = sqlite3Fts3DeferredTokenList(pDeferred, &pList, &nList); + int rc = sqlite3Fts3DeferredTokenList(pDeferred, &pList, &nList); if( rc!=SQLITE_OK ) return rc; if( pList==0 ){ @@ -3557,6 +3773,7 @@ static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){ && p->nToken==1 && pFirst->pSegcsr && pFirst->pSegcsr->bLookup + && pFirst->bFirst==0 ){ /* Use the incremental approach. */ int iCol = (p->iColumn >= pTab->nColumn ? -1 : p->iColumn); @@ -3786,7 +4003,7 @@ static void fts3EvalTokenCosts( Fts3Expr ***ppOr, /* Write new OR root to *(*ppOr)++ */ int *pRc /* IN/OUT: Error code */ ){ - if( *pRc==SQLITE_OK && pExpr ){ + if( *pRc==SQLITE_OK ){ if( pExpr->eType==FTSQUERY_PHRASE ){ Fts3Phrase *pPhrase = pExpr->pPhrase; int i; @@ -3800,6 +4017,11 @@ static void fts3EvalTokenCosts( *pRc = sqlite3Fts3MsrOvfl(pCsr, pTC->pToken->pSegcsr, &pTC->nOvfl); } }else if( pExpr->eType!=FTSQUERY_NOT ){ + assert( pExpr->eType==FTSQUERY_OR + || pExpr->eType==FTSQUERY_AND + || pExpr->eType==FTSQUERY_NEAR + ); + assert( pExpr->pLeft && pExpr->pRight ); if( pExpr->eType==FTSQUERY_OR ){ pRoot = pExpr->pLeft; **ppOr = pRoot; @@ -3904,6 +4126,15 @@ static int fts3EvalSelectDeferred( int nMinEst = 0; /* The minimum count for any phrase so far. */ int nLoad4 = 1; /* (Phrases that will be loaded)^4. */ + /* Tokens are never deferred for FTS tables created using the content=xxx + ** option. The reason being that it is not guaranteed that the content + ** table actually contains the same data as the index. To prevent this from + ** causing any problems, the deferred token optimization is completely + ** disabled for content=xxx tables. */ + if( pTab->zContentTbl ){ + return SQLITE_OK; + } + /* Count the tokens in this AND/NEAR cluster. If none of the doclists ** associated with the tokens spill onto overflow pages, or if there is ** only 1 token, exit early. No tokens to defer in this case. */ diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index f210aa4569..78392ec3f8 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -191,6 +191,7 @@ struct Fts3Table { int nColumn; /* number of named columns in virtual table */ char **azColumn; /* column names. malloced */ sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */ + char *zContentTbl; /* content=xxx option, or NULL */ /* Precompiled statements used by the implementation. Each of these ** statements is run and reset within a single virtual table API call. @@ -231,7 +232,7 @@ struct Fts3Table { int nPendingData; /* Current bytes of pending data */ sqlite_int64 iPrevDocid; /* Docid of most recently inserted document */ -#if defined(SQLITE_DEBUG) +#if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) /* State variables used for validating that the transaction control ** methods of the virtual table are called at appropriate times. These ** values do not contribution to the FTS computation; they are used for @@ -316,6 +317,7 @@ struct Fts3PhraseToken { char *z; /* Text of the token */ int n; /* Number of bytes in buffer z */ int isPrefix; /* True if token ends with a "*" character */ + int bFirst; /* True if token must appear at position 0 */ /* Variables above this point are populated when the expression is ** parsed (by code in fts3_expr.c). Below this point the variables are @@ -434,6 +436,7 @@ int sqlite3Fts3SegReaderCursor( #define FTS3_SEGMENT_COLUMN_FILTER 0x00000004 #define FTS3_SEGMENT_PREFIX 0x00000008 #define FTS3_SEGMENT_SCAN 0x00000010 +#define FTS3_SEGMENT_FIRST 0x00000020 /* Type passed as 4th argument to SegmentReaderIterate() */ struct Fts3SegFilter { @@ -473,8 +476,8 @@ int sqlite3Fts3GetVarint32(const char *, int *); int sqlite3Fts3VarintLen(sqlite3_uint64); void sqlite3Fts3Dequote(char *); void sqlite3Fts3DoclistPrev(int,char*,int,char**,sqlite3_int64*,int*,u8*); - int sqlite3Fts3EvalPhraseStats(Fts3Cursor *, Fts3Expr *, u32 *); +int sqlite3Fts3FirstFilter(sqlite3_int64, char *, int, char *); /* fts3_tokenizer.c */ const char *sqlite3Fts3NextToken(const char *, int *); @@ -493,7 +496,7 @@ void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *, const char *); /* fts3_expr.c */ int sqlite3Fts3ExprParse(sqlite3_tokenizer *, - char **, int, int, const char *, int, Fts3Expr ** + char **, int, int, int, const char *, int, Fts3Expr ** ); void sqlite3Fts3ExprFree(Fts3Expr *); #ifdef SQLITE_TEST diff --git a/ext/fts3/fts3_expr.c b/ext/fts3/fts3_expr.c index 6b2b414aab..1c3a79071c 100644 --- a/ext/fts3/fts3_expr.c +++ b/ext/fts3/fts3_expr.c @@ -93,6 +93,7 @@ typedef struct ParseContext ParseContext; struct ParseContext { sqlite3_tokenizer *pTokenizer; /* Tokenizer module */ const char **azCol; /* Array of column names for fts3 table */ + int bFts4; /* True to allow FTS4-only syntax */ int nCol; /* Number of entries in azCol[] */ int iDefaultCol; /* Default column to query */ int isNot; /* True if getNextNode() sees a unary - */ @@ -180,9 +181,21 @@ static int getNextToken( pRet->pPhrase->aToken[0].isPrefix = 1; iEnd++; } - if( !sqlite3_fts3_enable_parentheses && iStart>0 && z[iStart-1]=='-' ){ - pParse->isNot = 1; + + while( 1 ){ + if( !sqlite3_fts3_enable_parentheses + && iStart>0 && z[iStart-1]=='-' + ){ + pParse->isNot = 1; + iStart--; + }else if( pParse->bFts4 && iStart>0 && z[iStart-1]=='^' ){ + pRet->pPhrase->aToken[0].bFirst = 1; + iStart--; + }else{ + break; + } } + } nConsumed = iEnd; } @@ -281,6 +294,7 @@ static int getNextString( pToken->n = nByte; pToken->isPrefix = (iEndbFirst = (iBegin>0 && zInput[iBegin-1]=='^'); nToken = ii+1; } } @@ -732,6 +746,7 @@ exprparse_out: int sqlite3Fts3ExprParse( sqlite3_tokenizer *pTokenizer, /* Tokenizer module */ char **azCol, /* Array of column names for fts3 table */ + int bFts4, /* True to allow FTS4-only syntax */ int nCol, /* Number of entries in azCol[] */ int iDefaultCol, /* Default column to query */ const char *z, int n, /* Text of MATCH query */ @@ -745,6 +760,7 @@ int sqlite3Fts3ExprParse( sParse.nCol = nCol; sParse.iDefaultCol = iDefaultCol; sParse.nNest = 0; + sParse.bFts4 = bFts4; if( z==0 ){ *ppExpr = 0; return SQLITE_OK; @@ -934,7 +950,7 @@ static void fts3ExprTest( } rc = sqlite3Fts3ExprParse( - pTokenizer, azCol, nCol, nCol, zExpr, nExpr, &pExpr + pTokenizer, azCol, 0, nCol, nCol, zExpr, nExpr, &pExpr ); if( rc!=SQLITE_OK && rc!=SQLITE_NOMEM ){ sqlite3_result_error(context, "Error parsing expression", -1); diff --git a/ext/fts3/fts3_snippet.c b/ext/fts3/fts3_snippet.c index 3bb47a04ad..23ef25c5d4 100644 --- a/ext/fts3/fts3_snippet.c +++ b/ext/fts3/fts3_snippet.c @@ -368,6 +368,7 @@ static int fts3SnippetFindPositions(Fts3Expr *pExpr, int iPhrase, void *ctx){ int iFirst = 0; pPhrase->pList = pCsr; fts3GetDeltaPosition(&pCsr, &iFirst); + assert( iFirst>=0 ); pPhrase->pHead = pCsr; pPhrase->pTail = pCsr; pPhrase->iHead = iFirst; @@ -1409,7 +1410,7 @@ void sqlite3Fts3Offsets( if( !pTerm ){ /* All offsets for this column have been gathered. */ - break; + rc = SQLITE_DONE; }else{ assert( iCurrent<=iMinPos ); if( 0==(0xFE&*pTerm->pList) ){ @@ -1426,7 +1427,7 @@ void sqlite3Fts3Offsets( "%d %d %d %d ", iCol, pTerm-sCtx.aTerm, iStart, iEnd-iStart ); rc = fts3StringAppend(&res, aBuffer, -1); - }else if( rc==SQLITE_DONE ){ + }else if( rc==SQLITE_DONE && pTab->zContentTbl==0 ){ rc = FTS_CORRUPT_VTAB; } } diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 471feeec9f..d3a88cbe61 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -256,7 +256,7 @@ static int fts3SqlStmt( /* 4 */ "DELETE FROM %Q.'%q_segdir'", /* 5 */ "DELETE FROM %Q.'%q_docsize'", /* 6 */ "DELETE FROM %Q.'%q_stat'", -/* 7 */ "SELECT %s FROM %Q.'%q_content' AS x WHERE rowid=?", +/* 7 */ "SELECT %s WHERE rowid=?", /* 8 */ "SELECT (SELECT max(idx) FROM %Q.'%q_segdir' WHERE level = ?) + 1", /* 9 */ "INSERT INTO %Q.'%q_segments'(blockid, block) VALUES(?, ?)", /* 10 */ "SELECT coalesce((SELECT max(blockid) FROM %Q.'%q_segments') + 1, 1)", @@ -298,7 +298,7 @@ static int fts3SqlStmt( if( eStmt==SQL_CONTENT_INSERT ){ zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName, p->zWriteExprlist); }else if( eStmt==SQL_SELECT_CONTENT_BY_ROWID ){ - zSql = sqlite3_mprintf(azSql[eStmt], p->zReadExprlist, p->zDb, p->zName); + zSql = sqlite3_mprintf(azSql[eStmt], p->zReadExprlist); }else{ zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName); } @@ -409,17 +409,24 @@ static void fts3SqlExec( ** not what users expect when they get SQLITE_LOCKED_SHAREDCACHE. It can ** still happen if the user reads data directly from the %_segments or ** %_segdir tables instead of going through FTS3 though. +** +** This reasoning does not apply to a content=xxx table. */ int sqlite3Fts3ReadLock(Fts3Table *p){ int rc; /* Return code */ sqlite3_stmt *pStmt; /* Statement used to obtain lock */ - rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pStmt, 0); - if( rc==SQLITE_OK ){ - sqlite3_bind_null(pStmt, 1); - sqlite3_step(pStmt); - rc = sqlite3_reset(pStmt); + if( p->zContentTbl==0 ){ + rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_null(pStmt, 1); + sqlite3_step(pStmt); + rc = sqlite3_reset(pStmt); + } + }else{ + rc = SQLITE_OK; } + return rc; } @@ -780,6 +787,18 @@ static int fts3InsertData( int rc; /* Return code */ sqlite3_stmt *pContentInsert; /* INSERT INTO %_content VALUES(...) */ + if( p->zContentTbl ){ + sqlite3_value *pRowid = apVal[p->nColumn+3]; + if( sqlite3_value_type(pRowid)==SQLITE_NULL ){ + pRowid = apVal[1]; + } + if( sqlite3_value_type(pRowid)!=SQLITE_INTEGER ){ + return SQLITE_CONSTRAINT; + } + *piDocid = sqlite3_value_int64(pRowid); + return SQLITE_OK; + } + /* Locate the statement handle used to insert data into the %_content ** table. The SQL for this statement is: ** @@ -830,14 +849,16 @@ static int fts3InsertData( ** Remove all data from the FTS3 table. Clear the hash table containing ** pending terms. */ -static int fts3DeleteAll(Fts3Table *p){ +static int fts3DeleteAll(Fts3Table *p, int bContent){ int rc = SQLITE_OK; /* Return code */ /* Discard the contents of the pending-terms hash table. */ sqlite3Fts3PendingTermsClear(p); - /* Delete everything from the %_content, %_segments and %_segdir tables. */ - fts3SqlExec(&rc, p, SQL_DELETE_ALL_CONTENT, 0); + /* Delete everything from the shadow tables. Except, leave %_content as + ** is if bContent is false. */ + assert( p->zContentTbl==0 || bContent==0 ); + if( bContent ) fts3SqlExec(&rc, p, SQL_DELETE_ALL_CONTENT, 0); fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGMENTS, 0); fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGDIR, 0); if( p->bHasDocsize ){ @@ -2125,12 +2146,18 @@ static void fts3SegWriterFree(SegmentWriter *pWriter){ static int fts3IsEmpty(Fts3Table *p, sqlite3_value *pRowid, int *pisEmpty){ sqlite3_stmt *pStmt; int rc; - rc = fts3SqlStmt(p, SQL_IS_EMPTY, &pStmt, &pRowid); - if( rc==SQLITE_OK ){ - if( SQLITE_ROW==sqlite3_step(pStmt) ){ - *pisEmpty = sqlite3_column_int(pStmt, 0); + if( p->zContentTbl ){ + /* If using the content=xxx option, assume the table is never empty */ + *pisEmpty = 0; + rc = SQLITE_OK; + }else{ + rc = fts3SqlStmt(p, SQL_IS_EMPTY, &pStmt, &pRowid); + if( rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + *pisEmpty = sqlite3_column_int(pStmt, 0); + } + rc = sqlite3_reset(pStmt); } - rc = sqlite3_reset(pStmt); } return rc; } @@ -2482,6 +2509,7 @@ int sqlite3Fts3SegReaderStep( int isColFilter = (pCsr->pFilter->flags & FTS3_SEGMENT_COLUMN_FILTER); int isPrefix = (pCsr->pFilter->flags & FTS3_SEGMENT_PREFIX); int isScan = (pCsr->pFilter->flags & FTS3_SEGMENT_SCAN); + int isFirst = (pCsr->pFilter->flags & FTS3_SEGMENT_FIRST); Fts3SegReader **apSegment = pCsr->apSegment; int nSegment = pCsr->nSegment; @@ -2541,6 +2569,7 @@ int sqlite3Fts3SegReaderStep( assert( isIgnoreEmpty || (isRequirePos && !isColFilter) ); if( nMerge==1 && !isIgnoreEmpty + && !isFirst && (p->bDescIdx==0 || fts3SegReaderIsPending(apSegment[0])==0) ){ pCsr->nDoclist = apSegment[0]->nDoclist; @@ -2606,12 +2635,24 @@ int sqlite3Fts3SegReaderStep( } pCsr->aBuffer = aNew; } - nDoclist += sqlite3Fts3PutVarint(&pCsr->aBuffer[nDoclist], iDelta); - iPrev = iDocid; - if( isRequirePos ){ - memcpy(&pCsr->aBuffer[nDoclist], pList, nList); - nDoclist += nList; - pCsr->aBuffer[nDoclist++] = '\0'; + + if( isFirst ){ + char *a = &pCsr->aBuffer[nDoclist]; + int nWrite; + + nWrite = sqlite3Fts3FirstFilter(iDelta, pList, nList, a); + if( nWrite ){ + iPrev = iDocid; + nDoclist += nWrite; + } + }else{ + nDoclist += sqlite3Fts3PutVarint(&pCsr->aBuffer[nDoclist], iDelta); + iPrev = iDocid; + if( isRequirePos ){ + memcpy(&pCsr->aBuffer[nDoclist], pList, nList); + nDoclist += nList; + pCsr->aBuffer[nDoclist++] = '\0'; + } } } @@ -2787,9 +2828,9 @@ static void fts3DecodeIntArray( ** a blob of varints. */ static void fts3InsertDocsize( - int *pRC, /* Result code */ - Fts3Table *p, /* Table into which to insert */ - u32 *aSz /* Sizes of each column */ + int *pRC, /* Result code */ + Fts3Table *p, /* Table into which to insert */ + u32 *aSz /* Sizes of each column, in tokens */ ){ char *pBlob; /* The BLOB encoding of the document size */ int nBlob; /* Number of bytes in the BLOB */ @@ -2911,6 +2952,86 @@ static int fts3DoOptimize(Fts3Table *p, int bReturnDone){ return (rc==SQLITE_OK && bReturnDone && bSeenDone) ? SQLITE_DONE : rc; } +/* +** This function is called when the user executes the following statement: +** +** INSERT INTO () VALUES('rebuild'); +** +** The entire FTS index is discarded and rebuilt. If the table is one +** created using the content=xxx option, then the new index is based on +** the current contents of the xxx table. Otherwise, it is rebuilt based +** on the contents of the %_content table. +*/ +static int fts3DoRebuild(Fts3Table *p){ + int rc; /* Return Code */ + + rc = fts3DeleteAll(p, 0); + if( rc==SQLITE_OK ){ + u32 *aSz = 0; + u32 *aSzIns; + u32 *aSzDel; + sqlite3_stmt *pStmt = 0; + int nEntry = 0; + + /* Compose and prepare an SQL statement to loop through the content table */ + char *zSql = sqlite3_mprintf("SELECT %s" , p->zReadExprlist); + if( !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + } + + if( rc==SQLITE_OK ){ + int nByte = sizeof(u32) * (p->nColumn+1)*3; + aSz = (u32 *)sqlite3_malloc(nByte); + if( aSz==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(aSz, 0, nByte); + aSzIns = &aSz[p->nColumn+1]; + aSzDel = &aSzIns[p->nColumn+1]; + } + } + + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + int iCol; + rc = fts3PendingTermsDocid(p, sqlite3_column_int64(pStmt, 0)); + aSz[p->nColumn] = 0; + for(iCol=0; rc==SQLITE_OK && iColnColumn; iCol++){ + const char *z = (const char *) sqlite3_column_text(pStmt, iCol+1); + rc = fts3PendingTermsAdd(p, z, iCol, &aSz[iCol]); + aSz[p->nColumn] += sqlite3_column_bytes(pStmt, iCol+1); + } + if( p->bHasDocsize ){ + fts3InsertDocsize(&rc, p, aSz); + } + if( rc!=SQLITE_OK ){ + sqlite3_finalize(pStmt); + pStmt = 0; + }else{ + nEntry++; + for(iCol=0; iCol<=p->nColumn; iCol++){ + aSzIns[iCol] += aSz[iCol]; + } + } + } + if( p->bHasStat ){ + fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nEntry); + } + sqlite3_free(aSz); + + if( pStmt ){ + int rc2 = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ){ + rc = rc2; + } + } + } + + return rc; +} + /* ** Handle a 'special' INSERT of the form: ** @@ -2928,6 +3049,8 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){ return SQLITE_NOMEM; }else if( nVal==8 && 0==sqlite3_strnicmp(zVal, "optimize", 8) ){ rc = fts3DoOptimize(p, 0); + }else if( nVal==7 && 0==sqlite3_strnicmp(zVal, "rebuild", 7) ){ + rc = fts3DoRebuild(p); #ifdef SQLITE_TEST }else if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){ p->nNodeSize = atoi(&zVal[9]); @@ -3008,6 +3131,7 @@ int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *pCsr){ for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){ Fts3PhraseToken *pPT = pDef->pToken; if( (pDef->iCol>=p->nColumn || pDef->iCol==i) + && (pPT->bFirst==0 || iPos==0) && (pPT->n==nToken || (pPT->isPrefix && pPT->nz, pPT->n)) ){ @@ -3099,14 +3223,18 @@ static int fts3DeleteByRowid( /* Deleting this row means the whole table is empty. In this case ** delete the contents of all three tables and throw away any ** data in the pendingTerms hash table. */ - rc = fts3DeleteAll(p); + rc = fts3DeleteAll(p, 1); *pnDoc = *pnDoc - 1; }else{ sqlite3_int64 iRemove = sqlite3_value_int64(pRowid); rc = fts3PendingTermsDocid(p, iRemove); fts3DeleteTerms(&rc, p, pRowid, aSzDel); - fts3SqlExec(&rc, p, SQL_DELETE_CONTENT, &pRowid); - if( sqlite3_changes(p->db) ) *pnDoc = *pnDoc - 1; + if( p->zContentTbl==0 ){ + fts3SqlExec(&rc, p, SQL_DELETE_CONTENT, &pRowid); + if( sqlite3_changes(p->db) ) *pnDoc = *pnDoc - 1; + }else{ + *pnDoc = *pnDoc - 1; + } if( p->bHasDocsize ){ fts3SqlExec(&rc, p, SQL_DELETE_DOCSIZE, &pRowid); } @@ -3166,7 +3294,7 @@ int sqlite3Fts3UpdateMethod( ** detect the conflict and return SQLITE_CONSTRAINT before beginning to ** modify the database file. */ - if( nArg>1 ){ + if( nArg>1 && p->zContentTbl==0 ){ /* Find the value object that holds the new rowid value. */ sqlite3_value *pNewRowid = apVal[3+p->nColumn]; if( sqlite3_value_type(pNewRowid)==SQLITE_NULL ){ @@ -3217,7 +3345,9 @@ int sqlite3Fts3UpdateMethod( if( nArg>1 && rc==SQLITE_OK ){ if( bInsertDone==0 ){ rc = fts3InsertData(p, apVal, pRowid); - if( rc==SQLITE_CONSTRAINT ) rc = FTS_CORRUPT_VTAB; + if( rc==SQLITE_CONSTRAINT && p->zContentTbl==0 ){ + rc = FTS_CORRUPT_VTAB; + } } if( rc==SQLITE_OK && (!isRemove || *pRowid!=p->iPrevDocid ) ){ rc = fts3PendingTermsDocid(p, *pRowid); diff --git a/manifest b/manifest index a2eeca7d58..528e4691d0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\svirtual\stable\srename\slogic\sso\sthat\sit\sworks\seven\sif\sthe\sdatabase\nencoding\sis\ssomething\sother\sthan\sUTF8.\nTicket\s[8290242b2a9a81683] -D 2011-10-18T22:07:47.722 +C Merge\sthe\sfts4-content\sbranch\swith\sthe\strunk. +D 2011-10-19T16:20:40.613 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a162fe39e249b8ed4a65ee947c30152786cfe897 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -62,22 +62,22 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 19f36945148cfd4ee3655b5fac0879ba4e0f3117 +F ext/fts3/fts3.c 9c125699baf1c03a3d8d53fb7d8d27ab4ad7d6eb F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h 59c5a9475fed5d76c70a4763103b3c8e60424a68 +F ext/fts3/fts3Int.h def7a900f98c5ab5fa4772e922bfa219d5097f05 F ext/fts3/fts3_aux.c 0ebfa7b86cf8ff6a0861605fcc63b83ec1b70691 -F ext/fts3/fts3_expr.c 61ceee7c9698c7ae1ba55ab6ff7961456a8b465a +F ext/fts3/fts3_expr.c f5df26bddf46a5916b2a5f80c4027996e92b7b15 F ext/fts3/fts3_hash.c 8dd2d06b66c72c628c2732555a32bc0943114914 F ext/fts3/fts3_hash.h 8331fb2206c609f9fc4c4735b9ab5ad6137c88ec F ext/fts3/fts3_icu.c 6c8f395cdf9e1e3afa7fadb7e523dbbf381c6dfa F ext/fts3/fts3_porter.c 8d946908f4812c005d3d33fcbe78418b1f4eb70c -F ext/fts3/fts3_snippet.c 19a906f8ed73ad8b670dfc271ceae7b3338c157e +F ext/fts3/fts3_snippet.c 1f9ee6a8e0e242649645968dcec4deb253d86c2a F ext/fts3/fts3_term.c a5457992723455a58804cb75c8cbd8978db5c2ef F ext/fts3/fts3_test.c 24fa13f330db011500acb95590da9eee24951894 F ext/fts3/fts3_tokenizer.c 9ff7ec66ae3c5c0340fa081958e64f395c71a106 F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3 F ext/fts3/fts3_tokenizer1.c 0dde8f307b8045565cf63797ba9acfaff1c50c68 -F ext/fts3/fts3_write.c ffe13acc3867ea6b0fc8b9cfbf904bfae64eac84 +F ext/fts3/fts3_write.c f2545f59a4cc2eb6739acb3d026b8a91a1f3d429 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 @@ -459,7 +459,7 @@ F test/fts3ak.test bd14deafe9d1586e8e9bf032411026ac4f8c925d F test/fts3al.test 07d64326e79bbdbab20ee87fc3328fbf01641c9f F test/fts3am.test 218aa6ba0dfc50c7c16b2022aac5c6be593d08d8 F test/fts3an.test a49ccadc07a2f7d646ec1b81bc09da2d85a85b18 -F test/fts3ao.test 60a15590d3c8578e943e4a149524b16b9bc1be92 +F test/fts3ao.test e7b80272efcced57d1d087a9da5c690dd7c21fd9 F test/fts3atoken.test 402ef2f7c2fb4b3d4fa0587df6441c1447e799b3 F test/fts3auto.test c1a30b37002b7c764a96937fbc71065b73d69494 F test/fts3aux1.test 0b02743955d56fc0d4d66236a26177bd1b726de0 @@ -471,23 +471,25 @@ F test/fts3corrupt.test 7b0f91780ca36118d73324ec803187208ad33b32 F test/fts3corrupt2.test 6d96efae2f8a6af3eeaf283aba437e6d0e5447ba F test/fts3cov.test e0fb00d8b715ddae4a94c305992dfc3ef70353d7 F test/fts3d.test bf640d79722b720fa1c81834c48cdaa45d531b1a -F test/fts3defer.test ffd4e07f79a09660d4b3e2613b041ab9b6100d91 +F test/fts3defer.test 2ea3fa028f8d9523f9c33dd8acc4555d567ea4ac F test/fts3defer2.test 35867d33ba6db03f6c73bd6f5fc333ae14f68c81 F test/fts3e.test 1f6c6ac9cc8b772ca256e6b22aaeed50c9350851 F test/fts3expr.test 5e745b2b6348499d9ef8d59015de3182072c564c F test/fts3expr2.test 18da930352e5693eaa163a3eacf96233b7290d1a F test/fts3fault.test f83e556465bb69dc8bc676339eca408dce4ca246 -F test/fts3fault2.test dc96203af6ba31ce20163fc35460e1556e8edf4d -F test/fts3malloc.test 9c8cc3f885bb4dfc66d0460c52f68f45e4710d1b -F test/fts3matchinfo.test 08a82d18cc08abb28aec41d412b4c2ef25ba6a5f +F test/fts3fault2.test b62a2bc843c20414405f80e5eeb78e39bc68fe53 +F test/fts3first.test dbdedd20914c8d539aa3206c9b34a23775644641 +F test/fts3malloc.test b86ea33db9e8c58c0c2f8027a9fcadaf6a1568be +F test/fts3matchinfo.test 6507fe1c342e542300d65ea637d4110eccf894e6 F test/fts3near.test 2e318ee434d32babd27c167142e2b94ddbab4844 -F test/fts3prefix.test 36246609111ec1683f7ea5ed27666ce2cefb5676 +F test/fts3prefix.test b36d4f00b128a51e7b386cc013a874246d9d7dc1 F test/fts3query.test ef79d31fdb355d094baec1c1b24b60439a1fb8a2 F test/fts3rnd.test 1320d8826a845e38a96e769562bf83d7a92a15d0 F test/fts3shared.test 8bb266521d7c5495c0ae522bb4d376ad5387d4a2 F test/fts3snippet.test 8e956051221a34c7daeb504f023cb54d5fa5a8b2 -F test/fts3sort.test 9a5176c9317bb545ec5f144d62e6fedb4da6c66e +F test/fts3sort.test 95be0b19d7e41c44b29014f13ea8bddd495fd659 F test/fts4aa.test 6e7f90420b837b2c685f3bcbe84c868492d40a68 +F test/fts4content.test c5f531ecfc3d446b90032cae212549dbbb18dd78 F test/func.test 6c5ce11e3a0021ca3c0649234e2d4454c89110ca F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f F test/func3.test 001021e5b88bd02a3b365a5c5fd8f6f49d39744a @@ -622,7 +624,7 @@ F test/pageropt.test 9191867ed19a2b3db6c42d1b36b6fbc657cd1ab0 F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16 F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 -F test/permutations.test ad17319066a90e2db71823c3ff104795ffc71b31 +F test/permutations.test 522823b47238cb1754198f80817fe9f9158ede55 F test/pragma.test 1ea0c85be853135bb7468e6eed48ee12b04794d4 F test/pragma2.test 3a55f82b954242c642f8342b17dffc8b47472947 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 @@ -970,7 +972,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 54aecd929867606d14a062b501abbfb6f5f05e37 -R dccf2fe03db8f585fac4df0bc3670756 -U drh -Z aaf98f5abdd914fb531f49ab6760c2d4 +P d65f63531c3f8e3e55e656f049240714a3d7433f df36ac948179f37b432a88701b6c79299e073ce8 +R 3877ede50d6923a62a70ae0714da94fb +U dan +Z e6c406cc06dcce98e86f6b11f422dc50 diff --git a/manifest.uuid b/manifest.uuid index 10b56c6753..eef6d62dc6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d65f63531c3f8e3e55e656f049240714a3d7433f \ No newline at end of file +8a4077057ddeb08e8edc5f20a75abaaba7a278ba \ No newline at end of file diff --git a/test/fts3ao.test b/test/fts3ao.test index 0b6fcd3d19..786667a7f3 100644 --- a/test/fts3ao.test +++ b/test/fts3ao.test @@ -200,6 +200,9 @@ do_test fts3ao-4.7 { SELECT * FROM t5; } } {{the quick brown fox} {jumped over the} {lazy dog}} +do_execsql_test fts3ao-4.8 { + SELECT snippet(t5, '[', ']') FROM t5 WHERE t5 MATCH 'the' +} {{[the] quick brown fox} {jumped over [the]}} # Test that it is possible to rename an FTS4 table. Renaming an FTS4 table # involves renaming the extra %_docsize and %_stat tables. diff --git a/test/fts3defer.test b/test/fts3defer.test index ab30bbe2c3..bc50874e4a 100644 --- a/test/fts3defer.test +++ b/test/fts3defer.test @@ -426,6 +426,18 @@ foreach {tn setup} { SELECT rowid FROM t1 WHERE t1 MATCH '"jk xduvfhk" OR "zm azavwm"' } {8 15 26 92 96} } + + if {$tn>1} { + # These tests will not work with $tn==1, as in this case table t1 is + # created using FTS3. The ^ syntax is only available with FTS4 tables. + # + do_select_test 7.1 { + SELECT rowid FROM t1 WHERE t1 MATCH '^zm mjpavjuhw' + } {56 62} + do_select_test 7.2 { + SELECT rowid FROM t1 WHERE t1 MATCH '^azavwm zm' + } {43} + } } set testprefix fts3defer diff --git a/test/fts3fault2.test b/test/fts3fault2.test index fb877737f4..0178ed27cc 100644 --- a/test/fts3fault2.test +++ b/test/fts3fault2.test @@ -82,4 +82,53 @@ do_faultsim_test 2.1 -prep { faultsim_test_result {0 {a * 1 1 a 0 1 1 b * 1 1 b 0 1 1 c * 1 1 c 0 1 1 x * 1 1 x 1 1 1 y * 1 1 y 1 1 1 z * 1 1 z 1 1 1}} } +do_faultsim_test 3.0 -faults oom* -prep { + faultsim_delete_and_reopen + db eval { CREATE TABLE 'xx yy'(a, b); } +} -body { + execsql { + CREATE VIRTUAL TABLE tt USING fts4(content="xx yy"); + } +} -test { + faultsim_test_result {0 {}} +} + +do_faultsim_test 3.1 -faults oom* -prep { + faultsim_delete_and_reopen + db func zip zip + db func unzip unzip +} -body { + execsql { + CREATE VIRTUAL TABLE tt USING fts4(compress=zip, uncompress=unzip); + } +} -test { + faultsim_test_result {0 {}} +} + +do_test 4.0 { + faultsim_delete_and_reopen + execsql { + CREATE VIRTUAL TABLE ft USING fts4(a, b); + INSERT INTO ft VALUES('U U T C O', 'F N D E S'); + INSERT INTO ft VALUES('P H X G B', 'I D M R U'); + INSERT INTO ft VALUES('P P X D M', 'Y V N T C'); + INSERT INTO ft VALUES('Z L Q O W', 'D F U N Q'); + INSERT INTO ft VALUES('A J D U P', 'C H M Q E'); + INSERT INTO ft VALUES('P S A O H', 'S Z C W D'); + INSERT INTO ft VALUES('T B N L W', 'C A K T I'); + INSERT INTO ft VALUES('K E Z L O', 'L L Y C E'); + INSERT INTO ft VALUES('C R E S V', 'Q V F W P'); + INSERT INTO ft VALUES('S K H G W', 'R W Q F G'); + } + faultsim_save_and_close +} {} +do_faultsim_test 4.1 -prep { + faultsim_restore_and_reopen + db eval {SELECT * FROM sqlite_master} +} -body { + execsql { INSERT INTO ft(ft) VALUES('rebuild') } +} -test { + faultsim_test_result {0 {}} +} + finish_test diff --git a/test/fts3first.test b/test/fts3first.test new file mode 100644 index 0000000000..673f818deb --- /dev/null +++ b/test/fts3first.test @@ -0,0 +1,163 @@ +# 2011 October 18 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/malloc_common.tcl + +ifcapable !fts3 { + finish_test + return +} + +set testprefix fts3first + +proc lreverse {L} { + set res [list] + for {set ii [expr [llength $L]-1]} {$ii>=0} {incr ii -1} { + lappend res [lindex $L $ii] + } + set res +} + +proc mit {blob} { + set scan(littleEndian) i* + set scan(bigEndian) I* + binary scan $blob $scan($::tcl_platform(byteOrder)) r + return $r +} +db func mit mit + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE x1 USING FTS4(a, b, c); + INSERT INTO x1(docid,a,b,c) VALUES(0, 'K H D S T', 'V M N Y K', 'S Z N Q S'); + INSERT INTO x1(docid,a,b,c) VALUES(1, 'K N J L W', 'S Z W J Q', 'D U W S E'); + INSERT INTO x1(docid,a,b,c) VALUES(2, 'B P M O I', 'R P H W S', 'R J L L E'); + INSERT INTO x1(docid,a,b,c) VALUES(3, 'U R Q M L', 'M J K A V', 'Q W J T J'); + INSERT INTO x1(docid,a,b,c) VALUES(4, 'N J C Y N', 'R U D X V', 'B O U A Q'); + INSERT INTO x1(docid,a,b,c) VALUES(5, 'Q L X L U', 'I F N X S', 'U Q A N Y'); + INSERT INTO x1(docid,a,b,c) VALUES(6, 'M R G U T', 'U V I Q P', 'X Y D L S'); + INSERT INTO x1(docid,a,b,c) VALUES(7, 'D Y P O I', 'X J P K R', 'V O T H V'); + INSERT INTO x1(docid,a,b,c) VALUES(8, 'R Y D L R', 'U U E S J', 'N W L M R'); + INSERT INTO x1(docid,a,b,c) VALUES(9, 'Z P F N P', 'W A X D U', 'V A E Q A'); + INSERT INTO x1(docid,a,b,c) VALUES(10, 'Q I A Q M', 'N D K H C', 'A H T Q Z'); + INSERT INTO x1(docid,a,b,c) VALUES(11, 'T E R Q B', 'C I B C B', 'F Z U W R'); + INSERT INTO x1(docid,a,b,c) VALUES(12, 'E S V U W', 'T P F W H', 'A M D J Q'); + INSERT INTO x1(docid,a,b,c) VALUES(13, 'X S B X Y', 'U D N D P', 'X Z Y G F'); + INSERT INTO x1(docid,a,b,c) VALUES(14, 'K H A B L', 'S R C C Z', 'D W E H J'); + INSERT INTO x1(docid,a,b,c) VALUES(15, 'C E U C C', 'W F M N M', 'T Z U X T'); + INSERT INTO x1(docid,a,b,c) VALUES(16, 'Q G C G H', 'H N N B H', 'B Q I H Y'); + INSERT INTO x1(docid,a,b,c) VALUES(17, 'Q T S K B', 'W B D Y N', 'V J P E C'); + INSERT INTO x1(docid,a,b,c) VALUES(18, 'A J M O Q', 'L G Y Y A', 'G N M R N'); + INSERT INTO x1(docid,a,b,c) VALUES(19, 'T R Y P Y', 'N V Y B X', 'L Z T N T'); + + CREATE VIRTUAL TABLE x2 USING FTS4(a, b, c, order=DESC); + INSERT INTO x2(docid, a, b, c) SELECT docid, a, b, c FROM x1; +} + + +# Test queries. +# +foreach x {1 2} { + foreach {tn match res} { + 1 "^K" {0 1 14} + 2 "^S" {0 1 14} + 3 "^W" {9 15 17} + 4 "^J" {} + 5 "^E" {12} + 6 "V ^-E" {0 3 4 6 7 9 17 19} + 7 "V -^E" {0 3 4 6 7 9 17 19} + 8 "^-E V" {0 3 4 6 7 9 17 19} + 9 "-^E V" {0 3 4 6 7 9 17 19} + 10 "V" {0 3 4 6 7 9 12 17 19} + + 11 {"^K H"} {0 14} + 12 {"K H"} {0 10 14} + 13 {"K ^H"} {} + } { + set rev [lreverse $res] + do_execsql_test 1.$x.$tn.1 {SELECT docid FROM x1 WHERE x1 MATCH $match} $res + do_execsql_test 1.$x.$tn.2 {SELECT docid FROM x2 WHERE x2 MATCH $match} $rev + } + + do_execsql_test 1.$x.[expr $tn+1] { + INSERT INTO x1(x1) VALUES('optimize'); + INSERT INTO x2(x2) VALUES('optimize'); + } {} +} + +# Test the snippet() function. +# +foreach {tn match res} { + 1 {^K} {{[K] H D S T} {[K] N J L W} {[K] H A B L}} + 2 {^X} {{[X] Y D L S} {[X] J P K R} {[X] S B X Y}} + 3 {^X Y} {{[X] [Y] D L S} {D [Y] P O I...[X] J P K R} {[X] S B X [Y]}} +} { + set rev [lreverse $res] + + do_execsql_test 1.3.$tn.1 { + SELECT snippet(x1, '[', ']', '...') FROM x1 WHERE x1 MATCH $match + } $res + + do_execsql_test 1.3.$tn.2 { + SELECT snippet(x2, '[', ']', '...') FROM x2 WHERE x2 MATCH $match + } $rev +} + +# Test matchinfo(). +# +foreach {tn match res} { + 1 {^K} { + {1 3 3 0 0 0 0 0 0} + {1 3 3 0 0 0 0 0 0} + {1 3 3 0 0 0 0 0 0} + } + 2 {^X} { + {0 1 1 0 1 1 1 2 2} + {0 1 1 1 1 1 0 2 2} + {1 1 1 0 1 1 1 2 2} + } + 3 {^X Y} { + {0 1 1 0 1 1 1 2 2 0 6 5 0 5 4 1 4 4} + {0 1 1 1 1 1 0 2 2 1 6 5 0 5 4 0 4 4} + {1 1 1 0 1 1 1 2 2 1 6 5 0 5 4 1 4 4} + } +} { + set rev [lreverse $res] + + do_execsql_test 1.3.$tn.1 { + SELECT mit(matchinfo(x1, 'x')) FROM x1 WHERE x1 MATCH $match + } $res + do_execsql_test 1.3.$tn.2 { + SELECT mit(matchinfo(x2, 'x')) FROM x2 WHERE x2 MATCH $match + } $rev +} + +# Test that ^ is ignored for FTS3 tables. +# +do_execsql_test 2.1 { + CREATE VIRTUAL TABLE x3 USING fts3; + INSERT INTO x3 VALUES('A B C'); + INSERT INTO x3 VALUES('B A C'); + + CREATE VIRTUAL TABLE x4 USING fts4; + INSERT INTO x4 VALUES('A B C'); + INSERT INTO x4 VALUES('B A C'); +} + +do_execsql_test 2.2.1 { + SELECT * FROM x3 WHERE x3 MATCH '^A'; +} {{A B C} {B A C}} +do_execsql_test 2.2.2 { + SELECT * FROM x4 WHERE x4 MATCH '^A'; +} {{A B C}} + +finish_test diff --git a/test/fts3malloc.test b/test/fts3malloc.test index 932ea098d0..7eeee7fe0d 100644 --- a/test/fts3malloc.test +++ b/test/fts3malloc.test @@ -294,6 +294,7 @@ do_write_test fts3_malloc-5.1 ft_content { do_test fts3_malloc-5.2 { execsql { CREATE VIRTUAL TABLE ft8 USING fts3(x, tokenize porter) } } {} + do_write_test fts3_malloc-5.3 ft_content { INSERT INTO ft8 VALUES('short alongertoken reallyquitealotlongerimeanit andthistokenisjustsolongthatonemightbeforgivenforimaginingthatitwasmerelyacontrivedexampleandnotarealtoken') } diff --git a/test/fts3matchinfo.test b/test/fts3matchinfo.test index 40366b6aef..0e8885804f 100644 --- a/test/fts3matchinfo.test +++ b/test/fts3matchinfo.test @@ -19,6 +19,7 @@ source $testdir/tester.tcl ifcapable !fts3 { finish_test ; return } set testprefix fts3matchinfo +set sqlite_fts3_enable_parentheses 0 proc mit {blob} { set scan(littleEndian) i* @@ -57,6 +58,9 @@ do_catchsql_test 2.0 { do_catchsql_test 2.1 { CREATE VIRTUAL TABLE x2 USING fts4(mtchinfo=fts3); } {1 {unrecognized parameter: mtchinfo=fts3}} +do_catchsql_test 2.2 { + CREATE VIRTUAL TABLE x2 USING fts4(matchinfo=fts5); +} {1 {unrecognized matchinfo: fts5}} # Check that with fts3, the "=" character is permitted in column definitions. # @@ -224,6 +228,18 @@ do_matchinfo_test 4.1.3 t4 {t4 MATCH 'a b'} { s {{2 0} {0 2}} } do_matchinfo_test 4.1.4 t4 {t4 MATCH '"a b" c'} { s {{2 0} {0 2}} } do_matchinfo_test 4.1.5 t4 {t4 MATCH 'a "b c"'} { s {{2 0} {0 2}} } do_matchinfo_test 4.1.6 t4 {t4 MATCH 'd d'} { s {{1 0} {0 1}} } +do_matchinfo_test 4.1.7 t4 {t4 MATCH 'f OR abcd'} { + x { + {0 1 1 1 1 1 0 0 0 0 0 0} + {1 1 1 0 1 1 0 0 0 0 0 0} + } +} +do_matchinfo_test 4.1.8 t4 {t4 MATCH 'f -abcd'} { + x { + {0 1 1 1 1 1} + {1 1 1 0 1 1} + } +} do_execsql_test 4.2.0 { CREATE VIRTUAL TABLE t5 USING fts4; diff --git a/test/fts3prefix.test b/test/fts3prefix.test index f5e31f3208..e7c197da9d 100644 --- a/test/fts3prefix.test +++ b/test/fts3prefix.test @@ -200,4 +200,14 @@ do_execsql_test 4.6 { SELECT * FROM t3 WHERE t3 MATCH 'one*' } {{one two three}} +#------------------------------------------------------------------------- +# Syntax tests. +# +do_catchsql_test 5.1 { + CREATE VIRTUAL TABLE t4 USING fts4(prefix="abc"); +} {1 {error parsing prefix parameter: abc}} +do_catchsql_test 5.2 { + CREATE VIRTUAL TABLE t4 USING fts4(prefix=""); +} {0 {}} + finish_test diff --git a/test/fts3sort.test b/test/fts3sort.test index ccdc203442..be7560432b 100644 --- a/test/fts3sort.test +++ b/test/fts3sort.test @@ -138,6 +138,8 @@ foreach {tn param res} { 3 "order=dec" {1 {unrecognized order: dec}} 4 "order=xxx, order=asc" {1 {unrecognized order: xxx}} 5 "order=desc, order=asc" {0 {}} + 6 "order=xxxx, order=asc" {1 {unrecognized order: xxxx}} + 7 "order=desk" {1 {unrecognized order: desk}} } { execsql { DROP TABLE IF EXISTS t1 } do_catchsql_test 2.1.$tn " @@ -157,6 +159,9 @@ do_execsql_test 2.2 { do_execsql_test 2.3 { SELECT docid FROM t2 WHERE t2 MATCH 'aa'; } {3 1} +do_execsql_test 2.4 { + SELECT docid FROM t2 WHERE t2 MATCH 'aa' ORDER BY content; +} {1 3} #------------------------------------------------------------------------- # Test that ticket [56be976859] has been fixed. diff --git a/test/fts4content.test b/test/fts4content.test new file mode 100644 index 0000000000..8295d91d61 --- /dev/null +++ b/test/fts4content.test @@ -0,0 +1,478 @@ +# 2011 October 03 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the content=xxx FTS4 option. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set ::testprefix fts4content + +# If SQLITE_ENABLE_FTS3 is defined, omit this file. +ifcapable !fts3 { + finish_test + return +} + +#------------------------------------------------------------------------- +# Test organization: +# +# 1.* - Warm-body tests. +# +# 2.* - Querying a content=xxx FTS table. +# +# 3.* - Writing to a content=xxx FTS table. +# +# 4.* - The "INSERT INTO fts(fts) VALUES('rebuild')" command. +# +# 5.* - Check that CREATE TABLE, DROP TABLE and ALTER TABLE correctly +# ignore any %_content table when used with the content=xxx option. +# +# 6.* - Test the effects of messing with the schema of table xxx after +# creating a content=xxx FTS index. +# + +do_execsql_test 1.1.1 { + CREATE TABLE t1(a, b, c); + INSERT INTO t1 VALUES('w x', 'x y', 'y z'); + CREATE VIRTUAL TABLE ft1 USING fts4(content=t1); +} + +do_execsql_test 1.1.2 { + PRAGMA table_info(ft1); +} { + 0 a {} 0 {} 0 + 1 b {} 0 {} 0 + 2 c {} 0 {} 0 +} + +do_execsql_test 1.1.3 { SELECT *, rowid FROM ft1 } {{w x} {x y} {y z} 1} +do_execsql_test 1.1.4 { SELECT a, c FROM ft1 WHERE rowid=1 } {{w x} {y z}} + +do_execsql_test 1.1.5 { INSERT INTO ft1(ft1) VALUES('rebuild') } {} +do_execsql_test 1.1.6 { SELECT rowid FROM ft1 WHERE ft1 MATCH 'x' } {1} +do_execsql_test 1.1.7 { SELECT rowid FROM ft1 WHERE ft1 MATCH 'a' } {} + +do_execsql_test 1.2.1 { + DROP TABLE ft1; + CREATE VIRTUAL TABLE ft1 USING fts4(content=t1, b); + PRAGMA table_info(ft1); +} { + 0 b {} 0 {} 0 +} +do_execsql_test 1.2.2 { + SELECT *, rowid FROM ft1 +} {{x y} 1} + +#------------------------------------------------------------------------- +# The following block of tests - 2.* - test that a content=xxx FTS table +# can be queried. Also tested are cases where rows identified in the FTS +# are missing from the content table, and cases where the index is +# inconsistent with the content table. +# +do_execsql_test 2.0 { + CREATE TABLE t2(x); + INSERT INTO t2 VALUES('O S W W F U C R Q I C N P Z Y Y E Y Y E'); -- 1 + INSERT INTO t2 VALUES('Y X U V L B E H Y J C Y A I A P V F V K'); -- 2 + INSERT INTO t2 VALUES('P W I N J H I I N I F B K D U Q B Z S F'); -- 3 + INSERT INTO t2 VALUES('N R O R H J R H G M D I U U B O M P A U'); -- 4 + INSERT INTO t2 VALUES('Y O V O G T P N G T N F I V B U M J M G'); -- 5 + INSERT INTO t2 VALUES('J O B N K N E C H Z R K J O U G M K L S'); -- 6 + INSERT INTO t2 VALUES('S Z S R I Q U A P W R X H K C Z U L S P'); -- 7 + INSERT INTO t2 VALUES('J C H N R C K R V N M O F Z M Z A I H W'); -- 8 + INSERT INTO t2 VALUES('O Y G I S J U U W O D Z F J K N R P R L'); -- 9 + INSERT INTO t2 VALUES('B G L K U R U P V X Z I H V R W C Q A S'); -- 10 + INSERT INTO t2 VALUES('T F T J F F Y V F W N X K Q A Y L X W G'); -- 11 + INSERT INTO t2 VALUES('C J U H B Q X L C M M Y E G V F W V Z C'); -- 12 + INSERT INTO t2 VALUES('B W L T F S G X D P H N G M R I O A X I'); -- 13 + INSERT INTO t2 VALUES('N G Y O K Q K Z N M H U J E D H U W R K'); -- 14 + INSERT INTO t2 VALUES('U D T R U Y F J D S J X E H Q G V A S Z'); -- 15 + INSERT INTO t2 VALUES('M I W P J S H R J D Q I C G P C T P H R'); -- 16 + INSERT INTO t2 VALUES('J M N I S L X Q C A B F C B Y D H V R J'); -- 17 + INSERT INTO t2 VALUES('F V Z W J Q L P X Y E W B U Q N H X K T'); -- 18 + INSERT INTO t2 VALUES('R F S R Y O F Q E I E G H C B H R X Y N'); -- 19 + INSERT INTO t2 VALUES('U Q Q Q T E P D M F X P J G H X C Q D L'); -- 20 +} + +do_execsql_test 2.1 { + CREATE VIRTUAL TABLE ft2 USING fts4(content=t2); + INSERT INTO ft2(ft2) VALUES('rebuild'); + + -- Modify the backing table a bit: Row 17 is missing and the contents + -- of row 20 do not match the FTS index contents. + DELETE FROM t2 WHERE rowid = 17; + UPDATE t2 SET x = 'a b c d e f g h i j' WHERE rowid = 20; +} + +foreach {tn match rowidlist} { + 1 {S} {1 3 6 7 9 10 13 15 16 17 19} + 2 {"S R"} {7 19} + 3 {"N K N"} {6} + 4 {"Q Q"} {20} + 5 {"B Y D"} {17} +} { + do_execsql_test 2.2.1.$tn { + SELECT rowid FROM ft2 WHERE ft2 MATCH $match + } $rowidlist + + do_execsql_test 2.2.2.$tn { + SELECT docid FROM ft2 WHERE ft2 MATCH $match + } $rowidlist +} + +foreach {tn match result} { + 1 {"N K N"} {{J O B N K N E C H Z R K J O U G M K L S}} + 2 {"Q Q"} {{a b c d e f g h i j}} + 3 {"B Y D"} {{}} +} { + do_execsql_test 2.3.$tn { + SELECT * FROM ft2 WHERE ft2 MATCH $match + } $result +} + +foreach {tn match result} { + 1 {"N K N"} {{..O B [N] [K] [N] E..}} + 2 {"B Y D"} {{}} + 3 {"Q Q"} {{a [b] [c] [d] e f..}} +} { + do_execsql_test 2.4.$tn { + SELECT snippet(ft2, '[', ']', '..', -1, 6) FROM ft2 WHERE ft2 MATCH $match + } $result +} + +foreach {tn match result} { + 1 {"N K N"} {{0 0 6 1 0 1 8 1 0 2 10 1}} + 2 {"B Y D"} {{}} + 3 {"Q Q"} {{0 0 2 1 0 0 4 1 0 1 4 1 0 1 6 1}} + 4 {"Q D L"} {{}} +} { + do_execsql_test 2.5.$tn { + SELECT offsets(ft2) FROM ft2 WHERE ft2 MATCH $match + } $result +} + +#------------------------------------------------------------------------- +# The following block of tests - 3.* - test that the FTS index can be +# modified by writing to the table. But that this has no effect on the +# content table. +# + +do_execsql_test 3.1 { + CREATE TABLE t3(x, y); + CREATE VIRTUAL TABLE ft3 USING fts4(content=t3); +} + +do_catchsql_test 3.1.1 { + INSERT INTO ft3 VALUES('a b c', 'd e f'); +} {1 {constraint failed}} +do_execsql_test 3.1.2 { + INSERT INTO ft3(docid, x, y) VALUES(21, 'a b c', 'd e f'); + SELECT rowid FROM ft3 WHERE ft3 MATCH '"a b c"'; +} {21} +do_execsql_test 3.1.3 { SELECT * FROM t3 } {} + +# This DELETE does not work, since there is no row in [t3] to base the +# DELETE on. So the SELECT on [ft3] still returns rowid 21. +do_execsql_test 3.1.4 { + DELETE FROM ft3; + SELECT rowid FROM ft3 WHERE ft3 MATCH '"a b c"'; +} {21} + +# If the row is added to [t3] before the DELETE on [ft3], it works. +do_execsql_test 3.1.5 { + INSERT INTO t3(rowid, x, y) VALUES(21, 'a b c', 'd e f'); + DELETE FROM ft3; + SELECT rowid FROM ft3 WHERE ft3 MATCH '"a b c"'; +} {} +do_execsql_test 3.1.6 { SELECT rowid FROM t3 } {21} + +do_execsql_test 3.2.1 { + INSERT INTO ft3(rowid, x, y) VALUES(0, 'R T M S M', 'A F O K H'); + INSERT INTO ft3(rowid, x, y) VALUES(1, 'C Z J O X', 'U S Q D K'); + INSERT INTO ft3(rowid, x, y) VALUES(2, 'N G H P O', 'N O P O C'); + INSERT INTO ft3(rowid, x, y) VALUES(3, 'V H S D R', 'K N G E C'); + INSERT INTO ft3(rowid, x, y) VALUES(4, 'J T R V U', 'U X S L C'); + INSERT INTO ft3(rowid, x, y) VALUES(5, 'N A Y N G', 'X D G P Y'); + INSERT INTO ft3(rowid, x, y) VALUES(6, 'I Q I S P', 'D R O Q B'); + INSERT INTO ft3(rowid, x, y) VALUES(7, 'T K T Z J', 'B W D G O'); + INSERT INTO ft3(rowid, x, y) VALUES(8, 'Y K F X T', 'D F G V G'); + INSERT INTO ft3(rowid, x, y) VALUES(9, 'E L E T L', 'P W N F Z'); + INSERT INTO ft3(rowid, x, y) VALUES(10, 'O G J G X', 'G J F E P'); + INSERT INTO ft3(rowid, x, y) VALUES(11, 'O L N N Z', 'K E Z F D'); + INSERT INTO ft3(rowid, x, y) VALUES(12, 'R Z M R J', 'X G I M Z'); + INSERT INTO ft3(rowid, x, y) VALUES(13, 'L X N N X', 'R R N S T'); + INSERT INTO ft3(rowid, x, y) VALUES(14, 'F L B J H', 'K W F L C'); + INSERT INTO ft3(rowid, x, y) VALUES(15, 'P E B M V', 'E A A B U'); + INSERT INTO ft3(rowid, x, y) VALUES(16, 'V E C F P', 'L U T V K'); + INSERT INTO ft3(rowid, x, y) VALUES(17, 'T N O Z N', 'T P Q X N'); + INSERT INTO ft3(rowid, x, y) VALUES(18, 'V W U W R', 'H O A A V'); + INSERT INTO ft3(rowid, x, y) VALUES(19, 'A H N L F', 'I G H B O'); +} + +foreach {tn match rowidlist} { + 1 "N A" {5 19} + 2 "x:O" {1 2 10 11 17} + 3 "y:O" {0 2 6 7 18 19} +} { + set res [list] + foreach rowid $rowidlist { lappend res $rowid {} {} } + + do_execsql_test 3.2.2.$tn { + SELECT rowid, * FROM ft3 WHERE ft3 MATCH $match + } $res + do_execsql_test 3.2.3.$tn { + SELECT docid, * FROM ft3 WHERE ft3 MATCH $match + } $res +} + +do_execsql_test 3.3.1 { + INSERT INTO t3(rowid, x, y) VALUES(0, 'R T M S M', 'A F O K H'); + INSERT INTO t3(rowid, x, y) VALUES(1, 'C Z J O X', 'U S Q D K'); + INSERT INTO t3(rowid, x, y) VALUES(2, 'N G H P O', 'N O P O C'); + INSERT INTO t3(rowid, x, y) VALUES(3, 'V H S D R', 'K N G E C'); + INSERT INTO t3(rowid, x, y) VALUES(4, 'J T R V U', 'U X S L C'); + INSERT INTO t3(rowid, x, y) VALUES(5, 'N A Y N G', 'X D G P Y'); + UPDATE ft3 SET x = y, y = x; + DELETE FROM t3; +} + +foreach {tn match rowidlist} { + 1 "N A" {5 19} + 2 "x:O" {0 2 10 11 17} + 3 "y:O" {1 2 6 7 18 19} +} { + set res [list] + foreach rowid $rowidlist { lappend res $rowid {} {} } + + do_execsql_test 3.3.2.$tn { + SELECT rowid, * FROM ft3 WHERE ft3 MATCH $match + } $res + do_execsql_test 3.3.3.$tn { + SELECT docid, * FROM ft3 WHERE ft3 MATCH $match + } $res +} + +do_execsql_test 3.3.1 { + INSERT INTO t3(rowid, x, y) VALUES(15, 'P E B M V', 'E A A B U'); + INSERT INTO t3(rowid, x, y) VALUES(16, 'V E C F P', 'L U T V K'); + INSERT INTO t3(rowid, x, y) VALUES(17, 'T N O Z N', 'T P Q X N'); + INSERT INTO t3(rowid, x, y) VALUES(18, 'V W U W R', 'H O A A V'); + INSERT INTO t3(rowid, x, y) VALUES(19, 'A H N L F', 'I G H B O'); + DELETE FROM ft3; +} + +foreach {tn match rowidlist} { + 1 "N A" {5} + 2 "x:O" {0 2 10 11} + 3 "y:O" {1 2 6 7} +} { + set res [list] + foreach rowid $rowidlist { lappend res $rowid {} {} } + + do_execsql_test 3.3.2.$tn { + SELECT rowid, * FROM ft3 WHERE ft3 MATCH $match + } $res + do_execsql_test 3.3.3.$tn { + SELECT docid, * FROM ft3 WHERE ft3 MATCH $match + } $res +} + + +#------------------------------------------------------------------------- +# Test cases 4.* test the 'rebuild' command. On content=xxx and regular +# FTS tables. +# +do_execsql_test 4.0 { + CREATE TABLE t4(x); + CREATE VIRTUAL TABLE ft4 USING fts4(content=t4); + CREATE VIRTUAL TABLE ft4x USING fts4(x); +} + +do_execsql_test 4.1.1 { + INSERT INTO ft4x(ft4x) VALUES('rebuild'); + INSERT INTO ft4(ft4) VALUES('rebuild'); +} {} +do_execsql_test 4.1.2 { + SELECT id, quote(value) FROM ft4_stat +} {0 X'000000'} +do_execsql_test 4.1.3 { + SELECT id, quote(value) FROM ft4x_stat +} {0 X'000000'} + +do_execsql_test 4.2.1 { + INSERT INTO ft4x VALUES('M G M F T'); + INSERT INTO ft4x VALUES('Z Q C A U'); + INSERT INTO ft4x VALUES('N L L V'); + INSERT INTO ft4x VALUES('T F D X D'); + INSERT INTO ft4x VALUES('Z H I S D'); + + SELECT id, quote(value) FROM ft4x_stat +} {0 X'05182B'} + +do_execsql_test 4.2.2 { + INSERT INTO ft4(rowid, x) SELECT rowid, * FROM ft4x; + SELECT id, quote(value) FROM ft4_stat +} {0 X'05182B'} + +do_execsql_test 4.2.3 { + SELECT docid, quote(size) FROM ft4_docsize +} {1 X'05' 2 X'05' 3 X'04' 4 X'05' 5 X'05'} + +do_execsql_test 4.2.4 { + INSERT INTO ft4x(ft4x) VALUES('rebuild'); + SELECT id, quote(value) FROM ft4x_stat; + SELECT docid, quote(size) FROM ft4x_docsize +} {0 X'05182B' 1 X'05' 2 X'05' 3 X'04' 4 X'05' 5 X'05'} + +do_execsql_test 4.2.5 { + INSERT INTO ft4(ft4) VALUES('rebuild'); + SELECT id, quote(value) FROM ft4_stat; + SELECT docid, quote(size) FROM ft4_docsize +} {0 X'000000'} + +do_execsql_test 4.2.6 { + INSERT INTO t4(rowid, x) SELECT rowid, x FROM ft4x; + INSERT INTO ft4(ft4) VALUES('rebuild'); + SELECT id, quote(value) FROM ft4_stat; + SELECT docid, quote(size) FROM ft4_docsize +} {0 X'05182B' 1 X'05' 2 X'05' 3 X'04' 4 X'05' 5 X'05'} + + +#------------------------------------------------------------------------- +# Test cases 5.* test that the following commands do not create/move or +# delete a %_content table when used with a content=xxx FTS table. +# +do_execsql_test 5.1.1 { + CREATE TABLE t5(a, b, c, d); + CREATE VIRTUAL TABLE ft5 USING fts4(content=t5); + SELECT name FROM sqlite_master WHERE name LIKE '%t5%'; +} { + t5 ft5 ft5_segments ft5_segdir + sqlite_autoindex_ft5_segdir_1 ft5_docsize ft5_stat +} +do_execsql_test 5.1.2 { + ALTER TABLE ft5 RENAME TO ft6; + SELECT name FROM sqlite_master WHERE name LIKE '%t5%'; +} { + t5 +} +do_execsql_test 5.1.3 { + SELECT name FROM sqlite_master WHERE name LIKE '%t6%'; +} { + ft6 ft6_segments ft6_segdir + sqlite_autoindex_ft6_segdir_1 ft6_docsize ft6_stat +} +do_execsql_test 5.1.4 { + INSERT INTO t5 VALUES('a', 'b', 'c', 'd'); + INSERT INTO ft6(ft6) VALUES('rebuild'); + SELECT rowid FROM ft6 WHERE ft6 MATCH 'b'; +} {1} +do_execsql_test 5.1.5 { + DROP TABLE ft6; + SELECT * FROM t5; +} {a b c d} +do_execsql_test 5.1.6 { + SELECT name FROM sqlite_master WHERE name LIKE '%t6%'; +} { +} +do_execsql_test 5.1.7 { + CREATE VIRTUAL TABLE ft5 USING fts4(content=t5); + CREATE TABLE t5_content(a, b); + DROP TABLE ft5; + SELECT name FROM sqlite_master WHERE name LIKE '%t5%'; +} { + t5 t5_content +} + +#------------------------------------------------------------------------- +# Test cases 6.* test +# +do_catchsql_test 6.1.1 { + CREATE VIRTUAL TABLE ft7 USING fts4(content=t7); +} {1 {vtable constructor failed: ft7}} + +do_execsql_test 6.2.1 { + CREATE TABLE t7(one, two); + CREATE VIRTUAL TABLE ft7 USING fts4(content=t7); + INSERT INTO t7 VALUES('A B', 'B A'); + INSERT INTO t7 VALUES('C D', 'A A'); + SELECT * FROM ft7; +} { + {A B} {B A} {C D} {A A} +} + +do_catchsql_test 6.2.2 { + DROP TABLE t7; + SELECT * FROM ft7; +} {1 {SQL logic error or missing database}} + +db close +sqlite3 db test.db +do_execsql_test 6.2.3 { + SELECT name FROM sqlite_master WHERE name LIKE '%t7%' +} { + ft7 ft7_segments ft7_segdir sqlite_autoindex_ft7_segdir_1 + ft7_docsize ft7_stat +} +do_catchsql_test 6.2.4 { + SELECT * FROM ft7; +} {1 {vtable constructor failed: ft7}} +do_execsql_test 6.2.5 { + CREATE TABLE t7(x, y); + INSERT INTO t7 VALUES('A B', 'B A'); + INSERT INTO t7 VALUES('C D', 'A A'); + SELECT * FROM ft7; +} { + {A B} {B A} {C D} {A A} +} + +do_execsql_test 6.2.6 { + INSERT INTO ft7(ft7) VALUES('rebuild'); + SELECT rowid FROM ft7 WHERE ft7 MATCH '"A A"'; +} {2} + +do_execsql_test 6.2.7 { + DROP TABLE t7; + CREATE TABLE t7(x); +} +do_catchsql_test 6.2.8 { + SELECT * FROM ft7 WHERE ft7 MATCH '"A A"'; +} {1 {SQL logic error or missing database}} +do_catchsql_test 6.2.9 { + SELECT * FROM ft7 WHERE ft7 MATCH '"A A"'; +} {1 {SQL logic error or missing database}} + +db close +sqlite3 db test.db +do_catchsql_test 6.2.10 { + SELECT rowid FROM ft7 WHERE ft7 MATCH '"A A"'; +} {0 2} +do_catchsql_test 6.2.11 { + SELECT rowid, * FROM ft7 WHERE ft7 MATCH '"A A"'; +} {0 {2 {}}} + +#------------------------------------------------------------------------- +# Test cases 7.* +# +do_execsql_test 7.1.1 { + CREATE VIRTUAL TABLE ft8 USING fts4(content=nosuchtable, x); + INSERT INTO ft8(docid, x) VALUES(13, 'U O N X G'); + INSERT INTO ft8(docid, x) VALUES(14, 'C J J U B'); + INSERT INTO ft8(docid, x) VALUES(15, 'N J Y G X'); + INSERT INTO ft8(docid, x) VALUES(16, 'R Y D O R'); + INSERT INTO ft8(docid, x) VALUES(17, 'I Y T Q O'); +} + +do_execsql_test 7.1.2 { + SELECT docid FROM ft8 WHERE ft8 MATCH 'N'; +} {13 15} + +finish_test diff --git a/test/permutations.test b/test/permutations.test index c911e6f062..7c3b026c67 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -180,10 +180,12 @@ test_suite "fts3" -prefix "" -description { fts3defer.test fts3defer2.test fts3e.test fts3expr.test fts3expr2.test fts3near.test fts3query.test fts3shared.test fts3snippet.test fts3sort.test - fts3fault.test fts3malloc.test fts3matchinfo.test - fts3aux1.test fts3comp1.test fts3auto.test + fts4aa.test fts4content.test + fts3conf.test fts3prefix.test fts3fault2.test fts3corrupt.test + fts3corrupt2.test + fts3first.test }