From d8e454e19ac21660a134c7167c0e5787111c7504 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 4 Oct 2011 11:22:59 +0000 Subject: [PATCH 01/11] Add experimental 'content' option to FTS4. FossilOrigin-Name: 1d27ea741f61c624e18bdc6a3b1c2d8574a64ddc --- ext/fts3/fts3.c | 251 +++++++++++++++++++++------ ext/fts3/fts3Int.h | 2 + ext/fts3/fts3_snippet.c | 4 +- ext/fts3/fts3_write.c | 142 ++++++++++++++-- manifest | 26 +-- manifest.uuid | 2 +- test/fts4content.test | 366 ++++++++++++++++++++++++++++++++++++++++ test/permutations.test | 3 +- 8 files changed, 709 insertions(+), 87 deletions(-) create mode 100644 test/fts4content.test diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 29e071a50f..c188945b28 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -468,6 +468,7 @@ static int fts3DisconnectMethod(sqlite3_vtab *pVtab){ sqlite3_free(p->zSegmentsTbl); sqlite3_free(p->zReadExprlist); sqlite3_free(p->zWriteExprlist); + sqlite3_free(p->zContentTbl); /* Invoke the tokenizer destructor to free the tokenizer. */ p->pTokenizer->pModule->xDestroy(p->pTokenizer); @@ -507,16 +508,19 @@ static void fts3DbExec( ** The xDestroy() virtual table method. */ static int fts3DestroyMethod(sqlite3_vtab *pVtab){ - int rc = SQLITE_OK; /* Return code */ Fts3Table *p = (Fts3Table *)pVtab; - sqlite3 *db = p->db; + int rc = SQLITE_OK; /* Return code */ + const char *zDb = p->zDb; /* Name of database (e.g. "main", "temp") */ + sqlite3 *db = p->db; /* Database handle */ /* Drop the shadow tables */ - fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_content'", p->zDb, p->zName); - fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segments'", p->zDb,p->zName); - fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segdir'", p->zDb, p->zName); - fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_docsize'", p->zDb, p->zName); - fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_stat'", p->zDb, p->zName); + if( p->zContentTbl==0 ){ + fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_content'", zDb, p->zName); + } + fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segments'", zDb,p->zName); + fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segdir'", zDb, p->zName); + fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_docsize'", zDb, p->zName); + fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_stat'", zDb, p->zName); /* If everything has worked, invoke fts3DisconnectMethod() to free the ** memory associated with the Fts3Table structure and return SQLITE_OK. @@ -578,23 +582,27 @@ static void fts3DeclareVtab(int *pRc, Fts3Table *p){ static int fts3CreateTables(Fts3Table *p){ int rc = SQLITE_OK; /* Return code */ int i; /* Iterator variable */ - char *zContentCols; /* Columns of %_content table */ sqlite3 *db = p->db; /* The database connection */ - /* Create a list of user columns for the content table */ - zContentCols = sqlite3_mprintf("docid INTEGER PRIMARY KEY"); - for(i=0; zContentCols && inColumn; i++){ - char *z = p->azColumn[i]; - zContentCols = sqlite3_mprintf("%z, 'c%d%q'", zContentCols, i, z); - } - if( zContentCols==0 ) rc = SQLITE_NOMEM; + if( p->zContentTbl==0 ){ + char *zContentCols; /* Columns of %_content table */ + + /* Create a list of user columns for the content table */ + zContentCols = sqlite3_mprintf("docid INTEGER PRIMARY KEY"); + for(i=0; zContentCols && inColumn; i++){ + char *z = p->azColumn[i]; + zContentCols = sqlite3_mprintf("%z, 'c%d%q'", zContentCols, i, z); + } + if( zContentCols==0 ) rc = SQLITE_NOMEM; + + /* Create the content table */ + fts3DbExec(&rc, db, + "CREATE TABLE %Q.'%q_content'(%s)", + p->zDb, p->zName, zContentCols + ); + sqlite3_free(zContentCols); + } - /* Create the content table */ - fts3DbExec(&rc, db, - "CREATE TABLE %Q.'%q_content'(%s)", - p->zDb, p->zName, zContentCols - ); - sqlite3_free(zContentCols); /* Create other tables */ fts3DbExec(&rc, db, "CREATE TABLE %Q.'%q_segments'(blockid INTEGER PRIMARY KEY, block BLOB);", @@ -745,8 +753,8 @@ static char *fts3QuoteId(char const *zInput){ } /* -** Return a list of comma separated SQL expressions that could be used -** in a SELECT statement such as the following: +** Return a list of comma separated SQL expressions and a FROM clause that +** could be used in a SELECT statement such as the following: ** ** SELECT FROM %_content AS x ... ** @@ -757,7 +765,7 @@ static char *fts3QuoteId(char const *zInput){ ** table has the three user-defined columns "a", "b", and "c", the following ** string is returned: ** -** "docid, unzip(x.'a'), unzip(x.'b'), unzip(x.'c')" +** "docid, unzip(x.'a'), unzip(x.'b'), unzip(x.'c') FROM %_content AS x" ** ** The pointer returned points to a buffer allocated by sqlite3_malloc(). It ** is the responsibility of the caller to eventually free it. @@ -773,16 +781,28 @@ static char *fts3ReadExprList(Fts3Table *p, const char *zFunc, int *pRc){ char *zFunction; int i; - if( !zFunc ){ - zFunction = ""; + if( p->zContentTbl==0 ){ + if( !zFunc ){ + zFunction = ""; + }else{ + zFree = zFunction = fts3QuoteId(zFunc); + } + fts3Appendf(pRc, &zRet, "docid"); + for(i=0; inColumn; i++){ + fts3Appendf(pRc, &zRet, ",%s(x.'c%d%q')", zFunction, i, p->azColumn[i]); + } + sqlite3_free(zFree); }else{ - zFree = zFunction = fts3QuoteId(zFunc); + fts3Appendf(pRc, &zRet, "rowid"); + for(i=0; inColumn; i++){ + fts3Appendf(pRc, &zRet, ", x.'%q'", p->azColumn[i]); + } } - fts3Appendf(pRc, &zRet, "docid"); - for(i=0; inColumn; i++){ - fts3Appendf(pRc, &zRet, ",%s(x.'c%d%q')", zFunction, i, p->azColumn[i]); - } - sqlite3_free(zFree); + fts3Appendf(pRc, &zRet, "FROM '%q'.'%q%s' AS x", + p->zDb, + (p->zContentTbl ? p->zContentTbl : p->zName), + (p->zContentTbl ? "" : "_content") + ); return zRet; } @@ -906,6 +926,91 @@ static int fts3PrefixParameter( return SQLITE_OK; } +/* +** This function is called when initializing an FTS4 table that uses the +** content=xxx option. It determines the number of and names of the columns +** of the new FTS4 table. +** +** The third argument passed to this function is the value passed to the +** config=xxx option (i.e. "xxx"). This function queries the database for +** a table of that name. If found, the output variables are populated +** as follows: +** +** *pnCol: Set to the number of columns table xxx has, +** +** *pnStr: Set to the total amount of space required to store a copy +** of each columns name, including the nul-terminator. +** +** *pazCol: Set to point to an array of *pnCol strings. Each string is +** the name of the corresponding column in table xxx. The array +** and its contents are allocated using a single allocation. It +** is the responsibility of the caller to free this allocation +** by eventually passing the *pazCol value to sqlite3_free(). +** +** If the table cannot be found, an error code is returned and the output +** variables are undefined. Or, if an OOM is encountered, SQLITE_NOMEM is +** returned (and the output variables are undefined). +*/ +static int fts3ContentColumns( + sqlite3 *db, /* Database handle */ + const char *zDb, /* Name of db (i.e. "main", "temp" etc.) */ + const char *zTbl, /* Name of content table */ + const char ***pazCol, /* OUT: Malloc'd array of column names */ + int *pnCol, /* OUT: Size of array *pazCol */ + int *pnStr /* OUT: Bytes of string content */ +){ + int rc = SQLITE_OK; /* Return code */ + char *zSql; /* "SELECT *" statement on zTbl */ + sqlite3_stmt *pStmt = 0; /* Compiled version of zSql */ + + zSql = sqlite3_mprintf("SELECT * FROM %Q.%Q", zDb, zTbl); + if( !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); + } + sqlite3_free(zSql); + + if( rc==SQLITE_OK ){ + const char **azCol; /* Output array */ + int nStr = 0; /* Size of all column names (incl. 0x00) */ + int nCol; /* Number of table columns */ + int i; /* Used to iterate through columns */ + + /* Loop through the returned columns. Set nStr to the number of bytes of + ** space required to store a copy of each column name, including the + ** nul-terminator byte. */ + nCol = sqlite3_column_count(pStmt); + for(i=0; i MATCHINFO */ - { "prefix", 6, 0 }, /* 1 -> PREFIX */ - { "compress", 8, 0 }, /* 2 -> COMPRESS */ - { "uncompress", 10, 0 }, /* 3 -> UNCOMPRESS */ - { "order", 5, 0 } /* 4 -> ORDER */ + { "matchinfo", 9 }, /* 0 -> MATCHINFO */ + { "prefix", 6 }, /* 1 -> PREFIX */ + { "compress", 8 }, /* 2 -> COMPRESS */ + { "uncompress", 10 }, /* 3 -> UNCOMPRESS */ + { "order", 5 }, /* 4 -> ORDER */ + { "content", 7 } /* 5 -> CONTENT */ }; int iOpt; @@ -1052,6 +1158,12 @@ static int fts3InitVtab( } bDescIdx = (zVal[0]=='d' || zVal[0]=='D'); break; + + case 5: /* CONTENT */ + sqlite3_free(zUncompress); + zContent = zVal; + zVal = 0; + break; } } sqlite3_free(zVal); @@ -1064,6 +1176,27 @@ static int fts3InitVtab( aCol[nCol++] = z; } } + + /* If a content=xxx option was specified, the following: + ** + ** 1. Ignore any compress= and uncompress= options. + ** + ** 2. Ignore any column names that were specified as part of the + ** the CREATE VIRTUAL TABLE statement. + ** + ** 3. Determine the actual column names to use for the FTS table + ** based on the columns of the content= table. + */ + if( rc==SQLITE_OK && zContent ){ + sqlite3_free(aCol); + sqlite3_free(zCompress); + sqlite3_free(zUncompress); + zCompress = 0; + zUncompress = 0; + aCol = 0; + rc = fts3ContentColumns(db, argv[1], zContent, &aCol, &nCol, &nString); + assert( rc!=SQLITE_OK || nCol>0 ); + } if( rc!=SQLITE_OK ) goto fts3_init_out; if( nCol==0 ){ @@ -1108,6 +1241,8 @@ static int fts3InitVtab( p->bHasDocsize = (isFts4 && bNoDocsize==0); p->bHasStat = isFts4; p->bDescIdx = bDescIdx; + p->zContentTbl = zContent; + zContent = 0; TESTONLY( p->inTransaction = -1 ); TESTONLY( p->mxSavepoint = -1 ); @@ -1169,6 +1304,7 @@ fts3_init_out: sqlite3_free(aIndex); sqlite3_free(zCompress); sqlite3_free(zUncompress); + sqlite3_free(zContent); sqlite3_free((void *)aCol); if( rc!=SQLITE_OK ){ if( p ){ @@ -1334,11 +1470,16 @@ static int fts3CursorSeek(sqlite3_context *pContext, Fts3Cursor *pCsr){ }else{ int rc = sqlite3_reset(pCsr->pStmt); if( rc==SQLITE_OK ){ - /* If no row was found and no error has occured, then the %_content - ** table is missing a row that is present in the full-text index. - ** The data structures are corrupt. - */ - rc = SQLITE_CORRUPT_VTAB; + Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; + if( p->zContentTbl==0 ){ + /* If no row was found and no error has occured, then the %_content + ** table is missing a row that is present in the full-text index. + ** The data structures are corrupt. + */ + rc = SQLITE_CORRUPT_VTAB; + }else{ + return SQLITE_OK; + } } pCsr->isEof = 1; if( pContext ){ @@ -2713,12 +2854,13 @@ static int fts3FilterMethod( ** row by docid. */ if( idxNum==FTS3_FULLSCAN_SEARCH ){ - const char *zSort = (pCsr->bDesc ? "DESC" : "ASC"); - const char *zTmpl = "SELECT %s FROM %Q.'%q_content' AS x ORDER BY docid %s"; - zSql = sqlite3_mprintf(zTmpl, p->zReadExprlist, p->zDb, p->zName, zSort); + const char *zTmpl = "SELECT %s ORDER BY rowid %s"; + zSql = sqlite3_mprintf(zTmpl, + p->zReadExprlist, (pCsr->bDesc ? "DESC" : "ASC") + ); }else{ - const char *zTmpl = "SELECT %s FROM %Q.'%q_content' AS x WHERE docid = ?"; - zSql = sqlite3_mprintf(zTmpl, p->zReadExprlist, p->zDb, p->zName); + const char *zTmpl = "SELECT %s WHERE rowid = ?"; + zSql = sqlite3_mprintf(zTmpl, p->zReadExprlist); } if( !zSql ) return SQLITE_NOMEM; rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); @@ -2781,7 +2923,7 @@ static int fts3ColumnMethod( sqlite3_result_blob(pContext, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT); }else{ rc = fts3CursorSeek(0, pCsr); - if( rc==SQLITE_OK ){ + if( rc==SQLITE_OK && sqlite3_data_count(pCsr->pStmt)>(iCol+1) ){ sqlite3_result_value(pContext, sqlite3_column_value(pCsr->pStmt, iCol+1)); } } @@ -3079,10 +3221,13 @@ static int fts3RenameMethod( return rc; } - fts3DbExec(&rc, db, - "ALTER TABLE %Q.'%q_content' RENAME TO '%q_content';", - p->zDb, p->zName, zName - ); + if( p->zContentTbl==0 ){ + fts3DbExec(&rc, db, + "ALTER TABLE %Q.'%q_content' RENAME TO '%q_content';", + p->zDb, p->zName, zName + ); + } + if( p->bHasDocsize ){ fts3DbExec(&rc, db, "ALTER TABLE %Q.'%q_docsize' RENAME TO '%q_docsize';", diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index ed8043adf6..b4f26fc7a5 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -184,6 +184,7 @@ struct Fts3Table { int nColumn; /* number of named columns in virtual table */ char **azColumn; /* column names. malloced */ sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */ + char *zContentTbl; /* content=xxx option, or NULL */ /* Precompiled statements used by the implementation. Each of these ** statements is run and reset within a single virtual table API call. @@ -245,6 +246,7 @@ struct Fts3Cursor { i16 eSearch; /* Search strategy (see below) */ u8 isEof; /* True if at End Of Results */ u8 isRequireSeek; /* True if must seek pStmt to %_content row */ + u8 isNullRow; /* True for a row of NULLs */ sqlite3_stmt *pStmt; /* Prepared statement in use by the cursor */ Fts3Expr *pExpr; /* Parsed MATCH query string */ int nPhrase; /* Number of matchable phrases in query */ diff --git a/ext/fts3/fts3_snippet.c b/ext/fts3/fts3_snippet.c index b569eb131b..13d0ca3551 100644 --- a/ext/fts3/fts3_snippet.c +++ b/ext/fts3/fts3_snippet.c @@ -1409,7 +1409,7 @@ void sqlite3Fts3Offsets( if( !pTerm ){ /* All offsets for this column have been gathered. */ - break; + rc = SQLITE_DONE; }else{ assert( iCurrent<=iMinPos ); if( 0==(0xFE&*pTerm->pList) ){ @@ -1426,7 +1426,7 @@ void sqlite3Fts3Offsets( "%d %d %d %d ", iCol, pTerm-sCtx.aTerm, iStart, iEnd-iStart ); rc = fts3StringAppend(&res, aBuffer, -1); - }else if( rc==SQLITE_DONE ){ + }else if( rc==SQLITE_DONE && pTab->zContentTbl==0 ){ rc = SQLITE_CORRUPT_VTAB; } } diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 36f2249e12..46a1ead2d8 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -256,7 +256,7 @@ static int fts3SqlStmt( /* 4 */ "DELETE FROM %Q.'%q_segdir'", /* 5 */ "DELETE FROM %Q.'%q_docsize'", /* 6 */ "DELETE FROM %Q.'%q_stat'", -/* 7 */ "SELECT %s FROM %Q.'%q_content' AS x WHERE rowid=?", +/* 7 */ "SELECT %s WHERE rowid=?", /* 8 */ "SELECT (SELECT max(idx) FROM %Q.'%q_segdir' WHERE level = ?) + 1", /* 9 */ "INSERT INTO %Q.'%q_segments'(blockid, block) VALUES(?, ?)", /* 10 */ "SELECT coalesce((SELECT max(blockid) FROM %Q.'%q_segments') + 1, 1)", @@ -298,7 +298,7 @@ static int fts3SqlStmt( if( eStmt==SQL_CONTENT_INSERT ){ zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName, p->zWriteExprlist); }else if( eStmt==SQL_SELECT_CONTENT_BY_ROWID ){ - zSql = sqlite3_mprintf(azSql[eStmt], p->zReadExprlist, p->zDb, p->zName); + zSql = sqlite3_mprintf(azSql[eStmt], p->zReadExprlist); }else{ zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName); } @@ -780,6 +780,18 @@ static int fts3InsertData( int rc; /* Return code */ sqlite3_stmt *pContentInsert; /* INSERT INTO %_content VALUES(...) */ + if( p->zContentTbl ){ + sqlite3_value *pRowid = apVal[p->nColumn+3]; + if( sqlite3_value_type(pRowid)==SQLITE_NULL ){ + pRowid = apVal[1]; + } + if( sqlite3_value_type(pRowid)!=SQLITE_INTEGER ){ + return SQLITE_CONSTRAINT; + } + *piDocid = sqlite3_value_int64(pRowid); + return SQLITE_OK; + } + /* Locate the statement handle used to insert data into the %_content ** table. The SQL for this statement is: ** @@ -830,14 +842,16 @@ static int fts3InsertData( ** Remove all data from the FTS3 table. Clear the hash table containing ** pending terms. */ -static int fts3DeleteAll(Fts3Table *p){ +static int fts3DeleteAll(Fts3Table *p, int bContent){ int rc = SQLITE_OK; /* Return code */ /* Discard the contents of the pending-terms hash table. */ sqlite3Fts3PendingTermsClear(p); - /* Delete everything from the %_content, %_segments and %_segdir tables. */ - fts3SqlExec(&rc, p, SQL_DELETE_ALL_CONTENT, 0); + /* Delete everything from the shadow tables. Except, leave %_content as + ** is if bContent is false. */ + assert( p->zContentTbl==0 || bContent==0 ); + if( bContent ) fts3SqlExec(&rc, p, SQL_DELETE_ALL_CONTENT, 0); fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGMENTS, 0); fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGDIR, 0); if( p->bHasDocsize ){ @@ -2125,12 +2139,18 @@ static void fts3SegWriterFree(SegmentWriter *pWriter){ static int fts3IsEmpty(Fts3Table *p, sqlite3_value *pRowid, int *pisEmpty){ sqlite3_stmt *pStmt; int rc; - rc = fts3SqlStmt(p, SQL_IS_EMPTY, &pStmt, &pRowid); - if( rc==SQLITE_OK ){ - if( SQLITE_ROW==sqlite3_step(pStmt) ){ - *pisEmpty = sqlite3_column_int(pStmt, 0); + if( p->zContentTbl ){ + /* If using the content=xxx option, assume the table is never empty */ + *pisEmpty = 0; + rc = SQLITE_OK; + }else{ + rc = fts3SqlStmt(p, SQL_IS_EMPTY, &pStmt, &pRowid); + if( rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + *pisEmpty = sqlite3_column_int(pStmt, 0); + } + rc = sqlite3_reset(pStmt); } - rc = sqlite3_reset(pStmt); } return rc; } @@ -2787,9 +2807,9 @@ static void fts3DecodeIntArray( ** a blob of varints. */ static void fts3InsertDocsize( - int *pRC, /* Result code */ - Fts3Table *p, /* Table into which to insert */ - u32 *aSz /* Sizes of each column */ + int *pRC, /* Result code */ + Fts3Table *p, /* Table into which to insert */ + u32 *aSz /* Sizes of each column, in tokens */ ){ char *pBlob; /* The BLOB encoding of the document size */ int nBlob; /* Number of bytes in the BLOB */ @@ -2911,6 +2931,84 @@ static int fts3DoOptimize(Fts3Table *p, int bReturnDone){ return (rc==SQLITE_OK && bReturnDone && bSeenDone) ? SQLITE_DONE : rc; } +/* +** This function is called when the user executes the following statement: +** +** INSERT INTO () VALUES('rebuild'); +** +** The entire FTS index is discarded and rebuilt. If the table is one +** created using the content=xxx option, then the new index is based on +** the current contents of the xxx table. Otherwise, it is rebuilt based +** on the contents of the %_content table. +*/ +static int fts3DoRebuild(Fts3Table *p){ + int rc; /* Return Code */ + + rc = fts3DeleteAll(p, 0); + if( rc==SQLITE_OK ){ + u32 *aSz = 0; + u32 *aSzIns; + u32 *aSzDel; + sqlite3_stmt *pStmt = 0; + int nEntry = 0; + + /* Compose and prepare an SQL statement to loop through the content table */ + char *zSql = sqlite3_mprintf("SELECT %s" , p->zReadExprlist); + if( !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + } + + if( rc==SQLITE_OK ){ + int nByte = sizeof(u32) * (p->nColumn+1)*3; + aSz = (u32 *)sqlite3_malloc(nByte); + if( aSz==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(aSz, 0, nByte); + aSzIns = &aSz[p->nColumn+1]; + aSzDel = &aSzIns[p->nColumn+1]; + } + } + + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + int iCol; + rc = fts3PendingTermsDocid(p, sqlite3_column_int64(pStmt, 0)); + aSz[p->nColumn] = 0; + for(iCol=0; rc==SQLITE_OK && iColnColumn; iCol++){ + const char *z = (const char *) sqlite3_column_text(pStmt, iCol+1); + rc = fts3PendingTermsAdd(p, z, iCol, &aSz[iCol]); + aSz[p->nColumn] += sqlite3_column_bytes(pStmt, iCol+1); + } + if( p->bHasDocsize ){ + fts3InsertDocsize(&rc, p, aSz); + } + if( rc!=SQLITE_OK ){ + sqlite3_finalize(pStmt); + pStmt = 0; + }else{ + nEntry++; + for(iCol=0; iCol<=p->nColumn; iCol++){ + aSzIns[iCol] += aSz[iCol]; + } + } + } + if( p->bHasStat ){ + fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nEntry); + } + sqlite3_free(aSz); + + if( pStmt ){ + assert( rc==SQLITE_OK ); + rc = sqlite3_finalize(pStmt); + } + } + + return rc; +} + /* ** Handle a 'special' INSERT of the form: ** @@ -2928,6 +3026,8 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){ return SQLITE_NOMEM; }else if( nVal==8 && 0==sqlite3_strnicmp(zVal, "optimize", 8) ){ rc = fts3DoOptimize(p, 0); + }else if( nVal==7 && 0==sqlite3_strnicmp(zVal, "rebuild", 7) ){ + rc = fts3DoRebuild(p); #ifdef SQLITE_TEST }else if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){ p->nNodeSize = atoi(&zVal[9]); @@ -3099,14 +3199,18 @@ static int fts3DeleteByRowid( /* Deleting this row means the whole table is empty. In this case ** delete the contents of all three tables and throw away any ** data in the pendingTerms hash table. */ - rc = fts3DeleteAll(p); + rc = fts3DeleteAll(p, 1); *pnDoc = *pnDoc - 1; }else{ sqlite3_int64 iRemove = sqlite3_value_int64(pRowid); rc = fts3PendingTermsDocid(p, iRemove); fts3DeleteTerms(&rc, p, pRowid, aSzDel); - fts3SqlExec(&rc, p, SQL_DELETE_CONTENT, &pRowid); - if( sqlite3_changes(p->db) ) *pnDoc = *pnDoc - 1; + if( p->zContentTbl==0 ){ + fts3SqlExec(&rc, p, SQL_DELETE_CONTENT, &pRowid); + if( sqlite3_changes(p->db) ) *pnDoc = *pnDoc - 1; + }else{ + *pnDoc = *pnDoc - 1; + } if( p->bHasDocsize ){ fts3SqlExec(&rc, p, SQL_DELETE_DOCSIZE, &pRowid); } @@ -3167,7 +3271,7 @@ int sqlite3Fts3UpdateMethod( ** detect the conflict and return SQLITE_CONSTRAINT before beginning to ** modify the database file. */ - if( nArg>1 ){ + if( nArg>1 && p->zContentTbl==0 ){ /* Find the value object that holds the new rowid value. */ sqlite3_value *pNewRowid = apVal[3+p->nColumn]; if( sqlite3_value_type(pNewRowid)==SQLITE_NULL ){ @@ -3219,7 +3323,9 @@ int sqlite3Fts3UpdateMethod( if( nArg>1 && rc==SQLITE_OK ){ if( bInsertDone==0 ){ rc = fts3InsertData(p, apVal, pRowid); - if( rc==SQLITE_CONSTRAINT ) rc = SQLITE_CORRUPT_VTAB; + if( rc==SQLITE_CONSTRAINT && p->zContentTbl==0 ){ + rc = SQLITE_CORRUPT_VTAB; + } } if( rc==SQLITE_OK && (!isRemove || *pRowid!=iRemove) ){ rc = fts3PendingTermsDocid(p, *pRowid); diff --git a/manifest b/manifest index 916f02c341..6ca44ed83d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sMSVC\smakefile\sto\sallow\stargets\sto\sbe\sbuilt\swith\ssupport\sfor\sICU. -D 2011-10-02T05:23:16.470 +C Add\sexperimental\s'content'\soption\sto\sFTS4. +D 2011-10-04T11:22:59.677 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a162fe39e249b8ed4a65ee947c30152786cfe897 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -62,22 +62,22 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 195e4da669741c1f097434ec48c0ba5739193af9 +F ext/fts3/fts3.c 626fcf477e531b470f06aae3d427abcda2d5d31c F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h 30063fdd0bc433b5db1532e3a363cb0f2f7e8eb3 +F ext/fts3/fts3Int.h a335d671d42ca07528a93c5b830e48e0205b26bc F ext/fts3/fts3_aux.c 0ebfa7b86cf8ff6a0861605fcc63b83ec1b70691 F ext/fts3/fts3_expr.c 23791de01b3a5d313d76e02befd2601d4096bc2b F ext/fts3/fts3_hash.c 8dd2d06b66c72c628c2732555a32bc0943114914 F ext/fts3/fts3_hash.h 8331fb2206c609f9fc4c4735b9ab5ad6137c88ec F ext/fts3/fts3_icu.c 6c8f395cdf9e1e3afa7fadb7e523dbbf381c6dfa F ext/fts3/fts3_porter.c 8d946908f4812c005d3d33fcbe78418b1f4eb70c -F ext/fts3/fts3_snippet.c 58b2ba2b934c1e2a2f6ac857d7f3c7e1a14b4532 +F ext/fts3/fts3_snippet.c 8838a1de5f7df3a559596870caaa4a9895248998 F ext/fts3/fts3_term.c a5457992723455a58804cb75c8cbd8978db5c2ef F ext/fts3/fts3_test.c 24fa13f330db011500acb95590da9eee24951894 F ext/fts3/fts3_tokenizer.c 9ff7ec66ae3c5c0340fa081958e64f395c71a106 F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3 F ext/fts3/fts3_tokenizer1.c 0dde8f307b8045565cf63797ba9acfaff1c50c68 -F ext/fts3/fts3_write.c 194829c8fd024a448fc899e5ff02a8ed06595529 +F ext/fts3/fts3_write.c 9e14eb54310345b441d7a8c83bc3ae474230bea6 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 @@ -486,6 +486,7 @@ F test/fts3shared.test 8bb266521d7c5495c0ae522bb4d376ad5387d4a2 F test/fts3snippet.test 8e956051221a34c7daeb504f023cb54d5fa5a8b2 F test/fts3sort.test 9a5176c9317bb545ec5f144d62e6fedb4da6c66e F test/fts4aa.test 6e7f90420b837b2c685f3bcbe84c868492d40a68 +F test/fts4content.test fbafb7160eef1560b6a254b1ad5631acccd388a1 F test/func.test 6c5ce11e3a0021ca3c0649234e2d4454c89110ca F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f F test/func3.test 001021e5b88bd02a3b365a5c5fd8f6f49d39744a @@ -620,7 +621,7 @@ F test/pageropt.test 8146bf448cf09e87bb1867c2217b921fb5857806 F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16 F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 -F test/permutations.test ad17319066a90e2db71823c3ff104795ffc71b31 +F test/permutations.test 7b1c4cb40af3712e42364c82390ace0df207b5e0 F test/pragma.test c8108e01da04f16e67e5754e610bc62c1b993f6c F test/pragma2.test 3a55f82b954242c642f8342b17dffc8b47472947 F test/printf.test 05970cde31b1a9f54bd75af60597be75a5c54fea @@ -965,7 +966,10 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2 -P 9ddfe1e41300413bc9af7e5ce0ec9d1daf9136b1 -R 03863d4dd01ba5995289085115b33375 -U mistachkin -Z 7dfdbd4e337132033acb3bc17ce78d60 +P eb5da5e1dbe9c198095036827318fb381441cbd0 +R 7991a3071fb34636c7b5ed4b010ad95a +T *branch * fts4-content +T *sym-fts4-content * +T -sym-trunk * +U dan +Z 48ff718a3615a321d5bc0764cb014ed4 diff --git a/manifest.uuid b/manifest.uuid index c4bc7f0c61..be6a47c58b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -eb5da5e1dbe9c198095036827318fb381441cbd0 \ No newline at end of file +1d27ea741f61c624e18bdc6a3b1c2d8574a64ddc \ No newline at end of file diff --git a/test/fts4content.test b/test/fts4content.test new file mode 100644 index 0000000000..6ac5a6806a --- /dev/null +++ b/test/fts4content.test @@ -0,0 +1,366 @@ +# 2011 October 03 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the content=xxx FTS4 option. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set ::testprefix fts4content + +# If SQLITE_ENABLE_FTS3 is defined, omit this file. +ifcapable !fts3 { + finish_test + return +} + +do_execsql_test 1.1 { + CREATE TABLE t1(a, b, c); + INSERT INTO t1 VALUES('w x', 'x y', 'y z'); + CREATE VIRTUAL TABLE ft1 USING fts4(content=t1); +} + +do_execsql_test 1.2 { + PRAGMA table_info(ft1); +} { + 0 a {} 0 {} 0 + 1 b {} 0 {} 0 + 2 c {} 0 {} 0 +} + +do_execsql_test 1.3 { SELECT *, rowid FROM ft1 } {{w x} {x y} {y z} 1} +do_execsql_test 1.4 { SELECT a, c FROM ft1 WHERE rowid=1 } {{w x} {y z}} + +do_execsql_test 1.5 { INSERT INTO ft1(ft1) VALUES('rebuild') } {} +do_execsql_test 1.6 { SELECT rowid FROM ft1 WHERE ft1 MATCH 'x' } {1} +do_execsql_test 1.7 { SELECT rowid FROM ft1 WHERE ft1 MATCH 'a' } {} + +#------------------------------------------------------------------------- +# The following block of tests - 2.* - test that a content=xxx FTS table +# can be queried. Also tested are cases where rows identified in the FTS +# are missing from the content table, and cases where the index is +# inconsistent with the content table. +# +do_execsql_test 2.0 { + CREATE TABLE t2(x); + INSERT INTO t2 VALUES('O S W W F U C R Q I C N P Z Y Y E Y Y E'); -- 1 + INSERT INTO t2 VALUES('Y X U V L B E H Y J C Y A I A P V F V K'); -- 2 + INSERT INTO t2 VALUES('P W I N J H I I N I F B K D U Q B Z S F'); -- 3 + INSERT INTO t2 VALUES('N R O R H J R H G M D I U U B O M P A U'); -- 4 + INSERT INTO t2 VALUES('Y O V O G T P N G T N F I V B U M J M G'); -- 5 + INSERT INTO t2 VALUES('J O B N K N E C H Z R K J O U G M K L S'); -- 6 + INSERT INTO t2 VALUES('S Z S R I Q U A P W R X H K C Z U L S P'); -- 7 + INSERT INTO t2 VALUES('J C H N R C K R V N M O F Z M Z A I H W'); -- 8 + INSERT INTO t2 VALUES('O Y G I S J U U W O D Z F J K N R P R L'); -- 9 + INSERT INTO t2 VALUES('B G L K U R U P V X Z I H V R W C Q A S'); -- 10 + INSERT INTO t2 VALUES('T F T J F F Y V F W N X K Q A Y L X W G'); -- 11 + INSERT INTO t2 VALUES('C J U H B Q X L C M M Y E G V F W V Z C'); -- 12 + INSERT INTO t2 VALUES('B W L T F S G X D P H N G M R I O A X I'); -- 13 + INSERT INTO t2 VALUES('N G Y O K Q K Z N M H U J E D H U W R K'); -- 14 + INSERT INTO t2 VALUES('U D T R U Y F J D S J X E H Q G V A S Z'); -- 15 + INSERT INTO t2 VALUES('M I W P J S H R J D Q I C G P C T P H R'); -- 16 + INSERT INTO t2 VALUES('J M N I S L X Q C A B F C B Y D H V R J'); -- 17 + INSERT INTO t2 VALUES('F V Z W J Q L P X Y E W B U Q N H X K T'); -- 18 + INSERT INTO t2 VALUES('R F S R Y O F Q E I E G H C B H R X Y N'); -- 19 + INSERT INTO t2 VALUES('U Q Q Q T E P D M F X P J G H X C Q D L'); -- 20 +} + +do_execsql_test 2.1 { + CREATE VIRTUAL TABLE ft2 USING fts4(content=t2); + INSERT INTO ft2(ft2) VALUES('rebuild'); + + -- Modify the backing table a bit: Row 17 is missing and the contents + -- of row 20 do not match the FTS index contents. + DELETE FROM t2 WHERE rowid = 17; + UPDATE t2 SET x = 'a b c d e f g h i j' WHERE rowid = 20; +} + +foreach {tn match rowidlist} { + 1 {S} {1 3 6 7 9 10 13 15 16 17 19} + 2 {"S R"} {7 19} + 3 {"N K N"} {6} + 4 {"Q Q"} {20} + 5 {"B Y D"} {17} +} { + do_execsql_test 2.2.1.$tn { + SELECT rowid FROM ft2 WHERE ft2 MATCH $match + } $rowidlist + + do_execsql_test 2.2.2.$tn { + SELECT docid FROM ft2 WHERE ft2 MATCH $match + } $rowidlist +} + +foreach {tn match result} { + 1 {"N K N"} {{J O B N K N E C H Z R K J O U G M K L S}} + 2 {"Q Q"} {{a b c d e f g h i j}} + 3 {"B Y D"} {{}} +} { + do_execsql_test 2.3.$tn { + SELECT * FROM ft2 WHERE ft2 MATCH $match + } $result +} + +foreach {tn match result} { + 1 {"N K N"} {{..O B [N] [K] [N] E..}} + 2 {"B Y D"} {{}} + 3 {"Q Q"} {{a [b] [c] [d] e f..}} +} { + do_execsql_test 2.4.$tn { + SELECT snippet(ft2, '[', ']', '..', -1, 6) FROM ft2 WHERE ft2 MATCH $match + } $result +} + +foreach {tn match result} { + 1 {"N K N"} {{0 0 6 1 0 1 8 1 0 2 10 1}} + 2 {"B Y D"} {{}} + 3 {"Q Q"} {{0 0 2 1 0 0 4 1 0 1 4 1 0 1 6 1}} + 4 {"Q D L"} {{}} +} { + do_execsql_test 2.5.$tn { + SELECT offsets(ft2) FROM ft2 WHERE ft2 MATCH $match + } $result +} + +#------------------------------------------------------------------------- +# The following block of tests - 3.* - test that the FTS index can be +# modified by writing to the table. But that this has no effect on the +# content table. +# + +do_execsql_test 3.1 { + CREATE TABLE t3(x, y); + CREATE VIRTUAL TABLE ft3 USING fts4(content=t3); +} + +do_catchsql_test 3.1.1 { + INSERT INTO ft3 VALUES('a b c', 'd e f'); +} {1 {constraint failed}} +do_execsql_test 3.1.2 { + INSERT INTO ft3(docid, x, y) VALUES(21, 'a b c', 'd e f'); + SELECT rowid FROM ft3 WHERE ft3 MATCH '"a b c"'; +} {21} +do_execsql_test 3.1.3 { SELECT * FROM t3 } {} + +# This DELETE does not work, since there is no row in [t3] to base the +# DELETE on. So the SELECT on [ft3] still returns rowid 21. +do_execsql_test 3.1.4 { + DELETE FROM ft3; + SELECT rowid FROM ft3 WHERE ft3 MATCH '"a b c"'; +} {21} + +# If the row is added to [t3] before the DELETE on [ft3], it works. +do_execsql_test 3.1.5 { + INSERT INTO t3(rowid, x, y) VALUES(21, 'a b c', 'd e f'); + DELETE FROM ft3; + SELECT rowid FROM ft3 WHERE ft3 MATCH '"a b c"'; +} {} +do_execsql_test 3.1.6 { SELECT rowid FROM t3 } {21} + +do_execsql_test 3.2.1 { + INSERT INTO ft3(rowid, x, y) VALUES(0, 'R T M S M', 'A F O K H'); + INSERT INTO ft3(rowid, x, y) VALUES(1, 'C Z J O X', 'U S Q D K'); + INSERT INTO ft3(rowid, x, y) VALUES(2, 'N G H P O', 'N O P O C'); + INSERT INTO ft3(rowid, x, y) VALUES(3, 'V H S D R', 'K N G E C'); + INSERT INTO ft3(rowid, x, y) VALUES(4, 'J T R V U', 'U X S L C'); + INSERT INTO ft3(rowid, x, y) VALUES(5, 'N A Y N G', 'X D G P Y'); + INSERT INTO ft3(rowid, x, y) VALUES(6, 'I Q I S P', 'D R O Q B'); + INSERT INTO ft3(rowid, x, y) VALUES(7, 'T K T Z J', 'B W D G O'); + INSERT INTO ft3(rowid, x, y) VALUES(8, 'Y K F X T', 'D F G V G'); + INSERT INTO ft3(rowid, x, y) VALUES(9, 'E L E T L', 'P W N F Z'); + INSERT INTO ft3(rowid, x, y) VALUES(10, 'O G J G X', 'G J F E P'); + INSERT INTO ft3(rowid, x, y) VALUES(11, 'O L N N Z', 'K E Z F D'); + INSERT INTO ft3(rowid, x, y) VALUES(12, 'R Z M R J', 'X G I M Z'); + INSERT INTO ft3(rowid, x, y) VALUES(13, 'L X N N X', 'R R N S T'); + INSERT INTO ft3(rowid, x, y) VALUES(14, 'F L B J H', 'K W F L C'); + INSERT INTO ft3(rowid, x, y) VALUES(15, 'P E B M V', 'E A A B U'); + INSERT INTO ft3(rowid, x, y) VALUES(16, 'V E C F P', 'L U T V K'); + INSERT INTO ft3(rowid, x, y) VALUES(17, 'T N O Z N', 'T P Q X N'); + INSERT INTO ft3(rowid, x, y) VALUES(18, 'V W U W R', 'H O A A V'); + INSERT INTO ft3(rowid, x, y) VALUES(19, 'A H N L F', 'I G H B O'); +} + +foreach {tn match rowidlist} { + 1 "N A" {5 19} + 2 "x:O" {1 2 10 11 17} + 3 "y:O" {0 2 6 7 18 19} +} { + set res [list] + foreach rowid $rowidlist { lappend res $rowid {} {} } + + do_execsql_test 3.2.2.$tn { + SELECT rowid, * FROM ft3 WHERE ft3 MATCH $match + } $res + do_execsql_test 3.2.3.$tn { + SELECT docid, * FROM ft3 WHERE ft3 MATCH $match + } $res +} + +do_execsql_test 3.3.1 { + INSERT INTO t3(rowid, x, y) VALUES(0, 'R T M S M', 'A F O K H'); + INSERT INTO t3(rowid, x, y) VALUES(1, 'C Z J O X', 'U S Q D K'); + INSERT INTO t3(rowid, x, y) VALUES(2, 'N G H P O', 'N O P O C'); + INSERT INTO t3(rowid, x, y) VALUES(3, 'V H S D R', 'K N G E C'); + INSERT INTO t3(rowid, x, y) VALUES(4, 'J T R V U', 'U X S L C'); + INSERT INTO t3(rowid, x, y) VALUES(5, 'N A Y N G', 'X D G P Y'); + UPDATE ft3 SET x = y, y = x; + DELETE FROM t3; +} + +foreach {tn match rowidlist} { + 1 "N A" {5 19} + 2 "x:O" {0 2 10 11 17} + 3 "y:O" {1 2 6 7 18 19} +} { + set res [list] + foreach rowid $rowidlist { lappend res $rowid {} {} } + + do_execsql_test 3.3.2.$tn { + SELECT rowid, * FROM ft3 WHERE ft3 MATCH $match + } $res + do_execsql_test 3.3.3.$tn { + SELECT docid, * FROM ft3 WHERE ft3 MATCH $match + } $res +} + +do_execsql_test 3.3.1 { + INSERT INTO t3(rowid, x, y) VALUES(15, 'P E B M V', 'E A A B U'); + INSERT INTO t3(rowid, x, y) VALUES(16, 'V E C F P', 'L U T V K'); + INSERT INTO t3(rowid, x, y) VALUES(17, 'T N O Z N', 'T P Q X N'); + INSERT INTO t3(rowid, x, y) VALUES(18, 'V W U W R', 'H O A A V'); + INSERT INTO t3(rowid, x, y) VALUES(19, 'A H N L F', 'I G H B O'); + DELETE FROM ft3; +} + +foreach {tn match rowidlist} { + 1 "N A" {5} + 2 "x:O" {0 2 10 11} + 3 "y:O" {1 2 6 7} +} { + set res [list] + foreach rowid $rowidlist { lappend res $rowid {} {} } + + do_execsql_test 3.3.2.$tn { + SELECT rowid, * FROM ft3 WHERE ft3 MATCH $match + } $res + do_execsql_test 3.3.3.$tn { + SELECT docid, * FROM ft3 WHERE ft3 MATCH $match + } $res +} + + +#------------------------------------------------------------------------- +# Test cases 4.* test the 'rebuild' command. On content=xxx and regular +# FTS tables. +# +do_execsql_test 4.0 { + CREATE TABLE t4(x); + CREATE VIRTUAL TABLE ft4 USING fts4(content=t4); + CREATE VIRTUAL TABLE ft4x USING fts4(x); +} + +do_execsql_test 4.1.1 { + INSERT INTO ft4x(ft4x) VALUES('rebuild'); + INSERT INTO ft4(ft4) VALUES('rebuild'); +} {} +do_execsql_test 4.1.2 { + SELECT id, quote(value) FROM ft4_stat +} {0 X'000000'} +do_execsql_test 4.1.3 { + SELECT id, quote(value) FROM ft4x_stat +} {0 X'000000'} + +do_execsql_test 4.2.1 { + INSERT INTO ft4x VALUES('M G M F T'); + INSERT INTO ft4x VALUES('Z Q C A U'); + INSERT INTO ft4x VALUES('N L L V'); + INSERT INTO ft4x VALUES('T F D X D'); + INSERT INTO ft4x VALUES('Z H I S D'); + + SELECT id, quote(value) FROM ft4x_stat +} {0 X'05182B'} + +do_execsql_test 4.2.2 { + INSERT INTO ft4(rowid, x) SELECT rowid, * FROM ft4x; + SELECT id, quote(value) FROM ft4_stat +} {0 X'05182B'} + +do_execsql_test 4.2.3 { + SELECT docid, quote(size) FROM ft4_docsize +} {1 X'05' 2 X'05' 3 X'04' 4 X'05' 5 X'05'} + +do_execsql_test 4.2.4 { + INSERT INTO ft4x(ft4x) VALUES('rebuild'); + SELECT id, quote(value) FROM ft4x_stat; + SELECT docid, quote(size) FROM ft4x_docsize +} {0 X'05182B' 1 X'05' 2 X'05' 3 X'04' 4 X'05' 5 X'05'} + +do_execsql_test 4.2.5 { + INSERT INTO ft4(ft4) VALUES('rebuild'); + SELECT id, quote(value) FROM ft4_stat; + SELECT docid, quote(size) FROM ft4_docsize +} {0 X'000000'} + +do_execsql_test 4.2.6 { + INSERT INTO t4(rowid, x) SELECT rowid, x FROM ft4x; + INSERT INTO ft4(ft4) VALUES('rebuild'); + SELECT id, quote(value) FROM ft4_stat; + SELECT docid, quote(size) FROM ft4_docsize +} {0 X'05182B' 1 X'05' 2 X'05' 3 X'04' 4 X'05' 5 X'05'} + + +#------------------------------------------------------------------------- +# Test cases 5.* test that the following commands do not create/move or +# delete a %_content table when used with a content=xxx FTS table. +# +do_execsql_test 5.1.1 { + CREATE TABLE t5(a, b, c, d); + CREATE VIRTUAL TABLE ft5 USING fts4(content=t5); + SELECT name FROM sqlite_master WHERE name LIKE '%t5%'; +} { + t5 ft5 ft5_segments ft5_segdir + sqlite_autoindex_ft5_segdir_1 ft5_docsize ft5_stat +} +do_execsql_test 5.1.2 { + ALTER TABLE ft5 RENAME TO ft6; + SELECT name FROM sqlite_master WHERE name LIKE '%t5%'; +} { + t5 +} +do_execsql_test 5.1.3 { + SELECT name FROM sqlite_master WHERE name LIKE '%t6%'; +} { + ft6 ft6_segments ft6_segdir + sqlite_autoindex_ft6_segdir_1 ft6_docsize ft6_stat +} +do_execsql_test 5.1.4 { + INSERT INTO t5 VALUES('a', 'b', 'c', 'd'); + INSERT INTO ft6(ft6) VALUES('rebuild'); + SELECT rowid FROM ft6 WHERE ft6 MATCH 'b'; +} {1} +do_execsql_test 5.1.5 { + DROP TABLE ft6; + SELECT * FROM t5; +} {a b c d} +do_execsql_test 5.1.6 { + SELECT name FROM sqlite_master WHERE name LIKE '%t6%'; +} { +} +do_execsql_test 5.1.7 { + CREATE VIRTUAL TABLE ft5 USING fts4(content=t5); + CREATE TABLE t5_content(a, b); + DROP TABLE ft5; + SELECT name FROM sqlite_master WHERE name LIKE '%t5%'; +} { + t5 t5_content +} + +finish_test diff --git a/test/permutations.test b/test/permutations.test index c911e6f062..9ae881ba23 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -180,10 +180,9 @@ test_suite "fts3" -prefix "" -description { fts3defer.test fts3defer2.test fts3e.test fts3expr.test fts3expr2.test fts3near.test fts3query.test fts3shared.test fts3snippet.test fts3sort.test - fts3fault.test fts3malloc.test fts3matchinfo.test - fts3aux1.test fts3comp1.test fts3auto.test + fts4aa.test fts4content.test } From cc5b81464a4ea31d8a3357f10514bded304ab211 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 4 Oct 2011 16:37:35 +0000 Subject: [PATCH 02/11] Add tests to check that modifying the schema of an FTS content table does not cause a crash in the FTS module. Also disable the deferred token optimization for content=xxx FTS tables. FossilOrigin-Name: be86c7061b68f403730bf63ea1f7dc0d9ceb0a3b --- ext/fts3/fts3.c | 9 +++++ ext/fts3/fts3Int.h | 3 +- manifest | 21 +++++------ manifest.uuid | 2 +- test/fts4content.test | 85 ++++++++++++++++++++++++++++++++++++++++++ test/permutations.test | 2 + 6 files changed, 107 insertions(+), 15 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index c188945b28..c6b356f51f 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -4048,6 +4048,15 @@ static int fts3EvalSelectDeferred( int nMinEst = 0; /* The minimum count for any phrase so far. */ int nLoad4 = 1; /* (Phrases that will be loaded)^4. */ + /* Tokens are never deferred for FTS tables created using the content=xxx + ** option. The reason being that it is not guaranteed that the content + ** table actually contains the same data as the index. To prevent this from + ** causing any problems, the deferred token optimization is completely + ** disabled for content=xxx tables. */ + if( pTab->zContentTbl ){ + return SQLITE_OK; + } + /* Count the tokens in this AND/NEAR cluster. If none of the doclists ** associated with the tokens spill onto overflow pages, or if there is ** only 1 token, exit early. No tokens to defer in this case. */ diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index b4f26fc7a5..552d73d764 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -225,7 +225,7 @@ struct Fts3Table { int nPendingData; /* Current bytes of pending data */ sqlite_int64 iPrevDocid; /* Docid of most recently inserted document */ -#if defined(SQLITE_DEBUG) +#if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) /* State variables used for validating that the transaction control ** methods of the virtual table are called at appropriate times. These ** values do not contribution to the FTS computation; they are used for @@ -246,7 +246,6 @@ struct Fts3Cursor { i16 eSearch; /* Search strategy (see below) */ u8 isEof; /* True if at End Of Results */ u8 isRequireSeek; /* True if must seek pStmt to %_content row */ - u8 isNullRow; /* True for a row of NULLs */ sqlite3_stmt *pStmt; /* Prepared statement in use by the cursor */ Fts3Expr *pExpr; /* Parsed MATCH query string */ int nPhrase; /* Number of matchable phrases in query */ diff --git a/manifest b/manifest index 6ca44ed83d..9881895908 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sexperimental\s'content'\soption\sto\sFTS4. -D 2011-10-04T11:22:59.677 +C Add\stests\sto\scheck\sthat\smodifying\sthe\sschema\sof\san\sFTS\scontent\stable\sdoes\snot\scause\sa\scrash\sin\sthe\sFTS\smodule.\sAlso\sdisable\sthe\sdeferred\stoken\soptimization\sfor\scontent=xxx\sFTS\stables. +D 2011-10-04T16:37:35.422 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a162fe39e249b8ed4a65ee947c30152786cfe897 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -62,9 +62,9 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 626fcf477e531b470f06aae3d427abcda2d5d31c +F ext/fts3/fts3.c df149056426597b32b2b8488eefbc755258f55d7 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h a335d671d42ca07528a93c5b830e48e0205b26bc +F ext/fts3/fts3Int.h 06f442ce096e6254432a6b16a56b6fe7b24bd372 F ext/fts3/fts3_aux.c 0ebfa7b86cf8ff6a0861605fcc63b83ec1b70691 F ext/fts3/fts3_expr.c 23791de01b3a5d313d76e02befd2601d4096bc2b F ext/fts3/fts3_hash.c 8dd2d06b66c72c628c2732555a32bc0943114914 @@ -486,7 +486,7 @@ F test/fts3shared.test 8bb266521d7c5495c0ae522bb4d376ad5387d4a2 F test/fts3snippet.test 8e956051221a34c7daeb504f023cb54d5fa5a8b2 F test/fts3sort.test 9a5176c9317bb545ec5f144d62e6fedb4da6c66e F test/fts4aa.test 6e7f90420b837b2c685f3bcbe84c868492d40a68 -F test/fts4content.test fbafb7160eef1560b6a254b1ad5631acccd388a1 +F test/fts4content.test 5c226c7c666e250c175bcbdfbdd4be3f275c73ba F test/func.test 6c5ce11e3a0021ca3c0649234e2d4454c89110ca F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f F test/func3.test 001021e5b88bd02a3b365a5c5fd8f6f49d39744a @@ -621,7 +621,7 @@ F test/pageropt.test 8146bf448cf09e87bb1867c2217b921fb5857806 F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16 F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 -F test/permutations.test 7b1c4cb40af3712e42364c82390ace0df207b5e0 +F test/permutations.test d850b5000a13baf042d5a20eb747079477dad45e F test/pragma.test c8108e01da04f16e67e5754e610bc62c1b993f6c F test/pragma2.test 3a55f82b954242c642f8342b17dffc8b47472947 F test/printf.test 05970cde31b1a9f54bd75af60597be75a5c54fea @@ -966,10 +966,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2 -P eb5da5e1dbe9c198095036827318fb381441cbd0 -R 7991a3071fb34636c7b5ed4b010ad95a -T *branch * fts4-content -T *sym-fts4-content * -T -sym-trunk * +P 1d27ea741f61c624e18bdc6a3b1c2d8574a64ddc +R da82d432e1ef8526b881f10f91640d66 U dan -Z 48ff718a3615a321d5bc0764cb014ed4 +Z d642325dd5adee7274f24ac6f8449504 diff --git a/manifest.uuid b/manifest.uuid index be6a47c58b..18907d6d6b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1d27ea741f61c624e18bdc6a3b1c2d8574a64ddc \ No newline at end of file +be86c7061b68f403730bf63ea1f7dc0d9ceb0a3b \ No newline at end of file diff --git a/test/fts4content.test b/test/fts4content.test index 6ac5a6806a..7925ed218d 100644 --- a/test/fts4content.test +++ b/test/fts4content.test @@ -22,6 +22,24 @@ ifcapable !fts3 { return } +#------------------------------------------------------------------------- +# Test organization: +# +# 1.* - Warm-body tests. +# +# 2.* - Querying a content=xxx FTS table. +# +# 3.* - Writing to a content=xxx FTS table. +# +# 4.* - The "INSERT INTO fts(fts) VALUES('rebuild')" command. +# +# 5.* - Check that CREATE TABLE, DROP TABLE and ALTER TABLE correctly +# ignore any %_content table when used with the content=xxx option. +# +# 6.* - Test the effects of messing with the schema of table xxx after +# creating a content=xxx FTS index. +# + do_execsql_test 1.1 { CREATE TABLE t1(a, b, c); INSERT INTO t1 VALUES('w x', 'x y', 'y z'); @@ -363,4 +381,71 @@ do_execsql_test 5.1.7 { t5 t5_content } +#------------------------------------------------------------------------- +# Test cases 6.* test +# +do_catchsql_test 6.1.1 { + CREATE VIRTUAL TABLE ft7 USING fts4(content=t7); +} {1 {vtable constructor failed: ft7}} + +do_execsql_test 6.2.1 { + CREATE TABLE t7(one, two); + CREATE VIRTUAL TABLE ft7 USING fts4(content=t7); + INSERT INTO t7 VALUES('A B', 'B A'); + INSERT INTO t7 VALUES('C D', 'A A'); + SELECT * FROM ft7; +} { + {A B} {B A} {C D} {A A} +} + +do_catchsql_test 6.2.2 { + DROP TABLE t7; + SELECT * FROM ft7; +} {1 {SQL logic error or missing database}} + +db close +sqlite3 db test.db +do_execsql_test 6.2.3 { + SELECT name FROM sqlite_master WHERE name LIKE '%t7%' +} { + ft7 ft7_segments ft7_segdir sqlite_autoindex_ft7_segdir_1 + ft7_docsize ft7_stat +} +do_catchsql_test 6.2.4 { + SELECT * FROM ft7; +} {1 {vtable constructor failed: ft7}} +do_execsql_test 6.2.5 { + CREATE TABLE t7(x, y); + INSERT INTO t7 VALUES('A B', 'B A'); + INSERT INTO t7 VALUES('C D', 'A A'); + SELECT * FROM ft7; +} { + {A B} {B A} {C D} {A A} +} + +do_execsql_test 6.2.6 { + INSERT INTO ft7(ft7) VALUES('rebuild'); + SELECT rowid FROM ft7 WHERE ft7 MATCH '"A A"'; +} {2} + +do_execsql_test 6.2.7 { + DROP TABLE t7; + CREATE TABLE t7(x); +} +do_catchsql_test 6.2.8 { + SELECT rowid FROM ft7 WHERE ft7 MATCH '"A A"'; +} {1 {SQL logic error or missing database}} +do_catchsql_test 6.2.9 { + SELECT rowid FROM ft7 WHERE ft7 MATCH '"A A"'; +} {1 {SQL logic error or missing database}} + +db close +sqlite3 db test.db +do_catchsql_test 6.2.10 { + SELECT rowid FROM ft7 WHERE ft7 MATCH '"A A"'; +} {0 2} +do_catchsql_test 6.2.11 { + SELECT rowid, * FROM ft7 WHERE ft7 MATCH '"A A"'; +} {0 {2 {}}} + finish_test diff --git a/test/permutations.test b/test/permutations.test index 9ae881ba23..14330d0cc1 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -183,6 +183,8 @@ test_suite "fts3" -prefix "" -description { fts3fault.test fts3malloc.test fts3matchinfo.test fts3aux1.test fts3comp1.test fts3auto.test fts4aa.test fts4content.test + fts3conf.test fts3prefix.test fts3fault2.test fts3corrupt.test + fts3corrupt2.test } From 49fc336438e8bfca1427cbf1fe386259a111259b Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 4 Oct 2011 19:41:50 +0000 Subject: [PATCH 03/11] Improve test coverage of fts3.c. FossilOrigin-Name: 0f439944ab49a5691615bc170fdcf652055573df --- ext/fts3/fts3.c | 40 +++++++++++++++++++++------------------- manifest | 24 ++++++++++++------------ manifest.uuid | 2 +- test/fts3ao.test | 3 +++ test/fts3fault2.test | 23 +++++++++++++++++++++++ test/fts3malloc.test | 1 + test/fts3matchinfo.test | 16 ++++++++++++++++ test/fts3prefix.test | 10 ++++++++++ test/fts3sort.test | 5 +++++ 9 files changed, 92 insertions(+), 32 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index c6b356f51f..624221f979 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -1151,7 +1151,7 @@ static int fts3InitVtab( case 4: /* ORDER */ if( (strlen(zVal)!=3 || sqlite3_strnicmp(zVal, "asc", 3)) - && (strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "desc", 3)) + && (strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "desc", 4)) ){ *pzErr = sqlite3_mprintf("unrecognized order: %s", zVal); rc = SQLITE_ERROR; @@ -1159,7 +1159,8 @@ static int fts3InitVtab( bDescIdx = (zVal[0]=='d' || zVal[0]=='D'); break; - case 5: /* CONTENT */ + default: /* CONTENT */ + assert( iOpt==5 ); sqlite3_free(zUncompress); zContent = zVal; zVal = 0; @@ -1928,7 +1929,7 @@ static int fts3PoslistPhraseMerge( char **pp1, /* IN/OUT: Left input list */ char **pp2 /* IN/OUT: Right input list */ ){ - char *p = (pp ? *pp : 0); + char *p = *pp; char *p1 = *pp1; char *p2 = *pp2; int iCol1 = 0; @@ -1954,7 +1955,7 @@ static int fts3PoslistPhraseMerge( sqlite3_int64 iPos1 = 0; sqlite3_int64 iPos2 = 0; - if( pp && iCol1 ){ + if( iCol1 ){ *p++ = POS_COLUMN; p += sqlite3Fts3PutVarint(p, iCol1); } @@ -1969,13 +1970,6 @@ static int fts3PoslistPhraseMerge( || (isExact==0 && iPos2>iPos1 && iPos2<=iPos1+nToken) ){ sqlite3_int64 iSave; - if( !pp ){ - fts3PoslistCopy(0, &p2); - fts3PoslistCopy(0, &p1); - *pp1 = p1; - *pp2 = p2; - return 1; - } iSave = isSaveLeft ? iPos1 : iPos2; fts3PutDeltaVarint(&p, &iPrev, iSave+2); iPrev -= 2; pSave = 0; @@ -2027,7 +2021,7 @@ static int fts3PoslistPhraseMerge( fts3PoslistCopy(0, &p1); *pp1 = p1; *pp2 = p2; - if( !pp || *pp==p ){ + if( *pp==p ){ return 0; } *p++ = 0x00; @@ -3216,10 +3210,14 @@ static int fts3RenameMethod( sqlite3 *db = p->db; /* Database connection */ int rc; /* Return Code */ + /* As it happens, the pending terms table is always empty here. This is + ** because an "ALTER TABLE RENAME TABLE" statement inside a transaction + ** always opens a savepoint transaction. And the xSavepoint() method + ** flushes the pending terms table. But leave the (no-op) call to + ** PendingTermsFlush() in in case that changes. + */ + assert( p->nPendingData==0 ); rc = sqlite3Fts3PendingTermsFlush(p); - if( rc!=SQLITE_OK ){ - return rc; - } if( p->zContentTbl==0 ){ fts3DbExec(&rc, db, @@ -3586,21 +3584,20 @@ static int fts3EvalPhraseLoad( */ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ int iToken; /* Used to iterate through phrase tokens */ - int rc = SQLITE_OK; /* Return code */ char *aPoslist = 0; /* Position list for deferred tokens */ int nPoslist = 0; /* Number of bytes in aPoslist */ int iPrev = -1; /* Token number of previous deferred token */ assert( pPhrase->doclist.bFreeList==0 ); - for(iToken=0; rc==SQLITE_OK && iTokennToken; iToken++){ + for(iToken=0; iTokennToken; iToken++){ Fts3PhraseToken *pToken = &pPhrase->aToken[iToken]; Fts3DeferredToken *pDeferred = pToken->pDeferred; if( pDeferred ){ char *pList; int nList; - rc = sqlite3Fts3DeferredTokenList(pDeferred, &pList, &nList); + int rc = sqlite3Fts3DeferredTokenList(pDeferred, &pList, &nList); if( rc!=SQLITE_OK ) return rc; if( pList==0 ){ @@ -3930,7 +3927,7 @@ static void fts3EvalTokenCosts( Fts3Expr ***ppOr, /* Write new OR root to *(*ppOr)++ */ int *pRc /* IN/OUT: Error code */ ){ - if( *pRc==SQLITE_OK && pExpr ){ + if( *pRc==SQLITE_OK ){ if( pExpr->eType==FTSQUERY_PHRASE ){ Fts3Phrase *pPhrase = pExpr->pPhrase; int i; @@ -3944,6 +3941,11 @@ static void fts3EvalTokenCosts( *pRc = sqlite3Fts3MsrOvfl(pCsr, pTC->pToken->pSegcsr, &pTC->nOvfl); } }else if( pExpr->eType!=FTSQUERY_NOT ){ + assert( pExpr->eType==FTSQUERY_OR + || pExpr->eType==FTSQUERY_AND + || pExpr->eType==FTSQUERY_NEAR + ); + assert( pExpr->pLeft && pExpr->pRight ); if( pExpr->eType==FTSQUERY_OR ){ pRoot = pExpr->pLeft; **ppOr = pRoot; diff --git a/manifest b/manifest index 9881895908..e1d1bbee86 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stests\sto\scheck\sthat\smodifying\sthe\sschema\sof\san\sFTS\scontent\stable\sdoes\snot\scause\sa\scrash\sin\sthe\sFTS\smodule.\sAlso\sdisable\sthe\sdeferred\stoken\soptimization\sfor\scontent=xxx\sFTS\stables. -D 2011-10-04T16:37:35.422 +C Improve\stest\scoverage\sof\sfts3.c. +D 2011-10-04T19:41:50.550 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a162fe39e249b8ed4a65ee947c30152786cfe897 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -62,7 +62,7 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c df149056426597b32b2b8488eefbc755258f55d7 +F ext/fts3/fts3.c 6d6f3d331ed785d2e68608443eff66448ea95354 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h 06f442ce096e6254432a6b16a56b6fe7b24bd372 F ext/fts3/fts3_aux.c 0ebfa7b86cf8ff6a0861605fcc63b83ec1b70691 @@ -457,7 +457,7 @@ F test/fts3ak.test bd14deafe9d1586e8e9bf032411026ac4f8c925d F test/fts3al.test 07d64326e79bbdbab20ee87fc3328fbf01641c9f F test/fts3am.test 218aa6ba0dfc50c7c16b2022aac5c6be593d08d8 F test/fts3an.test a49ccadc07a2f7d646ec1b81bc09da2d85a85b18 -F test/fts3ao.test 60a15590d3c8578e943e4a149524b16b9bc1be92 +F test/fts3ao.test e7b80272efcced57d1d087a9da5c690dd7c21fd9 F test/fts3atoken.test 402ef2f7c2fb4b3d4fa0587df6441c1447e799b3 F test/fts3auto.test c1a30b37002b7c764a96937fbc71065b73d69494 F test/fts3aux1.test 0b02743955d56fc0d4d66236a26177bd1b726de0 @@ -475,16 +475,16 @@ F test/fts3e.test 1f6c6ac9cc8b772ca256e6b22aaeed50c9350851 F test/fts3expr.test 5e745b2b6348499d9ef8d59015de3182072c564c F test/fts3expr2.test 18da930352e5693eaa163a3eacf96233b7290d1a F test/fts3fault.test f83e556465bb69dc8bc676339eca408dce4ca246 -F test/fts3fault2.test dc96203af6ba31ce20163fc35460e1556e8edf4d -F test/fts3malloc.test 9c8cc3f885bb4dfc66d0460c52f68f45e4710d1b -F test/fts3matchinfo.test 08a82d18cc08abb28aec41d412b4c2ef25ba6a5f +F test/fts3fault2.test 253f9b336043ab7d0393d1b97a9f4ed21190331a +F test/fts3malloc.test b86ea33db9e8c58c0c2f8027a9fcadaf6a1568be +F test/fts3matchinfo.test 6507fe1c342e542300d65ea637d4110eccf894e6 F test/fts3near.test 2e318ee434d32babd27c167142e2b94ddbab4844 -F test/fts3prefix.test 36246609111ec1683f7ea5ed27666ce2cefb5676 +F test/fts3prefix.test b36d4f00b128a51e7b386cc013a874246d9d7dc1 F test/fts3query.test ef79d31fdb355d094baec1c1b24b60439a1fb8a2 F test/fts3rnd.test 1320d8826a845e38a96e769562bf83d7a92a15d0 F test/fts3shared.test 8bb266521d7c5495c0ae522bb4d376ad5387d4a2 F test/fts3snippet.test 8e956051221a34c7daeb504f023cb54d5fa5a8b2 -F test/fts3sort.test 9a5176c9317bb545ec5f144d62e6fedb4da6c66e +F test/fts3sort.test 95be0b19d7e41c44b29014f13ea8bddd495fd659 F test/fts4aa.test 6e7f90420b837b2c685f3bcbe84c868492d40a68 F test/fts4content.test 5c226c7c666e250c175bcbdfbdd4be3f275c73ba F test/func.test 6c5ce11e3a0021ca3c0649234e2d4454c89110ca @@ -966,7 +966,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2 -P 1d27ea741f61c624e18bdc6a3b1c2d8574a64ddc -R da82d432e1ef8526b881f10f91640d66 +P be86c7061b68f403730bf63ea1f7dc0d9ceb0a3b +R 5a9454f18e2824c0a4b3f3725eccae4e U dan -Z d642325dd5adee7274f24ac6f8449504 +Z e08086009c7ea460be359636e4d09cb8 diff --git a/manifest.uuid b/manifest.uuid index 18907d6d6b..2e94e1ac7c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -be86c7061b68f403730bf63ea1f7dc0d9ceb0a3b \ No newline at end of file +0f439944ab49a5691615bc170fdcf652055573df \ No newline at end of file diff --git a/test/fts3ao.test b/test/fts3ao.test index 0b6fcd3d19..786667a7f3 100644 --- a/test/fts3ao.test +++ b/test/fts3ao.test @@ -200,6 +200,9 @@ do_test fts3ao-4.7 { SELECT * FROM t5; } } {{the quick brown fox} {jumped over the} {lazy dog}} +do_execsql_test fts3ao-4.8 { + SELECT snippet(t5, '[', ']') FROM t5 WHERE t5 MATCH 'the' +} {{[the] quick brown fox} {jumped over [the]}} # Test that it is possible to rename an FTS4 table. Renaming an FTS4 table # involves renaming the extra %_docsize and %_stat tables. diff --git a/test/fts3fault2.test b/test/fts3fault2.test index fb877737f4..c24c52fe95 100644 --- a/test/fts3fault2.test +++ b/test/fts3fault2.test @@ -82,4 +82,27 @@ do_faultsim_test 2.1 -prep { faultsim_test_result {0 {a * 1 1 a 0 1 1 b * 1 1 b 0 1 1 c * 1 1 c 0 1 1 x * 1 1 x 1 1 1 y * 1 1 y 1 1 1 z * 1 1 z 1 1 1}} } +do_faultsim_test 3.0 -faults oom* -prep { + faultsim_delete_and_reopen + db eval { CREATE TABLE 'xx yy'(a, b); } +} -body { + execsql { + CREATE VIRTUAL TABLE tt USING fts4(content="xx yy"); + } +} -test { + faultsim_test_result {0 {}} +} + +do_faultsim_test 3.1 -faults oom* -prep { + faultsim_delete_and_reopen + db func zip zip + db func unzip unzip +} -body { + execsql { + CREATE VIRTUAL TABLE tt USING fts4(compress=zip, uncompress=unzip); + } +} -test { + faultsim_test_result {0 {}} +} + finish_test diff --git a/test/fts3malloc.test b/test/fts3malloc.test index 932ea098d0..7eeee7fe0d 100644 --- a/test/fts3malloc.test +++ b/test/fts3malloc.test @@ -294,6 +294,7 @@ do_write_test fts3_malloc-5.1 ft_content { do_test fts3_malloc-5.2 { execsql { CREATE VIRTUAL TABLE ft8 USING fts3(x, tokenize porter) } } {} + do_write_test fts3_malloc-5.3 ft_content { INSERT INTO ft8 VALUES('short alongertoken reallyquitealotlongerimeanit andthistokenisjustsolongthatonemightbeforgivenforimaginingthatitwasmerelyacontrivedexampleandnotarealtoken') } diff --git a/test/fts3matchinfo.test b/test/fts3matchinfo.test index 40366b6aef..0e8885804f 100644 --- a/test/fts3matchinfo.test +++ b/test/fts3matchinfo.test @@ -19,6 +19,7 @@ source $testdir/tester.tcl ifcapable !fts3 { finish_test ; return } set testprefix fts3matchinfo +set sqlite_fts3_enable_parentheses 0 proc mit {blob} { set scan(littleEndian) i* @@ -57,6 +58,9 @@ do_catchsql_test 2.0 { do_catchsql_test 2.1 { CREATE VIRTUAL TABLE x2 USING fts4(mtchinfo=fts3); } {1 {unrecognized parameter: mtchinfo=fts3}} +do_catchsql_test 2.2 { + CREATE VIRTUAL TABLE x2 USING fts4(matchinfo=fts5); +} {1 {unrecognized matchinfo: fts5}} # Check that with fts3, the "=" character is permitted in column definitions. # @@ -224,6 +228,18 @@ do_matchinfo_test 4.1.3 t4 {t4 MATCH 'a b'} { s {{2 0} {0 2}} } do_matchinfo_test 4.1.4 t4 {t4 MATCH '"a b" c'} { s {{2 0} {0 2}} } do_matchinfo_test 4.1.5 t4 {t4 MATCH 'a "b c"'} { s {{2 0} {0 2}} } do_matchinfo_test 4.1.6 t4 {t4 MATCH 'd d'} { s {{1 0} {0 1}} } +do_matchinfo_test 4.1.7 t4 {t4 MATCH 'f OR abcd'} { + x { + {0 1 1 1 1 1 0 0 0 0 0 0} + {1 1 1 0 1 1 0 0 0 0 0 0} + } +} +do_matchinfo_test 4.1.8 t4 {t4 MATCH 'f -abcd'} { + x { + {0 1 1 1 1 1} + {1 1 1 0 1 1} + } +} do_execsql_test 4.2.0 { CREATE VIRTUAL TABLE t5 USING fts4; diff --git a/test/fts3prefix.test b/test/fts3prefix.test index f5e31f3208..e7c197da9d 100644 --- a/test/fts3prefix.test +++ b/test/fts3prefix.test @@ -200,4 +200,14 @@ do_execsql_test 4.6 { SELECT * FROM t3 WHERE t3 MATCH 'one*' } {{one two three}} +#------------------------------------------------------------------------- +# Syntax tests. +# +do_catchsql_test 5.1 { + CREATE VIRTUAL TABLE t4 USING fts4(prefix="abc"); +} {1 {error parsing prefix parameter: abc}} +do_catchsql_test 5.2 { + CREATE VIRTUAL TABLE t4 USING fts4(prefix=""); +} {0 {}} + finish_test diff --git a/test/fts3sort.test b/test/fts3sort.test index ccdc203442..be7560432b 100644 --- a/test/fts3sort.test +++ b/test/fts3sort.test @@ -138,6 +138,8 @@ foreach {tn param res} { 3 "order=dec" {1 {unrecognized order: dec}} 4 "order=xxx, order=asc" {1 {unrecognized order: xxx}} 5 "order=desc, order=asc" {0 {}} + 6 "order=xxxx, order=asc" {1 {unrecognized order: xxxx}} + 7 "order=desk" {1 {unrecognized order: desk}} } { execsql { DROP TABLE IF EXISTS t1 } do_catchsql_test 2.1.$tn " @@ -157,6 +159,9 @@ do_execsql_test 2.2 { do_execsql_test 2.3 { SELECT docid FROM t2 WHERE t2 MATCH 'aa'; } {3 1} +do_execsql_test 2.4 { + SELECT docid FROM t2 WHERE t2 MATCH 'aa' ORDER BY content; +} {1 3} #------------------------------------------------------------------------- # Test that ticket [56be976859] has been fixed. From 8361b189c9c28f2fecdd329ccd897be819a78159 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 5 Oct 2011 06:07:00 +0000 Subject: [PATCH 04/11] Fix a problem with IO error handling in the rebuild-index code. FossilOrigin-Name: c6ba81fcad32192674bd510e607f787adc1f7038 --- ext/fts3/fts3_write.c | 6 ++++-- manifest | 14 +++++++------- manifest.uuid | 2 +- test/fts3fault2.test | 26 ++++++++++++++++++++++++++ 4 files changed, 38 insertions(+), 10 deletions(-) diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 46a1ead2d8..591f31f081 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -3001,8 +3001,10 @@ static int fts3DoRebuild(Fts3Table *p){ sqlite3_free(aSz); if( pStmt ){ - assert( rc==SQLITE_OK ); - rc = sqlite3_finalize(pStmt); + int rc2 = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ){ + rc = rc2; + } } } diff --git a/manifest b/manifest index e1d1bbee86..b582707f76 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improve\stest\scoverage\sof\sfts3.c. -D 2011-10-04T19:41:50.550 +C Fix\sa\sproblem\swith\sIO\serror\shandling\sin\sthe\srebuild-index\scode. +D 2011-10-05T06:07:00.875 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a162fe39e249b8ed4a65ee947c30152786cfe897 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -77,7 +77,7 @@ F ext/fts3/fts3_test.c 24fa13f330db011500acb95590da9eee24951894 F ext/fts3/fts3_tokenizer.c 9ff7ec66ae3c5c0340fa081958e64f395c71a106 F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3 F ext/fts3/fts3_tokenizer1.c 0dde8f307b8045565cf63797ba9acfaff1c50c68 -F ext/fts3/fts3_write.c 9e14eb54310345b441d7a8c83bc3ae474230bea6 +F ext/fts3/fts3_write.c 16fba93fc840f15421ebf1a783a8c3395700bbf9 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 @@ -475,7 +475,7 @@ F test/fts3e.test 1f6c6ac9cc8b772ca256e6b22aaeed50c9350851 F test/fts3expr.test 5e745b2b6348499d9ef8d59015de3182072c564c F test/fts3expr2.test 18da930352e5693eaa163a3eacf96233b7290d1a F test/fts3fault.test f83e556465bb69dc8bc676339eca408dce4ca246 -F test/fts3fault2.test 253f9b336043ab7d0393d1b97a9f4ed21190331a +F test/fts3fault2.test b62a2bc843c20414405f80e5eeb78e39bc68fe53 F test/fts3malloc.test b86ea33db9e8c58c0c2f8027a9fcadaf6a1568be F test/fts3matchinfo.test 6507fe1c342e542300d65ea637d4110eccf894e6 F test/fts3near.test 2e318ee434d32babd27c167142e2b94ddbab4844 @@ -966,7 +966,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2 -P be86c7061b68f403730bf63ea1f7dc0d9ceb0a3b -R 5a9454f18e2824c0a4b3f3725eccae4e +P 0f439944ab49a5691615bc170fdcf652055573df +R 29197ea13db81f2ba07fe324b03117e5 U dan -Z e08086009c7ea460be359636e4d09cb8 +Z a28913360f741831433fc7b43c93a8a9 diff --git a/manifest.uuid b/manifest.uuid index 2e94e1ac7c..5039f9295e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0f439944ab49a5691615bc170fdcf652055573df \ No newline at end of file +c6ba81fcad32192674bd510e607f787adc1f7038 \ No newline at end of file diff --git a/test/fts3fault2.test b/test/fts3fault2.test index c24c52fe95..0178ed27cc 100644 --- a/test/fts3fault2.test +++ b/test/fts3fault2.test @@ -105,4 +105,30 @@ do_faultsim_test 3.1 -faults oom* -prep { faultsim_test_result {0 {}} } +do_test 4.0 { + faultsim_delete_and_reopen + execsql { + CREATE VIRTUAL TABLE ft USING fts4(a, b); + INSERT INTO ft VALUES('U U T C O', 'F N D E S'); + INSERT INTO ft VALUES('P H X G B', 'I D M R U'); + INSERT INTO ft VALUES('P P X D M', 'Y V N T C'); + INSERT INTO ft VALUES('Z L Q O W', 'D F U N Q'); + INSERT INTO ft VALUES('A J D U P', 'C H M Q E'); + INSERT INTO ft VALUES('P S A O H', 'S Z C W D'); + INSERT INTO ft VALUES('T B N L W', 'C A K T I'); + INSERT INTO ft VALUES('K E Z L O', 'L L Y C E'); + INSERT INTO ft VALUES('C R E S V', 'Q V F W P'); + INSERT INTO ft VALUES('S K H G W', 'R W Q F G'); + } + faultsim_save_and_close +} {} +do_faultsim_test 4.1 -prep { + faultsim_restore_and_reopen + db eval {SELECT * FROM sqlite_master} +} -body { + execsql { INSERT INTO ft(ft) VALUES('rebuild') } +} -test { + faultsim_test_result {0 {}} +} + finish_test From deb9473250318fd3f361d59178df39129d6bd810 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 5 Oct 2011 15:11:30 +0000 Subject: [PATCH 05/11] Change FTS4 so that if both the content=xxx option and column names are specified, the virtual table assumes that the named columns correspond to columns of table xxx. FossilOrigin-Name: 289ee43179369fce2fde50870d72c445e184e896 --- ext/fts3/fts3.c | 105 +++++++++++++++++++++++++----------------- ext/fts3/fts3_write.c | 17 +++++-- manifest | 16 +++---- manifest.uuid | 2 +- test/fts4content.test | 45 ++++++++++++++---- 5 files changed, 121 insertions(+), 64 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 624221f979..6fa06907bf 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -1182,20 +1182,19 @@ static int fts3InitVtab( ** ** 1. Ignore any compress= and uncompress= options. ** - ** 2. Ignore any column names that were specified as part of the - ** the CREATE VIRTUAL TABLE statement. - ** - ** 3. Determine the actual column names to use for the FTS table - ** based on the columns of the content= table. + ** 2. If no column names were specified as part of the CREATE VIRTUAL + ** TABLE statement, use all columns from the content table. */ if( rc==SQLITE_OK && zContent ){ - sqlite3_free(aCol); sqlite3_free(zCompress); sqlite3_free(zUncompress); zCompress = 0; zUncompress = 0; - aCol = 0; - rc = fts3ContentColumns(db, argv[1], zContent, &aCol, &nCol, &nString); + if( nCol==0 ){ + sqlite3_free(aCol); + aCol = 0; + rc = fts3ContentColumns(db, argv[1], zContent, &aCol, &nCol, &nString); + } assert( rc!=SQLITE_OK || nCol>0 ); } if( rc!=SQLITE_OK ) goto fts3_init_out; @@ -1457,40 +1456,64 @@ static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){ return SQLITE_OK; } +/* +** If pCsr->pStmt has not been prepared (i.e. if pCsr->pStmt==0), then +** compose and prepare an SQL statement of the form: +** +** "SELECT FROM %_content WHERE rowid = ?" +** +** (or the equivalent for a content=xxx table) and set pCsr->pStmt to +** it. If an error occurs, return an SQLite error code. +** +** Otherwise, set *ppStmt to point to pCsr->pStmt and return SQLITE_OK. +*/ +static int fts3CursorSeekStmt(Fts3Cursor *pCsr, sqlite3_stmt **ppStmt){ + int rc = SQLITE_OK; + if( pCsr->pStmt==0 ){ + Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; + char *zSql; + zSql = sqlite3_mprintf("SELECT %s WHERE rowid = ?", p->zReadExprlist); + if( !zSql ) return SQLITE_NOMEM; + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); + sqlite3_free(zSql); + } + *ppStmt = pCsr->pStmt; + return rc; +} + /* ** Position the pCsr->pStmt statement so that it is on the row ** of the %_content table that contains the last match. Return ** SQLITE_OK on success. */ static int fts3CursorSeek(sqlite3_context *pContext, Fts3Cursor *pCsr){ + int rc = SQLITE_OK; if( pCsr->isRequireSeek ){ - sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId); - pCsr->isRequireSeek = 0; - if( SQLITE_ROW==sqlite3_step(pCsr->pStmt) ){ - return SQLITE_OK; - }else{ - int rc = sqlite3_reset(pCsr->pStmt); - if( rc==SQLITE_OK ){ - Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; - if( p->zContentTbl==0 ){ + sqlite3_stmt *pStmt = 0; + + rc = fts3CursorSeekStmt(pCsr, &pStmt); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId); + pCsr->isRequireSeek = 0; + if( SQLITE_ROW==sqlite3_step(pCsr->pStmt) ){ + return SQLITE_OK; + }else{ + rc = sqlite3_reset(pCsr->pStmt); + if( rc==SQLITE_OK && ((Fts3Table *)pCsr->base.pVtab)->zContentTbl==0 ){ /* If no row was found and no error has occured, then the %_content ** table is missing a row that is present in the full-text index. - ** The data structures are corrupt. - */ + ** The data structures are corrupt. */ rc = SQLITE_CORRUPT_VTAB; - }else{ - return SQLITE_OK; + pCsr->isEof = 1; } } - pCsr->isEof = 1; - if( pContext ){ - sqlite3_result_error_code(pContext, rc); - } - return rc; } - }else{ - return SQLITE_OK; } + + if( rc!=SQLITE_OK && pContext ){ + sqlite3_result_error_code(pContext, rc); + } + return rc; } /* @@ -2848,24 +2871,24 @@ static int fts3FilterMethod( ** row by docid. */ if( idxNum==FTS3_FULLSCAN_SEARCH ){ - const char *zTmpl = "SELECT %s ORDER BY rowid %s"; - zSql = sqlite3_mprintf(zTmpl, + zSql = sqlite3_mprintf( + "SELECT %s ORDER BY rowid %s", p->zReadExprlist, (pCsr->bDesc ? "DESC" : "ASC") ); - }else{ - const char *zTmpl = "SELECT %s WHERE rowid = ?"; - zSql = sqlite3_mprintf(zTmpl, p->zReadExprlist); + if( zSql ){ + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); + sqlite3_free(zSql); + }else{ + rc = SQLITE_NOMEM; + } + }else if( idxNum==FTS3_DOCID_SEARCH ){ + rc = fts3CursorSeekStmt(pCsr, &pCsr->pStmt); + if( rc==SQLITE_OK ){ + rc = sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]); + } } - if( !zSql ) return SQLITE_NOMEM; - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); - sqlite3_free(zSql); if( rc!=SQLITE_OK ) return rc; - if( idxNum==FTS3_DOCID_SEARCH ){ - rc = sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]); - if( rc!=SQLITE_OK ) return rc; - } - return fts3NextMethod(pCursor); } diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 591f31f081..47bed0dd5e 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -409,17 +409,24 @@ static void fts3SqlExec( ** not what users expect when they get SQLITE_LOCKED_SHAREDCACHE. It can ** still happen if the user reads data directly from the %_segments or ** %_segdir tables instead of going through FTS3 though. +** +** This reasoning does not apply to a content=xxx table. */ int sqlite3Fts3ReadLock(Fts3Table *p){ int rc; /* Return code */ sqlite3_stmt *pStmt; /* Statement used to obtain lock */ - rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pStmt, 0); - if( rc==SQLITE_OK ){ - sqlite3_bind_null(pStmt, 1); - sqlite3_step(pStmt); - rc = sqlite3_reset(pStmt); + if( p->zContentTbl==0 ){ + rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_null(pStmt, 1); + sqlite3_step(pStmt); + rc = sqlite3_reset(pStmt); + } + }else{ + rc = SQLITE_OK; } + return rc; } diff --git a/manifest b/manifest index b582707f76..0c36a83a5c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\swith\sIO\serror\shandling\sin\sthe\srebuild-index\scode. -D 2011-10-05T06:07:00.875 +C Change\sFTS4\sso\sthat\sif\sboth\sthe\scontent=xxx\soption\sand\scolumn\snames\sare\sspecified,\sthe\svirtual\stable\sassumes\sthat\sthe\snamed\scolumns\scorrespond\sto\scolumns\sof\stable\sxxx. +D 2011-10-05T15:11:30.760 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a162fe39e249b8ed4a65ee947c30152786cfe897 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -62,7 +62,7 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 6d6f3d331ed785d2e68608443eff66448ea95354 +F ext/fts3/fts3.c f2ed0ae669534e0c9c8ba95b60b6c137544e7e49 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h 06f442ce096e6254432a6b16a56b6fe7b24bd372 F ext/fts3/fts3_aux.c 0ebfa7b86cf8ff6a0861605fcc63b83ec1b70691 @@ -77,7 +77,7 @@ F ext/fts3/fts3_test.c 24fa13f330db011500acb95590da9eee24951894 F ext/fts3/fts3_tokenizer.c 9ff7ec66ae3c5c0340fa081958e64f395c71a106 F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3 F ext/fts3/fts3_tokenizer1.c 0dde8f307b8045565cf63797ba9acfaff1c50c68 -F ext/fts3/fts3_write.c 16fba93fc840f15421ebf1a783a8c3395700bbf9 +F ext/fts3/fts3_write.c 06520aa8a0a32a7bed08b29a9004fde1cb7f0318 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 @@ -486,7 +486,7 @@ F test/fts3shared.test 8bb266521d7c5495c0ae522bb4d376ad5387d4a2 F test/fts3snippet.test 8e956051221a34c7daeb504f023cb54d5fa5a8b2 F test/fts3sort.test 95be0b19d7e41c44b29014f13ea8bddd495fd659 F test/fts4aa.test 6e7f90420b837b2c685f3bcbe84c868492d40a68 -F test/fts4content.test 5c226c7c666e250c175bcbdfbdd4be3f275c73ba +F test/fts4content.test c5f531ecfc3d446b90032cae212549dbbb18dd78 F test/func.test 6c5ce11e3a0021ca3c0649234e2d4454c89110ca F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f F test/func3.test 001021e5b88bd02a3b365a5c5fd8f6f49d39744a @@ -966,7 +966,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2 -P 0f439944ab49a5691615bc170fdcf652055573df -R 29197ea13db81f2ba07fe324b03117e5 +P c6ba81fcad32192674bd510e607f787adc1f7038 +R 0217fe49810be9d8f9417ba3c648ab3e U dan -Z a28913360f741831433fc7b43c93a8a9 +Z 1eb4f36513f7844867f45162778888e7 diff --git a/manifest.uuid b/manifest.uuid index 5039f9295e..b8efcec3d4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c6ba81fcad32192674bd510e607f787adc1f7038 \ No newline at end of file +289ee43179369fce2fde50870d72c445e184e896 \ No newline at end of file diff --git a/test/fts4content.test b/test/fts4content.test index 7925ed218d..8295d91d61 100644 --- a/test/fts4content.test +++ b/test/fts4content.test @@ -40,13 +40,13 @@ ifcapable !fts3 { # creating a content=xxx FTS index. # -do_execsql_test 1.1 { +do_execsql_test 1.1.1 { CREATE TABLE t1(a, b, c); INSERT INTO t1 VALUES('w x', 'x y', 'y z'); CREATE VIRTUAL TABLE ft1 USING fts4(content=t1); } -do_execsql_test 1.2 { +do_execsql_test 1.1.2 { PRAGMA table_info(ft1); } { 0 a {} 0 {} 0 @@ -54,12 +54,23 @@ do_execsql_test 1.2 { 2 c {} 0 {} 0 } -do_execsql_test 1.3 { SELECT *, rowid FROM ft1 } {{w x} {x y} {y z} 1} -do_execsql_test 1.4 { SELECT a, c FROM ft1 WHERE rowid=1 } {{w x} {y z}} +do_execsql_test 1.1.3 { SELECT *, rowid FROM ft1 } {{w x} {x y} {y z} 1} +do_execsql_test 1.1.4 { SELECT a, c FROM ft1 WHERE rowid=1 } {{w x} {y z}} -do_execsql_test 1.5 { INSERT INTO ft1(ft1) VALUES('rebuild') } {} -do_execsql_test 1.6 { SELECT rowid FROM ft1 WHERE ft1 MATCH 'x' } {1} -do_execsql_test 1.7 { SELECT rowid FROM ft1 WHERE ft1 MATCH 'a' } {} +do_execsql_test 1.1.5 { INSERT INTO ft1(ft1) VALUES('rebuild') } {} +do_execsql_test 1.1.6 { SELECT rowid FROM ft1 WHERE ft1 MATCH 'x' } {1} +do_execsql_test 1.1.7 { SELECT rowid FROM ft1 WHERE ft1 MATCH 'a' } {} + +do_execsql_test 1.2.1 { + DROP TABLE ft1; + CREATE VIRTUAL TABLE ft1 USING fts4(content=t1, b); + PRAGMA table_info(ft1); +} { + 0 b {} 0 {} 0 +} +do_execsql_test 1.2.2 { + SELECT *, rowid FROM ft1 +} {{x y} 1} #------------------------------------------------------------------------- # The following block of tests - 2.* - test that a content=xxx FTS table @@ -433,10 +444,10 @@ do_execsql_test 6.2.7 { CREATE TABLE t7(x); } do_catchsql_test 6.2.8 { - SELECT rowid FROM ft7 WHERE ft7 MATCH '"A A"'; + SELECT * FROM ft7 WHERE ft7 MATCH '"A A"'; } {1 {SQL logic error or missing database}} do_catchsql_test 6.2.9 { - SELECT rowid FROM ft7 WHERE ft7 MATCH '"A A"'; + SELECT * FROM ft7 WHERE ft7 MATCH '"A A"'; } {1 {SQL logic error or missing database}} db close @@ -448,4 +459,20 @@ do_catchsql_test 6.2.11 { SELECT rowid, * FROM ft7 WHERE ft7 MATCH '"A A"'; } {0 {2 {}}} +#------------------------------------------------------------------------- +# Test cases 7.* +# +do_execsql_test 7.1.1 { + CREATE VIRTUAL TABLE ft8 USING fts4(content=nosuchtable, x); + INSERT INTO ft8(docid, x) VALUES(13, 'U O N X G'); + INSERT INTO ft8(docid, x) VALUES(14, 'C J J U B'); + INSERT INTO ft8(docid, x) VALUES(15, 'N J Y G X'); + INSERT INTO ft8(docid, x) VALUES(16, 'R Y D O R'); + INSERT INTO ft8(docid, x) VALUES(17, 'I Y T Q O'); +} + +do_execsql_test 7.1.2 { + SELECT docid FROM ft8 WHERE ft8 MATCH 'N'; +} {13 15} + finish_test From a986d33fd3e630ac8b16a5627761d242cb76d3be Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 18 Oct 2011 12:49:59 +0000 Subject: [PATCH 06/11] Cherrypick patch [3126754c72] from the trunk into the content= branch. FossilOrigin-Name: f9b5b217088a6aeb25eba184ab92d1a842a680a6 --- ext/fts3/fts3.c | 6 +++++- manifest | 14 +++++++------- manifest.uuid | 2 +- test/fts3defer.test | 28 ++++++++++++++++++++++++++++ 4 files changed, 41 insertions(+), 9 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 6fa06907bf..951d131667 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -4144,7 +4144,11 @@ static int fts3EvalSelectDeferred( fts3SegReaderCursorFree(pToken->pSegcsr); pToken->pSegcsr = 0; }else{ - nLoad4 = nLoad4*4; + /* Set nLoad4 to the value of (4^nOther) for the next iteration of the + ** for-loop. Except, limit the value to 2^24 to prevent it from + ** overflowing the 32-bit integer it is stored in. */ + if( ii<12 ) nLoad4 = nLoad4*4; + if( ii==0 || pTC->pPhrase->nToken>1 ){ /* Either this is the cheapest token in the entire query, or it is ** part of a multi-token phrase. Either way, the entire doclist will diff --git a/manifest b/manifest index 0c36a83a5c..78cf5c490d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sFTS4\sso\sthat\sif\sboth\sthe\scontent=xxx\soption\sand\scolumn\snames\sare\sspecified,\sthe\svirtual\stable\sassumes\sthat\sthe\snamed\scolumns\scorrespond\sto\scolumns\sof\stable\sxxx. -D 2011-10-05T15:11:30.760 +C Cherrypick\spatch\s[3126754c72]\sfrom\sthe\strunk\sinto\sthe\scontent=\sbranch. +D 2011-10-18T12:49:59.086 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a162fe39e249b8ed4a65ee947c30152786cfe897 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -62,7 +62,7 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c f2ed0ae669534e0c9c8ba95b60b6c137544e7e49 +F ext/fts3/fts3.c 15e1725f3dc7c0028676831d82b376e93b87527e F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h 06f442ce096e6254432a6b16a56b6fe7b24bd372 F ext/fts3/fts3_aux.c 0ebfa7b86cf8ff6a0861605fcc63b83ec1b70691 @@ -469,7 +469,7 @@ F test/fts3corrupt.test 7b0f91780ca36118d73324ec803187208ad33b32 F test/fts3corrupt2.test 6d96efae2f8a6af3eeaf283aba437e6d0e5447ba F test/fts3cov.test e0fb00d8b715ddae4a94c305992dfc3ef70353d7 F test/fts3d.test 95fb3c862cbc4297c93fceb9a635543744e9ef52 -F test/fts3defer.test 7c8a38d5f617d7b52ae1c43ed73c536e7e895a35 +F test/fts3defer.test ffd4e07f79a09660d4b3e2613b041ab9b6100d91 F test/fts3defer2.test 35867d33ba6db03f6c73bd6f5fc333ae14f68c81 F test/fts3e.test 1f6c6ac9cc8b772ca256e6b22aaeed50c9350851 F test/fts3expr.test 5e745b2b6348499d9ef8d59015de3182072c564c @@ -966,7 +966,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2 -P c6ba81fcad32192674bd510e607f787adc1f7038 -R 0217fe49810be9d8f9417ba3c648ab3e +P 289ee43179369fce2fde50870d72c445e184e896 +R 50f1458a2cdbee488c53894b18c71839 U dan -Z 1eb4f36513f7844867f45162778888e7 +Z e075e042ac91a648318521977fb348f1 diff --git a/manifest.uuid b/manifest.uuid index b8efcec3d4..f94940003d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -289ee43179369fce2fde50870d72c445e184e896 \ No newline at end of file +f9b5b217088a6aeb25eba184ab92d1a842a680a6 \ No newline at end of file diff --git a/test/fts3defer.test b/test/fts3defer.test index 4bc0b0a7c3..ab30bbe2c3 100644 --- a/test/fts3defer.test +++ b/test/fts3defer.test @@ -449,5 +449,33 @@ do_execsql_test 3.3 { SELECT count(*) FROM x1 WHERE x1 MATCH '"d e f"' } {16} +# At one point the following was causing a floating-point exception. +# +do_execsql_test 4.1 { + CREATE VIRTUAL TABLE x2 USING FTS4(x); + BEGIN; + INSERT INTO x2 VALUES('m m m m m m m m m m m m m m m m m m m m m m m m m m'); + INSERT INTO x2 SELECT * FROM x2; + INSERT INTO x2 SELECT * FROM x2; + INSERT INTO x2 SELECT * FROM x2; + INSERT INTO x2 SELECT * FROM x2; + INSERT INTO x2 SELECT * FROM x2; + INSERT INTO x2 SELECT * FROM x2; + INSERT INTO x2 SELECT * FROM x2; + INSERT INTO x2 SELECT * FROM x2; + INSERT INTO x2 SELECT * FROM x2; + INSERT INTO x2 SELECT * FROM x2; + INSERT INTO x2 SELECT * FROM x2; + INSERT INTO x2 SELECT * FROM x2; + INSERT INTO x2 SELECT * FROM x2; + INSERT INTO x2 SELECT * FROM x2; + INSERT INTO x2 SELECT * FROM x2; + INSERT INTO x2 VALUES('a b c d e f g h i j k l m n o p q r s t u v w x y m'); + COMMIT; +} +do_execsql_test 4.2 { + SELECT * FROM x2 WHERE x2 MATCH 'a b c d e f g h i j k l m n o p q r s'; +} {{a b c d e f g h i j k l m n o p q r s t u v w x y m}} + finish_test From 3f1ea8d114e20a377a12d5a7a88e5b4f6cc76e60 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 18 Oct 2011 19:39:41 +0000 Subject: [PATCH 07/11] If a token within an FTS query is prefixed with a '^' character, it must be the first token in a column of data to match. FossilOrigin-Name: 63ac33c860eb32ce96699f06bf83121cec2ffaca --- ext/fts3/fts3.c | 66 +++++++++++++++++++++++++++++++++++ ext/fts3/fts3Int.h | 1 + ext/fts3/fts3_expr.c | 17 +++++++-- ext/fts3/fts3_write.c | 1 + manifest | 23 ++++++------ manifest.uuid | 2 +- test/fts3defer.test | 7 ++++ test/fts3first.test | 79 ++++++++++++++++++++++++++++++++++++++++++ test/permutations.test | 1 + 9 files changed, 183 insertions(+), 14 deletions(-) create mode 100644 test/fts3first.test diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 951d131667..289adeb4df 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -2347,6 +2347,67 @@ static void fts3DoclistPhraseMerge( *pnRight = p - aOut; } +/* +** When this function is called, pList points to a doclist containing position +** data, length *pnList bytes. This removes all entries from the doclist that +** do not correspond to the first token in a column and overwrites pList +** with the result. *pnList is set to the length of the new doclist before +** returning. +** +** If bDescDoclist is true, then both the input and output are in descending +** order. Otherwise, ascending. +*/ +static void fts3DoclistFirstFilter( + int bDescDoclist, /* True if pList is a descending doclist */ + char *pList, /* Buffer containing doclist */ + int *pnList /* IN/OUT: Size of doclist */ +){ + char *p = pList; + char *pOut = pList; + char *pEnd = &pList[*pnList]; + + sqlite3_int64 iDoc; + sqlite3_int64 iPrev; + int bFirstOut = 0; + + fts3GetDeltaVarint3(&p, pEnd, 0, &iDoc); + while( p ){ + int bWritten = 0; + if( *p!=0x01 ){ + if( *p==0x02 ){ + fts3PutDeltaVarint3(&pOut, bDescDoclist, &iPrev, &bFirstOut, iDoc); + *pOut++ = 0x02; + bWritten = 1; + } + fts3ColumnlistCopy(0, &p); + } + + while( *p==0x01 ){ + sqlite3_int64 iCol; + p++; + p += sqlite3Fts3GetVarint(p, &iCol); + if( *p==0x02 ){ + if( bWritten==0 ){ + fts3PutDeltaVarint3(&pOut, bDescDoclist, &iPrev, &bFirstOut, iDoc); + bWritten = 1; + } + pOut += sqlite3Fts3PutVarint(pOut, iCol); + *pOut++ = 0x02; + } + fts3ColumnlistCopy(0, &p); + } + if( bWritten ){ + *pOut++ = 0x00; + } + + assert( *p==0x00 ); + p++; + fts3GetDeltaVarint3(&p, pEnd, bDescDoclist, &iDoc); + } + + *pnList = (pOut - pList); +} + /* ** Merge all doclists in the TermSelect.aaOutput[] array into a single @@ -3518,6 +3579,10 @@ static void fts3EvalPhraseMergeToken( ){ assert( iToken!=p->iDoclistToken ); + if( p->aToken[iToken].bFirst ){ + fts3DoclistFirstFilter(pTab->bDescIdx, pList, &nList); + } + if( pList==0 ){ sqlite3_free(p->doclist.aAll); p->doclist.aAll = 0; @@ -3721,6 +3786,7 @@ static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){ && p->nToken==1 && pFirst->pSegcsr && pFirst->pSegcsr->bLookup + && pFirst->bFirst==0 ){ /* Use the incremental approach. */ int iCol = (p->iColumn >= pTab->nColumn ? -1 : p->iColumn); diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index 552d73d764..c9b291c6cc 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -310,6 +310,7 @@ struct Fts3PhraseToken { char *z; /* Text of the token */ int n; /* Number of bytes in buffer z */ int isPrefix; /* True if token ends with a "*" character */ + int bFirst; /* True if token must appear at position 0 */ /* Variables above this point are populated when the expression is ** parsed (by code in fts3_expr.c). Below this point the variables are diff --git a/ext/fts3/fts3_expr.c b/ext/fts3/fts3_expr.c index 7eb2962d44..e6193a1392 100644 --- a/ext/fts3/fts3_expr.c +++ b/ext/fts3/fts3_expr.c @@ -180,9 +180,21 @@ static int getNextToken( pRet->pPhrase->aToken[0].isPrefix = 1; iEnd++; } - if( !sqlite3_fts3_enable_parentheses && iStart>0 && z[iStart-1]=='-' ){ - pParse->isNot = 1; + + while( 1 ){ + if( !sqlite3_fts3_enable_parentheses + && iStart>0 && z[iStart-1]=='-' + ){ + pParse->isNot = 1; + iStart--; + }else if( iStart>0 && z[iStart-1]=='^' ){ + pRet->pPhrase->aToken[0].bFirst = 1; + iStart--; + }else{ + break; + } } + } nConsumed = iEnd; } @@ -281,6 +293,7 @@ static int getNextString( pToken->n = nByte; pToken->isPrefix = (iEndbFirst = (iBegin>0 && zInput[iBegin-1]=='^'); nToken = ii+1; } } diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 47bed0dd5e..40c8e2f9ad 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -3117,6 +3117,7 @@ int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *pCsr){ for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){ Fts3PhraseToken *pPT = pDef->pToken; if( (pDef->iCol>=p->nColumn || pDef->iCol==i) + && (pPT->bFirst==0 || iPos==0) && (pPT->n==nToken || (pPT->isPrefix && pPT->nz, pPT->n)) ){ diff --git a/manifest b/manifest index 78cf5c490d..4ab3fba4d2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Cherrypick\spatch\s[3126754c72]\sfrom\sthe\strunk\sinto\sthe\scontent=\sbranch. -D 2011-10-18T12:49:59.086 +C If\sa\stoken\swithin\san\sFTS\squery\sis\sprefixed\swith\sa\s'^'\scharacter,\sit\smust\sbe\sthe\sfirst\stoken\sin\sa\scolumn\sof\sdata\sto\smatch. +D 2011-10-18T19:39:41.203 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a162fe39e249b8ed4a65ee947c30152786cfe897 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -62,11 +62,11 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 15e1725f3dc7c0028676831d82b376e93b87527e +F ext/fts3/fts3.c e12a151b5f0f8d444744554f91dbb89dbf0654df F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h 06f442ce096e6254432a6b16a56b6fe7b24bd372 +F ext/fts3/fts3Int.h bc27eebe2c5919115aa1858fdd308a230af6a359 F ext/fts3/fts3_aux.c 0ebfa7b86cf8ff6a0861605fcc63b83ec1b70691 -F ext/fts3/fts3_expr.c 23791de01b3a5d313d76e02befd2601d4096bc2b +F ext/fts3/fts3_expr.c dd0facbede8fd7d1376670cc6154f1fef3a4c5bc F ext/fts3/fts3_hash.c 8dd2d06b66c72c628c2732555a32bc0943114914 F ext/fts3/fts3_hash.h 8331fb2206c609f9fc4c4735b9ab5ad6137c88ec F ext/fts3/fts3_icu.c 6c8f395cdf9e1e3afa7fadb7e523dbbf381c6dfa @@ -77,7 +77,7 @@ F ext/fts3/fts3_test.c 24fa13f330db011500acb95590da9eee24951894 F ext/fts3/fts3_tokenizer.c 9ff7ec66ae3c5c0340fa081958e64f395c71a106 F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3 F ext/fts3/fts3_tokenizer1.c 0dde8f307b8045565cf63797ba9acfaff1c50c68 -F ext/fts3/fts3_write.c 06520aa8a0a32a7bed08b29a9004fde1cb7f0318 +F ext/fts3/fts3_write.c 567380f2d6671df16cfbb56324b321c71d5ab0d3 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 @@ -469,13 +469,14 @@ F test/fts3corrupt.test 7b0f91780ca36118d73324ec803187208ad33b32 F test/fts3corrupt2.test 6d96efae2f8a6af3eeaf283aba437e6d0e5447ba F test/fts3cov.test e0fb00d8b715ddae4a94c305992dfc3ef70353d7 F test/fts3d.test 95fb3c862cbc4297c93fceb9a635543744e9ef52 -F test/fts3defer.test ffd4e07f79a09660d4b3e2613b041ab9b6100d91 +F test/fts3defer.test b7bdf79da91365b00e7c21d70e9d0c617b9306b9 F test/fts3defer2.test 35867d33ba6db03f6c73bd6f5fc333ae14f68c81 F test/fts3e.test 1f6c6ac9cc8b772ca256e6b22aaeed50c9350851 F test/fts3expr.test 5e745b2b6348499d9ef8d59015de3182072c564c F test/fts3expr2.test 18da930352e5693eaa163a3eacf96233b7290d1a F test/fts3fault.test f83e556465bb69dc8bc676339eca408dce4ca246 F test/fts3fault2.test b62a2bc843c20414405f80e5eeb78e39bc68fe53 +F test/fts3first.test 10f42914701d559c9fabfd7725b56c9f1b542fe8 F test/fts3malloc.test b86ea33db9e8c58c0c2f8027a9fcadaf6a1568be F test/fts3matchinfo.test 6507fe1c342e542300d65ea637d4110eccf894e6 F test/fts3near.test 2e318ee434d32babd27c167142e2b94ddbab4844 @@ -621,7 +622,7 @@ F test/pageropt.test 8146bf448cf09e87bb1867c2217b921fb5857806 F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16 F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 -F test/permutations.test d850b5000a13baf042d5a20eb747079477dad45e +F test/permutations.test 522823b47238cb1754198f80817fe9f9158ede55 F test/pragma.test c8108e01da04f16e67e5754e610bc62c1b993f6c F test/pragma2.test 3a55f82b954242c642f8342b17dffc8b47472947 F test/printf.test 05970cde31b1a9f54bd75af60597be75a5c54fea @@ -966,7 +967,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2 -P 289ee43179369fce2fde50870d72c445e184e896 -R 50f1458a2cdbee488c53894b18c71839 +P f9b5b217088a6aeb25eba184ab92d1a842a680a6 +R 4545768a323f698040377ce29e450e89 U dan -Z e075e042ac91a648318521977fb348f1 +Z a7972271e048cbba7c3ba6dcd813d2e6 diff --git a/manifest.uuid b/manifest.uuid index f94940003d..1ccbaef9b8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f9b5b217088a6aeb25eba184ab92d1a842a680a6 \ No newline at end of file +63ac33c860eb32ce96699f06bf83121cec2ffaca \ No newline at end of file diff --git a/test/fts3defer.test b/test/fts3defer.test index ab30bbe2c3..7fbe6b14ea 100644 --- a/test/fts3defer.test +++ b/test/fts3defer.test @@ -426,6 +426,13 @@ foreach {tn setup} { SELECT rowid FROM t1 WHERE t1 MATCH '"jk xduvfhk" OR "zm azavwm"' } {8 15 26 92 96} } + + do_select_test 7.1 { + SELECT rowid FROM t1 WHERE t1 MATCH '^zm mjpavjuhw' + } {56 62} + do_select_test 7.2 { + SELECT rowid FROM t1 WHERE t1 MATCH '^azavwm zm' + } {43} } set testprefix fts3defer diff --git a/test/fts3first.test b/test/fts3first.test new file mode 100644 index 0000000000..6e309d0d9d --- /dev/null +++ b/test/fts3first.test @@ -0,0 +1,79 @@ +# 2011 October 18 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/malloc_common.tcl + +ifcapable !fts3 { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE x1 USING FTS4(a, b, c); + INSERT INTO x1(docid,a,b,c) VALUES(0, 'K H D S T', 'V M N Y K', 'S Z N Q S'); + INSERT INTO x1(docid,a,b,c) VALUES(1, 'K N J L W', 'S Z W J Q', 'D U W S E'); + INSERT INTO x1(docid,a,b,c) VALUES(2, 'B P M O I', 'R P H W S', 'R J L L E'); + INSERT INTO x1(docid,a,b,c) VALUES(3, 'U R Q M L', 'M J K A V', 'Q W J T J'); + INSERT INTO x1(docid,a,b,c) VALUES(4, 'N J C Y N', 'R U D X V', 'B O U A Q'); + INSERT INTO x1(docid,a,b,c) VALUES(5, 'Q L X L U', 'I F N X S', 'U Q A N Y'); + INSERT INTO x1(docid,a,b,c) VALUES(6, 'M R G U T', 'U V I Q P', 'X Y D L S'); + INSERT INTO x1(docid,a,b,c) VALUES(7, 'D Y P O I', 'X J P K R', 'V O T H V'); + INSERT INTO x1(docid,a,b,c) VALUES(8, 'R Y D L R', 'U U E S J', 'N W L M R'); + INSERT INTO x1(docid,a,b,c) VALUES(9, 'Z P F N P', 'W A X D U', 'V A E Q A'); + INSERT INTO x1(docid,a,b,c) VALUES(10, 'Q I A Q M', 'N D K H C', 'A H T Q Z'); + INSERT INTO x1(docid,a,b,c) VALUES(11, 'T E R Q B', 'C I B C B', 'F Z U W R'); + INSERT INTO x1(docid,a,b,c) VALUES(12, 'E S V U W', 'T P F W H', 'A M D J Q'); + INSERT INTO x1(docid,a,b,c) VALUES(13, 'X S B T Y', 'U D N D P', 'X Z Y G F'); + INSERT INTO x1(docid,a,b,c) VALUES(14, 'K H A B L', 'S R C C Z', 'D W E H J'); + INSERT INTO x1(docid,a,b,c) VALUES(15, 'C E U C C', 'W F M N M', 'T Z U X T'); + INSERT INTO x1(docid,a,b,c) VALUES(16, 'Q G C G H', 'H N N B H', 'B Q I H Y'); + INSERT INTO x1(docid,a,b,c) VALUES(17, 'Q T S K B', 'W B D Y N', 'V J P E C'); + INSERT INTO x1(docid,a,b,c) VALUES(18, 'A J M O Q', 'L G Y Y A', 'G N M R N'); + INSERT INTO x1(docid,a,b,c) VALUES(19, 'T R Y P Y', 'N V Y B X', 'L Z T N T'); + + CREATE VIRTUAL TABLE x2 USING FTS4(a, b, c, order=DESC); + INSERT INTO x2(docid, a, b, c) SELECT docid, a, b, c FROM x1; +} + +foreach x {1 2} { + foreach {tn match res} { + 1 "^K" {0 1 14} + 2 "^S" {0 1 14} + 3 "^W" {9 15 17} + 4 "^J" {} + 5 "^E" {12} + 6 "V ^-E" {0 3 4 6 7 9 17 19} + 7 "V -^E" {0 3 4 6 7 9 17 19} + 8 "^-E V" {0 3 4 6 7 9 17 19} + 9 "-^E V" {0 3 4 6 7 9 17 19} + 10 "V" {0 3 4 6 7 9 12 17 19} + + 11 {"^K H"} {0 14} + 12 {"K H"} {0 10 14} + 13 {"K ^H"} {} + } { + set rev [list] + for {set ii [expr [llength $res]-1]} {$ii>=0} {incr ii -1} { + lappend rev [lindex $res $ii] + } + do_execsql_test 1.$x.$tn.1 {SELECT docid FROM x1 WHERE x1 MATCH $match} $res + do_execsql_test 1.$x.$tn.2 {SELECT docid FROM x2 WHERE x2 MATCH $match} $rev + } + + do_execsql_test 1.$x.[expr $tn+1] { + INSERT INTO x1(x1) VALUES('optimize'); + INSERT INTO x2(x2) VALUES('optimize'); + } {} +} + +finish_test diff --git a/test/permutations.test b/test/permutations.test index 14330d0cc1..7c3b026c67 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -185,6 +185,7 @@ test_suite "fts3" -prefix "" -description { fts4aa.test fts4content.test fts3conf.test fts3prefix.test fts3fault2.test fts3corrupt.test fts3corrupt2.test + fts3first.test } From 50a7544d6f281d0adf43930d00fd40db2e5acbc6 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 19 Oct 2011 09:40:49 +0000 Subject: [PATCH 08/11] Fix a problem in FTS to do with ^ tokens and the snippet() function. FossilOrigin-Name: 2c03b24f4cc6f2c28c9d5b9984320d41b8486c32 --- ext/fts3/fts3.c | 1 + ext/fts3/fts3_snippet.c | 1 + manifest | 16 ++++++++-------- manifest.uuid | 2 +- test/fts3first.test | 31 ++++++++++++++++++++++++++----- 5 files changed, 37 insertions(+), 14 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 289adeb4df..e82d1f5e89 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -2391,6 +2391,7 @@ static void fts3DoclistFirstFilter( fts3PutDeltaVarint3(&pOut, bDescDoclist, &iPrev, &bFirstOut, iDoc); bWritten = 1; } + *pOut++ = 0x01; pOut += sqlite3Fts3PutVarint(pOut, iCol); *pOut++ = 0x02; } diff --git a/ext/fts3/fts3_snippet.c b/ext/fts3/fts3_snippet.c index 13d0ca3551..6a3d1ec893 100644 --- a/ext/fts3/fts3_snippet.c +++ b/ext/fts3/fts3_snippet.c @@ -368,6 +368,7 @@ static int fts3SnippetFindPositions(Fts3Expr *pExpr, int iPhrase, void *ctx){ int iFirst = 0; pPhrase->pList = pCsr; fts3GetDeltaPosition(&pCsr, &iFirst); + assert( iFirst>=0 ); pPhrase->pHead = pCsr; pPhrase->pTail = pCsr; pPhrase->iHead = iFirst; diff --git a/manifest b/manifest index 4ab3fba4d2..f9ee83d4ef 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C If\sa\stoken\swithin\san\sFTS\squery\sis\sprefixed\swith\sa\s'^'\scharacter,\sit\smust\sbe\sthe\sfirst\stoken\sin\sa\scolumn\sof\sdata\sto\smatch. -D 2011-10-18T19:39:41.203 +C Fix\sa\sproblem\sin\sFTS\sto\sdo\swith\s^\stokens\sand\sthe\ssnippet()\sfunction. +D 2011-10-19T09:40:49.185 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a162fe39e249b8ed4a65ee947c30152786cfe897 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -62,7 +62,7 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c e12a151b5f0f8d444744554f91dbb89dbf0654df +F ext/fts3/fts3.c 708122f0ed7b7b0aa9813fe302eb40a238956276 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h bc27eebe2c5919115aa1858fdd308a230af6a359 F ext/fts3/fts3_aux.c 0ebfa7b86cf8ff6a0861605fcc63b83ec1b70691 @@ -71,7 +71,7 @@ F ext/fts3/fts3_hash.c 8dd2d06b66c72c628c2732555a32bc0943114914 F ext/fts3/fts3_hash.h 8331fb2206c609f9fc4c4735b9ab5ad6137c88ec F ext/fts3/fts3_icu.c 6c8f395cdf9e1e3afa7fadb7e523dbbf381c6dfa F ext/fts3/fts3_porter.c 8d946908f4812c005d3d33fcbe78418b1f4eb70c -F ext/fts3/fts3_snippet.c 8838a1de5f7df3a559596870caaa4a9895248998 +F ext/fts3/fts3_snippet.c e1f3ed049efa35337d393a0ae971b32c28f4c88d F ext/fts3/fts3_term.c a5457992723455a58804cb75c8cbd8978db5c2ef F ext/fts3/fts3_test.c 24fa13f330db011500acb95590da9eee24951894 F ext/fts3/fts3_tokenizer.c 9ff7ec66ae3c5c0340fa081958e64f395c71a106 @@ -476,7 +476,7 @@ F test/fts3expr.test 5e745b2b6348499d9ef8d59015de3182072c564c F test/fts3expr2.test 18da930352e5693eaa163a3eacf96233b7290d1a F test/fts3fault.test f83e556465bb69dc8bc676339eca408dce4ca246 F test/fts3fault2.test b62a2bc843c20414405f80e5eeb78e39bc68fe53 -F test/fts3first.test 10f42914701d559c9fabfd7725b56c9f1b542fe8 +F test/fts3first.test 8402101caa140802fdea7322652b22caa7662010 F test/fts3malloc.test b86ea33db9e8c58c0c2f8027a9fcadaf6a1568be F test/fts3matchinfo.test 6507fe1c342e542300d65ea637d4110eccf894e6 F test/fts3near.test 2e318ee434d32babd27c167142e2b94ddbab4844 @@ -967,7 +967,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2 -P f9b5b217088a6aeb25eba184ab92d1a842a680a6 -R 4545768a323f698040377ce29e450e89 +P 63ac33c860eb32ce96699f06bf83121cec2ffaca +R 3121ed7658c54e3e7fae4ae1475b5670 U dan -Z a7972271e048cbba7c3ba6dcd813d2e6 +Z 5ad246c0f30695f9ad4b0b18222b341a diff --git a/manifest.uuid b/manifest.uuid index 1ccbaef9b8..4a839fd314 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -63ac33c860eb32ce96699f06bf83121cec2ffaca \ No newline at end of file +2c03b24f4cc6f2c28c9d5b9984320d41b8486c32 \ No newline at end of file diff --git a/test/fts3first.test b/test/fts3first.test index 6e309d0d9d..0c6de520fd 100644 --- a/test/fts3first.test +++ b/test/fts3first.test @@ -18,6 +18,14 @@ ifcapable !fts3 { return } +proc lreverse {L} { + set res [list] + for {set ii [expr [llength $L]-1]} {$ii>=0} {incr ii -1} { + lappend res [lindex $L $ii] + } + set res +} + do_execsql_test 1.0 { CREATE VIRTUAL TABLE x1 USING FTS4(a, b, c); INSERT INTO x1(docid,a,b,c) VALUES(0, 'K H D S T', 'V M N Y K', 'S Z N Q S'); @@ -33,7 +41,7 @@ do_execsql_test 1.0 { INSERT INTO x1(docid,a,b,c) VALUES(10, 'Q I A Q M', 'N D K H C', 'A H T Q Z'); INSERT INTO x1(docid,a,b,c) VALUES(11, 'T E R Q B', 'C I B C B', 'F Z U W R'); INSERT INTO x1(docid,a,b,c) VALUES(12, 'E S V U W', 'T P F W H', 'A M D J Q'); - INSERT INTO x1(docid,a,b,c) VALUES(13, 'X S B T Y', 'U D N D P', 'X Z Y G F'); + INSERT INTO x1(docid,a,b,c) VALUES(13, 'X S B X Y', 'U D N D P', 'X Z Y G F'); INSERT INTO x1(docid,a,b,c) VALUES(14, 'K H A B L', 'S R C C Z', 'D W E H J'); INSERT INTO x1(docid,a,b,c) VALUES(15, 'C E U C C', 'W F M N M', 'T Z U X T'); INSERT INTO x1(docid,a,b,c) VALUES(16, 'Q G C G H', 'H N N B H', 'B Q I H Y'); @@ -45,6 +53,7 @@ do_execsql_test 1.0 { INSERT INTO x2(docid, a, b, c) SELECT docid, a, b, c FROM x1; } + foreach x {1 2} { foreach {tn match res} { 1 "^K" {0 1 14} @@ -62,10 +71,7 @@ foreach x {1 2} { 12 {"K H"} {0 10 14} 13 {"K ^H"} {} } { - set rev [list] - for {set ii [expr [llength $res]-1]} {$ii>=0} {incr ii -1} { - lappend rev [lindex $res $ii] - } + set rev [lreverse $res] do_execsql_test 1.$x.$tn.1 {SELECT docid FROM x1 WHERE x1 MATCH $match} $res do_execsql_test 1.$x.$tn.2 {SELECT docid FROM x2 WHERE x2 MATCH $match} $rev } @@ -76,4 +82,19 @@ foreach x {1 2} { } {} } +foreach {tn match res} { + 1 {^K} {{[K] H D S T} {[K] N J L W} {[K] H A B L}} + 2 {^X} {{[X] Y D L S} {[X] J P K R} {[X] S B X Y}} + 3 {^X Y} {{[X] [Y] D L S} {D [Y] P O I...[X] J P K R} {[X] S B X [Y]}} +} { + set rev [lreverse $res] + + do_execsql_test 1.3.$tn.1 { + SELECT snippet(x1, '[', ']', '...') FROM x1 WHERE x1 MATCH $match + } $res + do_execsql_test 1.3.$tn.2 { + SELECT snippet(x2, '[', ']', '...') FROM x2 WHERE x2 MATCH $match + } $rev +} + finish_test From 8653fa850471e219c34e12bf4a7218422935c152 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 19 Oct 2011 10:18:01 +0000 Subject: [PATCH 09/11] Add tests for FTS ^ searches and matchinfo(). FossilOrigin-Name: 92618c1463fb304cf8057d082b2c7096152dff27 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/fts3first.test | 42 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index f9ee83d4ef..f155d09e14 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\sin\sFTS\sto\sdo\swith\s^\stokens\sand\sthe\ssnippet()\sfunction. -D 2011-10-19T09:40:49.185 +C Add\stests\sfor\sFTS\s^\ssearches\sand\smatchinfo(). +D 2011-10-19T10:18:01.912 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a162fe39e249b8ed4a65ee947c30152786cfe897 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -476,7 +476,7 @@ F test/fts3expr.test 5e745b2b6348499d9ef8d59015de3182072c564c F test/fts3expr2.test 18da930352e5693eaa163a3eacf96233b7290d1a F test/fts3fault.test f83e556465bb69dc8bc676339eca408dce4ca246 F test/fts3fault2.test b62a2bc843c20414405f80e5eeb78e39bc68fe53 -F test/fts3first.test 8402101caa140802fdea7322652b22caa7662010 +F test/fts3first.test 5aa9e82202461a82066427df4cea9188155a4cd5 F test/fts3malloc.test b86ea33db9e8c58c0c2f8027a9fcadaf6a1568be F test/fts3matchinfo.test 6507fe1c342e542300d65ea637d4110eccf894e6 F test/fts3near.test 2e318ee434d32babd27c167142e2b94ddbab4844 @@ -967,7 +967,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2 -P 63ac33c860eb32ce96699f06bf83121cec2ffaca -R 3121ed7658c54e3e7fae4ae1475b5670 +P 2c03b24f4cc6f2c28c9d5b9984320d41b8486c32 +R 402b5e477b47b6aa728a3149c74db091 U dan -Z 5ad246c0f30695f9ad4b0b18222b341a +Z 87cae0b85a25fbf32c49680729e400c4 diff --git a/manifest.uuid b/manifest.uuid index 4a839fd314..4e40bb93ac 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2c03b24f4cc6f2c28c9d5b9984320d41b8486c32 \ No newline at end of file +92618c1463fb304cf8057d082b2c7096152dff27 \ No newline at end of file diff --git a/test/fts3first.test b/test/fts3first.test index 0c6de520fd..e82a23dd0e 100644 --- a/test/fts3first.test +++ b/test/fts3first.test @@ -26,6 +26,14 @@ proc lreverse {L} { set res } +proc mit {blob} { + set scan(littleEndian) i* + set scan(bigEndian) I* + binary scan $blob $scan($::tcl_platform(byteOrder)) r + return $r +} +db func mit mit + do_execsql_test 1.0 { CREATE VIRTUAL TABLE x1 USING FTS4(a, b, c); INSERT INTO x1(docid,a,b,c) VALUES(0, 'K H D S T', 'V M N Y K', 'S Z N Q S'); @@ -54,6 +62,8 @@ do_execsql_test 1.0 { } +# Test queries. +# foreach x {1 2} { foreach {tn match res} { 1 "^K" {0 1 14} @@ -82,6 +92,8 @@ foreach x {1 2} { } {} } +# Test the snippet() function. +# foreach {tn match res} { 1 {^K} {{[K] H D S T} {[K] N J L W} {[K] H A B L}} 2 {^X} {{[X] Y D L S} {[X] J P K R} {[X] S B X Y}} @@ -92,9 +104,39 @@ foreach {tn match res} { do_execsql_test 1.3.$tn.1 { SELECT snippet(x1, '[', ']', '...') FROM x1 WHERE x1 MATCH $match } $res + do_execsql_test 1.3.$tn.2 { SELECT snippet(x2, '[', ']', '...') FROM x2 WHERE x2 MATCH $match } $rev } +# Test matchinfo(). +# +foreach {tn match res} { + 1 {^K} { + {1 3 3 0 0 0 0 0 0} + {1 3 3 0 0 0 0 0 0} + {1 3 3 0 0 0 0 0 0} + } + 2 {^X} { + {0 1 1 0 1 1 1 2 2} + {0 1 1 1 1 1 0 2 2} + {1 1 1 0 1 1 1 2 2} + } + 3 {^X Y} { + {0 1 1 0 1 1 1 2 2 0 6 5 0 5 4 1 4 4} + {0 1 1 1 1 1 0 2 2 1 6 5 0 5 4 0 4 4} + {1 1 1 0 1 1 1 2 2 1 6 5 0 5 4 1 4 4} + } +} { + set rev [lreverse $res] + + do_execsql_test 1.3.$tn.1 { + SELECT mit(matchinfo(x1, 'x')) FROM x1 WHERE x1 MATCH $match + } $res + do_execsql_test 1.3.$tn.2 { + SELECT mit(matchinfo(x2, 'x')) FROM x2 WHERE x2 MATCH $match + } $rev +} + finish_test From d17f70a624dd40670b547ebb50ac35015b35f7b7 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 19 Oct 2011 11:57:13 +0000 Subject: [PATCH 10/11] Change the way ^ tokens work in FTS so that the filtering is done as part of reading the FTS index instead of waiting until an entire doclist has been retrieved and then filtering it. FossilOrigin-Name: 9b58c59eb4efaa38ce50a3ce1b52f9ba578c71d6 --- ext/fts3/fts3.c | 97 ++++++++++++++++++------------------------- ext/fts3/fts3Int.h | 3 +- ext/fts3/fts3_write.c | 26 +++++++++--- manifest | 16 +++---- manifest.uuid | 2 +- 5 files changed, 72 insertions(+), 72 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index e82d1f5e89..e1330004a2 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -2348,65 +2348,53 @@ static void fts3DoclistPhraseMerge( } /* -** When this function is called, pList points to a doclist containing position -** data, length *pnList bytes. This removes all entries from the doclist that -** do not correspond to the first token in a column and overwrites pList -** with the result. *pnList is set to the length of the new doclist before -** returning. -** -** If bDescDoclist is true, then both the input and output are in descending -** order. Otherwise, ascending. +** Argument pList points to a position list nList bytes in size. This +** function checks to see if the position list contains any entries for +** a token in position 0 (of any column). If so, it writes argument iDelta +** to the output buffer pOut, followed by a position list consisting only +** of the entries from pList at position 0, and terminated by an 0x00 byte. +** The value returned is the number of bytes written to pOut (if any). */ -static void fts3DoclistFirstFilter( - int bDescDoclist, /* True if pList is a descending doclist */ - char *pList, /* Buffer containing doclist */ - int *pnList /* IN/OUT: Size of doclist */ +int sqlite3Fts3FirstFilter( + sqlite3_int64 iDelta, /* Varint that may be written to pOut */ + char *pList, /* Position list (no 0x00 term) */ + int nList, /* Size of pList in bytes */ + char *pOut /* Write output here */ ){ + int nOut = 0; + int bWritten = 0; /* True once iDelta has been written */ char *p = pList; - char *pOut = pList; - char *pEnd = &pList[*pnList]; + char *pEnd = &pList[nList]; - sqlite3_int64 iDoc; - sqlite3_int64 iPrev; - int bFirstOut = 0; - - fts3GetDeltaVarint3(&p, pEnd, 0, &iDoc); - while( p ){ - int bWritten = 0; - if( *p!=0x01 ){ - if( *p==0x02 ){ - fts3PutDeltaVarint3(&pOut, bDescDoclist, &iPrev, &bFirstOut, iDoc); - *pOut++ = 0x02; - bWritten = 1; - } - fts3ColumnlistCopy(0, &p); + if( *p!=0x01 ){ + if( *p==0x02 ){ + nOut += sqlite3Fts3PutVarint(&pOut[nOut], iDelta); + pOut[nOut++] = 0x02; + bWritten = 1; } - - while( *p==0x01 ){ - sqlite3_int64 iCol; - p++; - p += sqlite3Fts3GetVarint(p, &iCol); - if( *p==0x02 ){ - if( bWritten==0 ){ - fts3PutDeltaVarint3(&pOut, bDescDoclist, &iPrev, &bFirstOut, iDoc); - bWritten = 1; - } - *pOut++ = 0x01; - pOut += sqlite3Fts3PutVarint(pOut, iCol); - *pOut++ = 0x02; - } - fts3ColumnlistCopy(0, &p); - } - if( bWritten ){ - *pOut++ = 0x00; - } - - assert( *p==0x00 ); - p++; - fts3GetDeltaVarint3(&p, pEnd, bDescDoclist, &iDoc); + fts3ColumnlistCopy(0, &p); } - *pnList = (pOut - pList); + while( pisPrefix ? FTS3_SEGMENT_PREFIX : 0) + | (pTok->bFirst ? FTS3_SEGMENT_FIRST : 0) | (iColumnnColumn ? FTS3_SEGMENT_COLUMN_FILTER : 0); filter.iCol = iColumn; filter.zTerm = pTok->z; @@ -3580,10 +3569,6 @@ static void fts3EvalPhraseMergeToken( ){ assert( iToken!=p->iDoclistToken ); - if( p->aToken[iToken].bFirst ){ - fts3DoclistFirstFilter(pTab->bDescIdx, pList, &nList); - } - if( pList==0 ){ sqlite3_free(p->doclist.aAll); p->doclist.aAll = 0; diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index c9b291c6cc..5f0f8dea01 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -429,6 +429,7 @@ int sqlite3Fts3SegReaderCursor( #define FTS3_SEGMENT_COLUMN_FILTER 0x00000004 #define FTS3_SEGMENT_PREFIX 0x00000008 #define FTS3_SEGMENT_SCAN 0x00000010 +#define FTS3_SEGMENT_FIRST 0x00000020 /* Type passed as 4th argument to SegmentReaderIterate() */ struct Fts3SegFilter { @@ -468,8 +469,8 @@ int sqlite3Fts3GetVarint32(const char *, int *); int sqlite3Fts3VarintLen(sqlite3_uint64); void sqlite3Fts3Dequote(char *); void sqlite3Fts3DoclistPrev(int,char*,int,char**,sqlite3_int64*,int*,u8*); - int sqlite3Fts3EvalPhraseStats(Fts3Cursor *, Fts3Expr *, u32 *); +int sqlite3Fts3FirstFilter(sqlite3_int64, char *, int, char *); /* fts3_tokenizer.c */ const char *sqlite3Fts3NextToken(const char *, int *); diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 40c8e2f9ad..855e97764e 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -2509,6 +2509,7 @@ int sqlite3Fts3SegReaderStep( int isColFilter = (pCsr->pFilter->flags & FTS3_SEGMENT_COLUMN_FILTER); int isPrefix = (pCsr->pFilter->flags & FTS3_SEGMENT_PREFIX); int isScan = (pCsr->pFilter->flags & FTS3_SEGMENT_SCAN); + int isFirst = (pCsr->pFilter->flags & FTS3_SEGMENT_FIRST); Fts3SegReader **apSegment = pCsr->apSegment; int nSegment = pCsr->nSegment; @@ -2568,6 +2569,7 @@ int sqlite3Fts3SegReaderStep( assert( isIgnoreEmpty || (isRequirePos && !isColFilter) ); if( nMerge==1 && !isIgnoreEmpty + && !isFirst && (p->bDescIdx==0 || fts3SegReaderIsPending(apSegment[0])==0) ){ pCsr->nDoclist = apSegment[0]->nDoclist; @@ -2633,12 +2635,24 @@ int sqlite3Fts3SegReaderStep( } pCsr->aBuffer = aNew; } - nDoclist += sqlite3Fts3PutVarint(&pCsr->aBuffer[nDoclist], iDelta); - iPrev = iDocid; - if( isRequirePos ){ - memcpy(&pCsr->aBuffer[nDoclist], pList, nList); - nDoclist += nList; - pCsr->aBuffer[nDoclist++] = '\0'; + + if( isFirst ){ + char *a = &pCsr->aBuffer[nDoclist]; + int nWrite; + + nWrite = sqlite3Fts3FirstFilter(iDelta, pList, nList, a); + if( nWrite ){ + iPrev = iDocid; + nDoclist += nWrite; + } + }else{ + nDoclist += sqlite3Fts3PutVarint(&pCsr->aBuffer[nDoclist], iDelta); + iPrev = iDocid; + if( isRequirePos ){ + memcpy(&pCsr->aBuffer[nDoclist], pList, nList); + nDoclist += nList; + pCsr->aBuffer[nDoclist++] = '\0'; + } } } diff --git a/manifest b/manifest index f155d09e14..df0a118c08 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stests\sfor\sFTS\s^\ssearches\sand\smatchinfo(). -D 2011-10-19T10:18:01.912 +C Change\sthe\sway\s^\stokens\swork\sin\sFTS\sso\sthat\sthe\sfiltering\sis\sdone\sas\spart\sof\sreading\sthe\sFTS\sindex\sinstead\sof\swaiting\suntil\san\sentire\sdoclist\shas\sbeen\sretrieved\sand\sthen\sfiltering\sit. +D 2011-10-19T11:57:13.985 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a162fe39e249b8ed4a65ee947c30152786cfe897 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -62,9 +62,9 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 708122f0ed7b7b0aa9813fe302eb40a238956276 +F ext/fts3/fts3.c 064b660a11ae29651b647fa7c3e9954d901ab58a F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h bc27eebe2c5919115aa1858fdd308a230af6a359 +F ext/fts3/fts3Int.h 7a0deb219371d29b8d385fb5e929ede2bdc7c239 F ext/fts3/fts3_aux.c 0ebfa7b86cf8ff6a0861605fcc63b83ec1b70691 F ext/fts3/fts3_expr.c dd0facbede8fd7d1376670cc6154f1fef3a4c5bc F ext/fts3/fts3_hash.c 8dd2d06b66c72c628c2732555a32bc0943114914 @@ -77,7 +77,7 @@ F ext/fts3/fts3_test.c 24fa13f330db011500acb95590da9eee24951894 F ext/fts3/fts3_tokenizer.c 9ff7ec66ae3c5c0340fa081958e64f395c71a106 F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3 F ext/fts3/fts3_tokenizer1.c 0dde8f307b8045565cf63797ba9acfaff1c50c68 -F ext/fts3/fts3_write.c 567380f2d6671df16cfbb56324b321c71d5ab0d3 +F ext/fts3/fts3_write.c aaf0885fd5d37c6869071ee58b5aa3ba07cc0d87 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 @@ -967,7 +967,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2 -P 2c03b24f4cc6f2c28c9d5b9984320d41b8486c32 -R 402b5e477b47b6aa728a3149c74db091 +P 92618c1463fb304cf8057d082b2c7096152dff27 +R c2d7bdd9838ac956262a194e0ae43b40 U dan -Z 87cae0b85a25fbf32c49680729e400c4 +Z 8d1b6b02c37e3947f3ee71176bf9a674 diff --git a/manifest.uuid b/manifest.uuid index 4e40bb93ac..4b15a85377 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -92618c1463fb304cf8057d082b2c7096152dff27 \ No newline at end of file +9b58c59eb4efaa38ce50a3ce1b52f9ba578c71d6 \ No newline at end of file From 97439483869d65fdbbbaba1695df93bb70050a0d Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 19 Oct 2011 15:52:48 +0000 Subject: [PATCH 11/11] Have FTS3 ignore ^ prefixes. The ^ syntax is only supported on FTS4 tables. FossilOrigin-Name: df36ac948179f37b432a88701b6c79299e073ce8 --- ext/fts3/fts3.c | 4 ++-- ext/fts3/fts3Int.h | 2 +- ext/fts3/fts3_expr.c | 7 +++++-- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- test/fts3defer.test | 17 +++++++++++------ test/fts3first.test | 21 +++++++++++++++++++++ 7 files changed, 51 insertions(+), 22 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index e1330004a2..c5f51a4ccf 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -2894,8 +2894,8 @@ static int fts3FilterMethod( return SQLITE_NOMEM; } - rc = sqlite3Fts3ExprParse(p->pTokenizer, p->azColumn, p->nColumn, - iCol, zQuery, -1, &pCsr->pExpr + rc = sqlite3Fts3ExprParse(p->pTokenizer, p->azColumn, p->bHasStat, + p->nColumn, iCol, zQuery, -1, &pCsr->pExpr ); if( rc!=SQLITE_OK ){ if( rc==SQLITE_ERROR ){ diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index 5f0f8dea01..89672da971 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -489,7 +489,7 @@ void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *, const char *); /* fts3_expr.c */ int sqlite3Fts3ExprParse(sqlite3_tokenizer *, - char **, int, int, const char *, int, Fts3Expr ** + char **, int, int, int, const char *, int, Fts3Expr ** ); void sqlite3Fts3ExprFree(Fts3Expr *); #ifdef SQLITE_TEST diff --git a/ext/fts3/fts3_expr.c b/ext/fts3/fts3_expr.c index e6193a1392..46add008c7 100644 --- a/ext/fts3/fts3_expr.c +++ b/ext/fts3/fts3_expr.c @@ -93,6 +93,7 @@ typedef struct ParseContext ParseContext; struct ParseContext { sqlite3_tokenizer *pTokenizer; /* Tokenizer module */ const char **azCol; /* Array of column names for fts3 table */ + int bFts4; /* True to allow FTS4-only syntax */ int nCol; /* Number of entries in azCol[] */ int iDefaultCol; /* Default column to query */ int isNot; /* True if getNextNode() sees a unary - */ @@ -187,7 +188,7 @@ static int getNextToken( ){ pParse->isNot = 1; iStart--; - }else if( iStart>0 && z[iStart-1]=='^' ){ + }else if( pParse->bFts4 && iStart>0 && z[iStart-1]=='^' ){ pRet->pPhrase->aToken[0].bFirst = 1; iStart--; }else{ @@ -741,6 +742,7 @@ exprparse_out: int sqlite3Fts3ExprParse( sqlite3_tokenizer *pTokenizer, /* Tokenizer module */ char **azCol, /* Array of column names for fts3 table */ + int bFts4, /* True to allow FTS4-only syntax */ int nCol, /* Number of entries in azCol[] */ int iDefaultCol, /* Default column to query */ const char *z, int n, /* Text of MATCH query */ @@ -754,6 +756,7 @@ int sqlite3Fts3ExprParse( sParse.nCol = nCol; sParse.iDefaultCol = iDefaultCol; sParse.nNest = 0; + sParse.bFts4 = bFts4; if( z==0 ){ *ppExpr = 0; return SQLITE_OK; @@ -943,7 +946,7 @@ static void fts3ExprTest( } rc = sqlite3Fts3ExprParse( - pTokenizer, azCol, nCol, nCol, zExpr, nExpr, &pExpr + pTokenizer, azCol, 0, nCol, nCol, zExpr, nExpr, &pExpr ); if( rc!=SQLITE_OK && rc!=SQLITE_NOMEM ){ sqlite3_result_error(context, "Error parsing expression", -1); diff --git a/manifest b/manifest index df0a118c08..50449c9718 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\sway\s^\stokens\swork\sin\sFTS\sso\sthat\sthe\sfiltering\sis\sdone\sas\spart\sof\sreading\sthe\sFTS\sindex\sinstead\sof\swaiting\suntil\san\sentire\sdoclist\shas\sbeen\sretrieved\sand\sthen\sfiltering\sit. -D 2011-10-19T11:57:13.985 +C Have\sFTS3\signore\s^\sprefixes.\sThe\s^\ssyntax\sis\sonly\ssupported\son\sFTS4\stables. +D 2011-10-19T15:52:48.921 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a162fe39e249b8ed4a65ee947c30152786cfe897 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -62,11 +62,11 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 064b660a11ae29651b647fa7c3e9954d901ab58a +F ext/fts3/fts3.c e8ee5c78a3c7715b597db4e5c890ec658ee0401a F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h 7a0deb219371d29b8d385fb5e929ede2bdc7c239 +F ext/fts3/fts3Int.h 74afec80054924976b2d3a21eda17e33c6ff6b84 F ext/fts3/fts3_aux.c 0ebfa7b86cf8ff6a0861605fcc63b83ec1b70691 -F ext/fts3/fts3_expr.c dd0facbede8fd7d1376670cc6154f1fef3a4c5bc +F ext/fts3/fts3_expr.c 34dd0edf1e2723ce22d88039dc1ad687a95fb1bb F ext/fts3/fts3_hash.c 8dd2d06b66c72c628c2732555a32bc0943114914 F ext/fts3/fts3_hash.h 8331fb2206c609f9fc4c4735b9ab5ad6137c88ec F ext/fts3/fts3_icu.c 6c8f395cdf9e1e3afa7fadb7e523dbbf381c6dfa @@ -469,14 +469,14 @@ F test/fts3corrupt.test 7b0f91780ca36118d73324ec803187208ad33b32 F test/fts3corrupt2.test 6d96efae2f8a6af3eeaf283aba437e6d0e5447ba F test/fts3cov.test e0fb00d8b715ddae4a94c305992dfc3ef70353d7 F test/fts3d.test 95fb3c862cbc4297c93fceb9a635543744e9ef52 -F test/fts3defer.test b7bdf79da91365b00e7c21d70e9d0c617b9306b9 +F test/fts3defer.test 2ea3fa028f8d9523f9c33dd8acc4555d567ea4ac F test/fts3defer2.test 35867d33ba6db03f6c73bd6f5fc333ae14f68c81 F test/fts3e.test 1f6c6ac9cc8b772ca256e6b22aaeed50c9350851 F test/fts3expr.test 5e745b2b6348499d9ef8d59015de3182072c564c F test/fts3expr2.test 18da930352e5693eaa163a3eacf96233b7290d1a F test/fts3fault.test f83e556465bb69dc8bc676339eca408dce4ca246 F test/fts3fault2.test b62a2bc843c20414405f80e5eeb78e39bc68fe53 -F test/fts3first.test 5aa9e82202461a82066427df4cea9188155a4cd5 +F test/fts3first.test dbdedd20914c8d539aa3206c9b34a23775644641 F test/fts3malloc.test b86ea33db9e8c58c0c2f8027a9fcadaf6a1568be F test/fts3matchinfo.test 6507fe1c342e542300d65ea637d4110eccf894e6 F test/fts3near.test 2e318ee434d32babd27c167142e2b94ddbab4844 @@ -967,7 +967,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2 -P 92618c1463fb304cf8057d082b2c7096152dff27 -R c2d7bdd9838ac956262a194e0ae43b40 +P 9b58c59eb4efaa38ce50a3ce1b52f9ba578c71d6 +R b05473465da4d8af6fcb4e85b9ce8174 U dan -Z 8d1b6b02c37e3947f3ee71176bf9a674 +Z 1ed48059f0e9f692b253c99c9d63572e diff --git a/manifest.uuid b/manifest.uuid index 4b15a85377..0f7fa81637 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9b58c59eb4efaa38ce50a3ce1b52f9ba578c71d6 \ No newline at end of file +df36ac948179f37b432a88701b6c79299e073ce8 \ No newline at end of file diff --git a/test/fts3defer.test b/test/fts3defer.test index 7fbe6b14ea..bc50874e4a 100644 --- a/test/fts3defer.test +++ b/test/fts3defer.test @@ -427,12 +427,17 @@ foreach {tn setup} { } {8 15 26 92 96} } - do_select_test 7.1 { - SELECT rowid FROM t1 WHERE t1 MATCH '^zm mjpavjuhw' - } {56 62} - do_select_test 7.2 { - SELECT rowid FROM t1 WHERE t1 MATCH '^azavwm zm' - } {43} + if {$tn>1} { + # These tests will not work with $tn==1, as in this case table t1 is + # created using FTS3. The ^ syntax is only available with FTS4 tables. + # + do_select_test 7.1 { + SELECT rowid FROM t1 WHERE t1 MATCH '^zm mjpavjuhw' + } {56 62} + do_select_test 7.2 { + SELECT rowid FROM t1 WHERE t1 MATCH '^azavwm zm' + } {43} + } } set testprefix fts3defer diff --git a/test/fts3first.test b/test/fts3first.test index e82a23dd0e..673f818deb 100644 --- a/test/fts3first.test +++ b/test/fts3first.test @@ -18,6 +18,8 @@ ifcapable !fts3 { return } +set testprefix fts3first + proc lreverse {L} { set res [list] for {set ii [expr [llength $L]-1]} {$ii>=0} {incr ii -1} { @@ -139,4 +141,23 @@ foreach {tn match res} { } $rev } +# Test that ^ is ignored for FTS3 tables. +# +do_execsql_test 2.1 { + CREATE VIRTUAL TABLE x3 USING fts3; + INSERT INTO x3 VALUES('A B C'); + INSERT INTO x3 VALUES('B A C'); + + CREATE VIRTUAL TABLE x4 USING fts4; + INSERT INTO x4 VALUES('A B C'); + INSERT INTO x4 VALUES('B A C'); +} + +do_execsql_test 2.2.1 { + SELECT * FROM x3 WHERE x3 MATCH '^A'; +} {{A B C} {B A C}} +do_execsql_test 2.2.2 { + SELECT * FROM x4 WHERE x4 MATCH '^A'; +} {{A B C}} + finish_test