Merge the fts4-content branch with the trunk.

FossilOrigin-Name: 8a4077057ddeb08e8edc5f20a75abaaba7a278ba

ext/fts3/fts3.c

@@ -468,6 +468,7 @@ static int fts3DisconnectMethod(sqlite3_vtab *pVtab){
   sqlite3_free(p->zSegmentsTbl);
   sqlite3_free(p->zReadExprlist);
   sqlite3_free(p->zWriteExprlist);
+  sqlite3_free(p->zContentTbl);
 
   /* Invoke the tokenizer destructor to free the tokenizer. */
   p->pTokenizer->pModule->xDestroy(p->pTokenizer);
@@ -507,16 +508,19 @@ static void fts3DbExec(
 ** The xDestroy() virtual table method.
 */
 static int fts3DestroyMethod(sqlite3_vtab *pVtab){
-  int rc = SQLITE_OK;              /* Return code */
   Fts3Table *p = (Fts3Table *)pVtab;
-  sqlite3 *db = p->db;
+  int rc = SQLITE_OK;              /* Return code */
+  const char *zDb = p->zDb;        /* Name of database (e.g. "main", "temp") */
+  sqlite3 *db = p->db;             /* Database handle */
 
   /* Drop the shadow tables */
-  fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_content'", p->zDb, p->zName);
+  if( p->zContentTbl==0 ){
-  fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segments'", p->zDb,p->zName);
+    fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_content'", zDb, p->zName);
-  fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segdir'", p->zDb, p->zName);
+  }
-  fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_docsize'", p->zDb, p->zName);
+  fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segments'", zDb,p->zName);
-  fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_stat'", p->zDb, p->zName);
+  fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segdir'", zDb, p->zName);
+  fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_docsize'", zDb, p->zName);
+  fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_stat'", zDb, p->zName);
 
   /* If everything has worked, invoke fts3DisconnectMethod() to free the
   ** memory associated with the Fts3Table structure and return SQLITE_OK.
@@ -578,9 +582,11 @@ static void fts3DeclareVtab(int *pRc, Fts3Table *p){
 static int fts3CreateTables(Fts3Table *p){
   int rc = SQLITE_OK;             /* Return code */
   int i;                          /* Iterator variable */
-  char *zContentCols;             /* Columns of %_content table */
   sqlite3 *db = p->db;            /* The database connection */
 
+  if( p->zContentTbl==0 ){
+    char *zContentCols;           /* Columns of %_content table */
+
     /* Create a list of user columns for the content table */
     zContentCols = sqlite3_mprintf("docid INTEGER PRIMARY KEY");
     for(i=0; zContentCols && i<p->nColumn; i++){
@@ -595,6 +601,8 @@ static int fts3CreateTables(Fts3Table *p){
        p->zDb, p->zName, zContentCols
     );
     sqlite3_free(zContentCols);
+  }
+
   /* Create other tables */
   fts3DbExec(&rc, db, 
       "CREATE TABLE %Q.'%q_segments'(blockid INTEGER PRIMARY KEY, block BLOB);",
@@ -745,8 +753,8 @@ static char *fts3QuoteId(char const *zInput){
 }
 
 /*
-** Return a list of comma separated SQL expressions that could be used
+** Return a list of comma separated SQL expressions and a FROM clause that 
-** in a SELECT statement such as the following:
+** could be used in a SELECT statement such as the following:
 **
 **     SELECT <list of expressions> FROM %_content AS x ...
 **
@@ -757,7 +765,7 @@ static char *fts3QuoteId(char const *zInput){
 ** table has the three user-defined columns "a", "b", and "c", the following
 ** string is returned:
 **
-**     "docid, unzip(x.'a'), unzip(x.'b'), unzip(x.'c')"
+**     "docid, unzip(x.'a'), unzip(x.'b'), unzip(x.'c') FROM %_content AS x"
 **
 ** The pointer returned points to a buffer allocated by sqlite3_malloc(). It
 ** is the responsibility of the caller to eventually free it.
@@ -773,6 +781,7 @@ static char *fts3ReadExprList(Fts3Table *p, const char *zFunc, int *pRc){
   char *zFunction;
   int i;
 
+  if( p->zContentTbl==0 ){
     if( !zFunc ){
       zFunction = "";
     }else{
@@ -783,6 +792,17 @@ static char *fts3ReadExprList(Fts3Table *p, const char *zFunc, int *pRc){
       fts3Appendf(pRc, &zRet, ",%s(x.'c%d%q')", zFunction, i, p->azColumn[i]);
     }
     sqlite3_free(zFree);
+  }else{
+    fts3Appendf(pRc, &zRet, "rowid");
+    for(i=0; i<p->nColumn; i++){
+      fts3Appendf(pRc, &zRet, ", x.'%q'", p->azColumn[i]);
+    }
+  }
+  fts3Appendf(pRc, &zRet, "FROM '%q'.'%q%s' AS x", 
+      p->zDb,
+      (p->zContentTbl ? p->zContentTbl : p->zName),
+      (p->zContentTbl ? "" : "_content")
+  );
   return zRet;
 }
 
@@ -906,6 +926,91 @@ static int fts3PrefixParameter(
   return SQLITE_OK;
 }
 
+/*
+** This function is called when initializing an FTS4 table that uses the
+** content=xxx option. It determines the number of and names of the columns
+** of the new FTS4 table.
+**
+** The third argument passed to this function is the value passed to the
+** config=xxx option (i.e. "xxx"). This function queries the database for
+** a table of that name. If found, the output variables are populated
+** as follows:
+**
+**   *pnCol:   Set to the number of columns table xxx has,
+**
+**   *pnStr:   Set to the total amount of space required to store a copy
+**             of each columns name, including the nul-terminator.
+**
+**   *pazCol:  Set to point to an array of *pnCol strings. Each string is
+**             the name of the corresponding column in table xxx. The array
+**             and its contents are allocated using a single allocation. It
+**             is the responsibility of the caller to free this allocation
+**             by eventually passing the *pazCol value to sqlite3_free().
+**
+** If the table cannot be found, an error code is returned and the output
+** variables are undefined. Or, if an OOM is encountered, SQLITE_NOMEM is
+** returned (and the output variables are undefined).
+*/
+static int fts3ContentColumns(
+  sqlite3 *db,                    /* Database handle */
+  const char *zDb,                /* Name of db (i.e. "main", "temp" etc.) */
+  const char *zTbl,               /* Name of content table */
+  const char ***pazCol,           /* OUT: Malloc'd array of column names */
+  int *pnCol,                     /* OUT: Size of array *pazCol */
+  int *pnStr                      /* OUT: Bytes of string content */
+){
+  int rc = SQLITE_OK;             /* Return code */
+  char *zSql;                     /* "SELECT *" statement on zTbl */  
+  sqlite3_stmt *pStmt = 0;        /* Compiled version of zSql */
+
+  zSql = sqlite3_mprintf("SELECT * FROM %Q.%Q", zDb, zTbl);
+  if( !zSql ){
+    rc = SQLITE_NOMEM;
+  }else{
+    rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0);
+  }
+  sqlite3_free(zSql);
+
+  if( rc==SQLITE_OK ){
+    const char **azCol;           /* Output array */
+    int nStr = 0;                 /* Size of all column names (incl. 0x00) */
+    int nCol;                     /* Number of table columns */
+    int i;                        /* Used to iterate through columns */
+
+    /* Loop through the returned columns. Set nStr to the number of bytes of
+    ** space required to store a copy of each column name, including the
+    ** nul-terminator byte.  */
+    nCol = sqlite3_column_count(pStmt);
+    for(i=0; i<nCol; i++){
+      const char *zCol = sqlite3_column_name(pStmt, i);
+      nStr += strlen(zCol) + 1;
+    }
+
+    /* Allocate and populate the array to return. */
+    azCol = (const char **)sqlite3_malloc(sizeof(char *) * nCol + nStr);
+    if( azCol==0 ){
+      rc = SQLITE_NOMEM;
+    }else{
+      char *p = (char *)&azCol[nCol];
+      for(i=0; i<nCol; i++){
+        const char *zCol = sqlite3_column_name(pStmt, i);
+        int n = strlen(zCol)+1;
+        memcpy(p, zCol, n);
+        azCol[i] = p;
+        p += n;
+      }
+    }
+    sqlite3_finalize(pStmt);
+
+    /* Set the output variables. */
+    *pnCol = nCol;
+    *pnStr = nStr;
+    *pazCol = azCol;
+  }
+
+  return rc;
+}
+
 /*
 ** This function is the implementation of both the xConnect and xCreate
 ** methods of the FTS3 virtual table.
@@ -950,6 +1055,7 @@ static int fts3InitVtab(
   char *zPrefix = 0;              /* Prefix parameter value (or NULL) */
   char *zCompress = 0;            /* compress=? parameter (or NULL) */
   char *zUncompress = 0;          /* uncompress=? parameter (or NULL) */
+  char *zContent = 0;             /* content=? parameter (or NULL) */
 
   assert( strlen(argv[0])==4 );
   assert( (sqlite3_strnicmp(argv[0], "fts4", 4)==0 && isFts4)
@@ -993,13 +1099,13 @@ static int fts3InitVtab(
       struct Fts4Option {
         const char *zOpt;
         int nOpt;
-        char **pzVar;
       } aFts4Opt[] = {
-        { "matchinfo",   9, 0 },            /* 0 -> MATCHINFO */
+        { "matchinfo",   9 },     /* 0 -> MATCHINFO */
-        { "prefix",      6, 0 },            /* 1 -> PREFIX */
+        { "prefix",      6 },     /* 1 -> PREFIX */
-        { "compress",    8, 0 },            /* 2 -> COMPRESS */
+        { "compress",    8 },     /* 2 -> COMPRESS */
-        { "uncompress", 10, 0 },            /* 3 -> UNCOMPRESS */
+        { "uncompress", 10 },     /* 3 -> UNCOMPRESS */
-        { "order",       5, 0 }             /* 4 -> ORDER */
+        { "order",       5 },     /* 4 -> ORDER */
+        { "content",     7 }      /* 5 -> CONTENT */
       };
 
       int iOpt;
@@ -1045,13 +1151,20 @@ static int fts3InitVtab(
 
             case 4:               /* ORDER */
               if( (strlen(zVal)!=3 || sqlite3_strnicmp(zVal, "asc", 3)) 
-               && (strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "desc", 3)) 
+               && (strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "desc", 4)) 
               ){
                 *pzErr = sqlite3_mprintf("unrecognized order: %s", zVal);
                 rc = SQLITE_ERROR;
               }
               bDescIdx = (zVal[0]=='d' || zVal[0]=='D');
               break;
+
+            default:              /* CONTENT */
+              assert( iOpt==5 );
+              sqlite3_free(zUncompress);
+              zContent = zVal;
+              zVal = 0;
+              break;
           }
         }
         sqlite3_free(zVal);
@@ -1064,6 +1177,26 @@ static int fts3InitVtab(
       aCol[nCol++] = z;
     }
   }
+
+  /* If a content=xxx option was specified, the following:
+  **
+  **   1. Ignore any compress= and uncompress= options.
+  **
+  **   2. If no column names were specified as part of the CREATE VIRTUAL
+  **      TABLE statement, use all columns from the content table.
+  */
+  if( rc==SQLITE_OK && zContent ){
+    sqlite3_free(zCompress); 
+    sqlite3_free(zUncompress); 
+    zCompress = 0;
+    zUncompress = 0;
+    if( nCol==0 ){
+      sqlite3_free(aCol); 
+      aCol = 0;
+      rc = fts3ContentColumns(db, argv[1], zContent, &aCol, &nCol, &nString);
+    }
+    assert( rc!=SQLITE_OK || nCol>0 );
+  }
   if( rc!=SQLITE_OK ) goto fts3_init_out;
 
   if( nCol==0 ){
@@ -1108,6 +1241,8 @@ static int fts3InitVtab(
   p->bHasDocsize = (isFts4 && bNoDocsize==0);
   p->bHasStat = isFts4;
   p->bDescIdx = bDescIdx;
+  p->zContentTbl = zContent;
+  zContent = 0;
   TESTONLY( p->inTransaction = -1 );
   TESTONLY( p->mxSavepoint = -1 );
 
@@ -1169,6 +1304,7 @@ fts3_init_out:
   sqlite3_free(aIndex);
   sqlite3_free(zCompress);
   sqlite3_free(zUncompress);
+  sqlite3_free(zContent);
   sqlite3_free((void *)aCol);
   if( rc!=SQLITE_OK ){
     if( p ){
@@ -1320,35 +1456,64 @@ static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){
   return SQLITE_OK;
 }
 
+/*
+** If pCsr->pStmt has not been prepared (i.e. if pCsr->pStmt==0), then
+** compose and prepare an SQL statement of the form:
+**
+**    "SELECT <columns> FROM %_content WHERE rowid = ?"
+**
+** (or the equivalent for a content=xxx table) and set pCsr->pStmt to
+** it. If an error occurs, return an SQLite error code.
+**
+** Otherwise, set *ppStmt to point to pCsr->pStmt and return SQLITE_OK.
+*/
+static int fts3CursorSeekStmt(Fts3Cursor *pCsr, sqlite3_stmt **ppStmt){
+  int rc = SQLITE_OK;
+  if( pCsr->pStmt==0 ){
+    Fts3Table *p = (Fts3Table *)pCsr->base.pVtab;
+    char *zSql;
+    zSql = sqlite3_mprintf("SELECT %s WHERE rowid = ?", p->zReadExprlist);
+    if( !zSql ) return SQLITE_NOMEM;
+    rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0);
+    sqlite3_free(zSql);
+  }
+  *ppStmt = pCsr->pStmt;
+  return rc;
+}
+
 /*
 ** Position the pCsr->pStmt statement so that it is on the row
 ** of the %_content table that contains the last match.  Return
 ** SQLITE_OK on success.  
 */
 static int fts3CursorSeek(sqlite3_context *pContext, Fts3Cursor *pCsr){
+  int rc = SQLITE_OK;
   if( pCsr->isRequireSeek ){
+    sqlite3_stmt *pStmt = 0;
+
+    rc = fts3CursorSeekStmt(pCsr, &pStmt);
+    if( rc==SQLITE_OK ){
       sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId);
       pCsr->isRequireSeek = 0;
       if( SQLITE_ROW==sqlite3_step(pCsr->pStmt) ){
         return SQLITE_OK;
       }else{
-      int rc = sqlite3_reset(pCsr->pStmt);
+        rc = sqlite3_reset(pCsr->pStmt);
-      if( rc==SQLITE_OK ){
+        if( rc==SQLITE_OK && ((Fts3Table *)pCsr->base.pVtab)->zContentTbl==0 ){
           /* If no row was found and no error has occured, then the %_content
           ** table is missing a row that is present in the full-text index.
-        ** The data structures are corrupt.
+          ** The data structures are corrupt.  */
-        */
           rc = FTS_CORRUPT_VTAB;
-      }
           pCsr->isEof = 1;
-      if( pContext ){
+        }
+      }
+    }
+  }
+
+  if( rc!=SQLITE_OK && pContext ){
     sqlite3_result_error_code(pContext, rc);
   }
   return rc;
-    }
-  }else{
-    return SQLITE_OK;
-  }
 }
 
 /*
@@ -1788,7 +1953,7 @@ static int fts3PoslistPhraseMerge(
   char **pp1,                     /* IN/OUT: Left input list */
   char **pp2                      /* IN/OUT: Right input list */
 ){
-  char *p = (pp ? *pp : 0);
+  char *p = *pp;
   char *p1 = *pp1;
   char *p2 = *pp2;
   int iCol1 = 0;
@@ -1814,7 +1979,7 @@ static int fts3PoslistPhraseMerge(
       sqlite3_int64 iPos1 = 0;
       sqlite3_int64 iPos2 = 0;
 
-      if( pp && iCol1 ){
+      if( iCol1 ){
         *p++ = POS_COLUMN;
         p += sqlite3Fts3PutVarint(p, iCol1);
       }
@@ -1829,13 +1994,6 @@ static int fts3PoslistPhraseMerge(
          || (isExact==0 && iPos2>iPos1 && iPos2<=iPos1+nToken) 
         ){
           sqlite3_int64 iSave;
-          if( !pp ){
-            fts3PoslistCopy(0, &p2);
-            fts3PoslistCopy(0, &p1);
-            *pp1 = p1;
-            *pp2 = p2;
-            return 1;
-          }
           iSave = isSaveLeft ? iPos1 : iPos2;
           fts3PutDeltaVarint(&p, &iPrev, iSave+2); iPrev -= 2;
           pSave = 0;
@@ -1887,7 +2045,7 @@ static int fts3PoslistPhraseMerge(
   fts3PoslistCopy(0, &p1);
   *pp1 = p1;
   *pp2 = p2;
-  if( !pp || *pp==p ){
+  if( *pp==p ){
     return 0;
   }
   *p++ = 0x00;
@@ -2190,6 +2348,56 @@ static void fts3DoclistPhraseMerge(
   *pnRight = p - aOut;
 }
 
+/*
+** Argument pList points to a position list nList bytes in size. This
+** function checks to see if the position list contains any entries for
+** a token in position 0 (of any column). If so, it writes argument iDelta
+** to the output buffer pOut, followed by a position list consisting only
+** of the entries from pList at position 0, and terminated by an 0x00 byte.
+** The value returned is the number of bytes written to pOut (if any).
+*/
+int sqlite3Fts3FirstFilter(
+  sqlite3_int64 iDelta,           /* Varint that may be written to pOut */
+  char *pList,                    /* Position list (no 0x00 term) */
+  int nList,                      /* Size of pList in bytes */
+  char *pOut                      /* Write output here */
+){
+  int nOut = 0;
+  int bWritten = 0;               /* True once iDelta has been written */
+  char *p = pList;
+  char *pEnd = &pList[nList];
+
+  if( *p!=0x01 ){
+    if( *p==0x02 ){
+      nOut += sqlite3Fts3PutVarint(&pOut[nOut], iDelta);
+      pOut[nOut++] = 0x02;
+      bWritten = 1;
+    }
+    fts3ColumnlistCopy(0, &p);
+  }
+
+  while( p<pEnd && *p==0x01 ){
+    sqlite3_int64 iCol;
+    p++;
+    p += sqlite3Fts3GetVarint(p, &iCol);
+    if( *p==0x02 ){
+      if( bWritten==0 ){
+        nOut += sqlite3Fts3PutVarint(&pOut[nOut], iDelta);
+        bWritten = 1;
+      }
+      pOut[nOut++] = 0x01;
+      nOut += sqlite3Fts3PutVarint(&pOut[nOut], iCol);
+      pOut[nOut++] = 0x02;
+    }
+    fts3ColumnlistCopy(0, &p);
+  }
+  if( bWritten ){
+    pOut[nOut++] = 0x00;
+  }
+
+  return nOut;
+}
+
 
 /*
 ** Merge all doclists in the TermSelect.aaOutput[] array into a single
@@ -2546,6 +2754,7 @@ static int fts3TermSelect(
 
   filter.flags = FTS3_SEGMENT_IGNORE_EMPTY | FTS3_SEGMENT_REQUIRE_POS
         | (pTok->isPrefix ? FTS3_SEGMENT_PREFIX : 0)
+        | (pTok->bFirst ? FTS3_SEGMENT_FIRST : 0)
         | (iColumn<p->nColumn ? FTS3_SEGMENT_COLUMN_FILTER : 0);
   filter.iCol = iColumn;
   filter.zTerm = pTok->z;
@@ -2686,8 +2895,8 @@ static int fts3FilterMethod(
       return SQLITE_NOMEM;
     }
 
-    rc = sqlite3Fts3ExprParse(p->pTokenizer, p->azColumn, p->nColumn, 
+    rc = sqlite3Fts3ExprParse(p->pTokenizer, p->azColumn, p->bHasStat, 
-        iCol, zQuery, -1, &pCsr->pExpr
+        p->nColumn, iCol, zQuery, -1, &pCsr->pExpr
     );
     if( rc!=SQLITE_OK ){
       if( rc==SQLITE_ERROR ){
@@ -2714,22 +2923,23 @@ static int fts3FilterMethod(
   ** row by docid.
   */
   if( idxNum==FTS3_FULLSCAN_SEARCH ){
-    const char *zSort = (pCsr->bDesc ? "DESC" : "ASC");
+    zSql = sqlite3_mprintf(
-    const char *zTmpl = "SELECT %s FROM %Q.'%q_content' AS x ORDER BY docid %s";
+        "SELECT %s ORDER BY rowid %s",
-    zSql = sqlite3_mprintf(zTmpl, p->zReadExprlist, p->zDb, p->zName, zSort);
+        p->zReadExprlist, (pCsr->bDesc ? "DESC" : "ASC")
-  }else{
+    );
-    const char *zTmpl = "SELECT %s FROM %Q.'%q_content' AS x WHERE docid = ?";
+    if( zSql ){
-    zSql = sqlite3_mprintf(zTmpl, p->zReadExprlist, p->zDb, p->zName);
-  }
-  if( !zSql ) return SQLITE_NOMEM;
       rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0);
       sqlite3_free(zSql);
-  if( rc!=SQLITE_OK ) return rc;
+    }else{
-
+      rc = SQLITE_NOMEM;
-  if( idxNum==FTS3_DOCID_SEARCH ){
-    rc = sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]);
-    if( rc!=SQLITE_OK ) return rc;
     }
+  }else if( idxNum==FTS3_DOCID_SEARCH ){
+    rc = fts3CursorSeekStmt(pCsr, &pCsr->pStmt);
+    if( rc==SQLITE_OK ){
+      rc = sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]);
+    }
+  }
+  if( rc!=SQLITE_OK ) return rc;
 
   return fts3NextMethod(pCursor);
 }
@@ -2782,7 +2992,7 @@ static int fts3ColumnMethod(
     sqlite3_result_blob(pContext, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT);
   }else{
     rc = fts3CursorSeek(0, pCsr);
-    if( rc==SQLITE_OK ){
+    if( rc==SQLITE_OK && sqlite3_data_count(pCsr->pStmt)>(iCol+1) ){
       sqlite3_result_value(pContext, sqlite3_column_value(pCsr->pStmt, iCol+1));
     }
   }
@@ -3075,15 +3285,22 @@ static int fts3RenameMethod(
   sqlite3 *db = p->db;            /* Database connection */
   int rc;                         /* Return Code */
 
+  /* As it happens, the pending terms table is always empty here. This is
+  ** because an "ALTER TABLE RENAME TABLE" statement inside a transaction 
+  ** always opens a savepoint transaction. And the xSavepoint() method 
+  ** flushes the pending terms table. But leave the (no-op) call to
+  ** PendingTermsFlush() in in case that changes.
+  */
+  assert( p->nPendingData==0 );
   rc = sqlite3Fts3PendingTermsFlush(p);
-  if( rc!=SQLITE_OK ){
-    return rc;
-  }
 
+  if( p->zContentTbl==0 ){
     fts3DbExec(&rc, db,
       "ALTER TABLE %Q.'%q_content'  RENAME TO '%q_content';",
       p->zDb, p->zName, zName
     );
+  }
+
   if( p->bHasDocsize ){
     fts3DbExec(&rc, db,
       "ALTER TABLE %Q.'%q_docsize'  RENAME TO '%q_docsize';",
@@ -3442,21 +3659,20 @@ static int fts3EvalPhraseLoad(
 */
 static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){
   int iToken;                     /* Used to iterate through phrase tokens */
-  int rc = SQLITE_OK;             /* Return code */
   char *aPoslist = 0;             /* Position list for deferred tokens */
   int nPoslist = 0;               /* Number of bytes in aPoslist */
   int iPrev = -1;                 /* Token number of previous deferred token */
 
   assert( pPhrase->doclist.bFreeList==0 );
 
-  for(iToken=0; rc==SQLITE_OK && iToken<pPhrase->nToken; iToken++){
+  for(iToken=0; iToken<pPhrase->nToken; iToken++){
     Fts3PhraseToken *pToken = &pPhrase->aToken[iToken];
     Fts3DeferredToken *pDeferred = pToken->pDeferred;
 
     if( pDeferred ){
       char *pList;
       int nList;
-      rc = sqlite3Fts3DeferredTokenList(pDeferred, &pList, &nList);
+      int rc = sqlite3Fts3DeferredTokenList(pDeferred, &pList, &nList);
       if( rc!=SQLITE_OK ) return rc;
 
       if( pList==0 ){
@@ -3557,6 +3773,7 @@ static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){
    && p->nToken==1 
    && pFirst->pSegcsr 
    && pFirst->pSegcsr->bLookup 
+   && pFirst->bFirst==0
   ){
     /* Use the incremental approach. */
     int iCol = (p->iColumn >= pTab->nColumn ? -1 : p->iColumn);
@@ -3786,7 +4003,7 @@ static void fts3EvalTokenCosts(
   Fts3Expr ***ppOr,               /* Write new OR root to *(*ppOr)++ */
   int *pRc                        /* IN/OUT: Error code */
 ){
-  if( *pRc==SQLITE_OK && pExpr ){
+  if( *pRc==SQLITE_OK ){
     if( pExpr->eType==FTSQUERY_PHRASE ){
       Fts3Phrase *pPhrase = pExpr->pPhrase;
       int i;
@@ -3800,6 +4017,11 @@ static void fts3EvalTokenCosts(
         *pRc = sqlite3Fts3MsrOvfl(pCsr, pTC->pToken->pSegcsr, &pTC->nOvfl);
       }
     }else if( pExpr->eType!=FTSQUERY_NOT ){
+      assert( pExpr->eType==FTSQUERY_OR
+           || pExpr->eType==FTSQUERY_AND
+           || pExpr->eType==FTSQUERY_NEAR
+      );
+      assert( pExpr->pLeft && pExpr->pRight );
       if( pExpr->eType==FTSQUERY_OR ){
         pRoot = pExpr->pLeft;
         **ppOr = pRoot;
@@ -3904,6 +4126,15 @@ static int fts3EvalSelectDeferred(
   int nMinEst = 0;                /* The minimum count for any phrase so far. */
   int nLoad4 = 1;                 /* (Phrases that will be loaded)^4. */
 
+  /* Tokens are never deferred for FTS tables created using the content=xxx
+  ** option. The reason being that it is not guaranteed that the content
+  ** table actually contains the same data as the index. To prevent this from
+  ** causing any problems, the deferred token optimization is completely
+  ** disabled for content=xxx tables. */
+  if( pTab->zContentTbl ){
+    return SQLITE_OK;
+  }
+
   /* Count the tokens in this AND/NEAR cluster. If none of the doclists
   ** associated with the tokens spill onto overflow pages, or if there is
   ** only 1 token, exit early. No tokens to defer in this case. */

ext/fts3/fts3Int.h

@@ -191,6 +191,7 @@ struct Fts3Table {
   int nColumn;                    /* number of named columns in virtual table */
   char **azColumn;                /* column names.  malloced */
   sqlite3_tokenizer *pTokenizer;  /* tokenizer for inserts and queries */
+  char *zContentTbl;              /* content=xxx option, or NULL */
 
   /* Precompiled statements used by the implementation. Each of these 
   ** statements is run and reset within a single virtual table API call. 
@@ -231,7 +232,7 @@ struct Fts3Table {
   int nPendingData;               /* Current bytes of pending data */
   sqlite_int64 iPrevDocid;        /* Docid of most recently inserted document */
 
-#if defined(SQLITE_DEBUG)
+#if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST)
   /* State variables used for validating that the transaction control
   ** methods of the virtual table are called at appropriate times.  These
   ** values do not contribution to the FTS computation; they are used for
@@ -316,6 +317,7 @@ struct Fts3PhraseToken {
   char *z;                        /* Text of the token */
   int n;                          /* Number of bytes in buffer z */
   int isPrefix;                   /* True if token ends with a "*" character */
+  int bFirst;                     /* True if token must appear at position 0 */
 
   /* Variables above this point are populated when the expression is
   ** parsed (by code in fts3_expr.c). Below this point the variables are
@@ -434,6 +436,7 @@ int sqlite3Fts3SegReaderCursor(
 #define FTS3_SEGMENT_COLUMN_FILTER 0x00000004
 #define FTS3_SEGMENT_PREFIX        0x00000008
 #define FTS3_SEGMENT_SCAN          0x00000010
+#define FTS3_SEGMENT_FIRST         0x00000020
 
 /* Type passed as 4th argument to SegmentReaderIterate() */
 struct Fts3SegFilter {
@@ -473,8 +476,8 @@ int sqlite3Fts3GetVarint32(const char *, int *);
 int sqlite3Fts3VarintLen(sqlite3_uint64);
 void sqlite3Fts3Dequote(char *);
 void sqlite3Fts3DoclistPrev(int,char*,int,char**,sqlite3_int64*,int*,u8*);
-
 int sqlite3Fts3EvalPhraseStats(Fts3Cursor *, Fts3Expr *, u32 *);
+int sqlite3Fts3FirstFilter(sqlite3_int64, char *, int, char *);
 
 /* fts3_tokenizer.c */
 const char *sqlite3Fts3NextToken(const char *, int *);
@@ -493,7 +496,7 @@ void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *, const char *);
 
 /* fts3_expr.c */
 int sqlite3Fts3ExprParse(sqlite3_tokenizer *, 
-  char **, int, int, const char *, int, Fts3Expr **
+  char **, int, int, int, const char *, int, Fts3Expr **
 );
 void sqlite3Fts3ExprFree(Fts3Expr *);
 #ifdef SQLITE_TEST

ext/fts3/fts3_expr.c

@@ -93,6 +93,7 @@ typedef struct ParseContext ParseContext;
 struct ParseContext {
   sqlite3_tokenizer *pTokenizer;      /* Tokenizer module */
   const char **azCol;                 /* Array of column names for fts3 table */
+  int bFts4;                          /* True to allow FTS4-only syntax */
   int nCol;                           /* Number of entries in azCol[] */
   int iDefaultCol;                    /* Default column to query */
   int isNot;                          /* True if getNextNode() sees a unary - */
@@ -180,10 +181,22 @@ static int getNextToken(
           pRet->pPhrase->aToken[0].isPrefix = 1;
           iEnd++;
         }
-        if( !sqlite3_fts3_enable_parentheses && iStart>0 && z[iStart-1]=='-' ){
+
+        while( 1 ){
+          if( !sqlite3_fts3_enable_parentheses 
+           && iStart>0 && z[iStart-1]=='-' 
+          ){
             pParse->isNot = 1;
+            iStart--;
+          }else if( pParse->bFts4 && iStart>0 && z[iStart-1]=='^' ){
+            pRet->pPhrase->aToken[0].bFirst = 1;
+            iStart--;
+          }else{
+            break;
           }
         }
+
+      }
       nConsumed = iEnd;
     }
 
@@ -281,6 +294,7 @@ static int getNextString(
 
         pToken->n = nByte;
         pToken->isPrefix = (iEnd<nInput && zInput[iEnd]=='*');
+        pToken->bFirst = (iBegin>0 && zInput[iBegin-1]=='^');
         nToken = ii+1;
       }
     }
@@ -732,6 +746,7 @@ exprparse_out:
 int sqlite3Fts3ExprParse(
   sqlite3_tokenizer *pTokenizer,      /* Tokenizer module */
   char **azCol,                       /* Array of column names for fts3 table */
+  int bFts4,                          /* True to allow FTS4-only syntax */
   int nCol,                           /* Number of entries in azCol[] */
   int iDefaultCol,                    /* Default column to query */
   const char *z, int n,               /* Text of MATCH query */
@@ -745,6 +760,7 @@ int sqlite3Fts3ExprParse(
   sParse.nCol = nCol;
   sParse.iDefaultCol = iDefaultCol;
   sParse.nNest = 0;
+  sParse.bFts4 = bFts4;
   if( z==0 ){
     *ppExpr = 0;
     return SQLITE_OK;
@@ -934,7 +950,7 @@ static void fts3ExprTest(
   }
 
   rc = sqlite3Fts3ExprParse(
-      pTokenizer, azCol, nCol, nCol, zExpr, nExpr, &pExpr
+      pTokenizer, azCol, 0, nCol, nCol, zExpr, nExpr, &pExpr
   );
   if( rc!=SQLITE_OK && rc!=SQLITE_NOMEM ){
     sqlite3_result_error(context, "Error parsing expression", -1);

ext/fts3/fts3_snippet.c

@@ -368,6 +368,7 @@ static int fts3SnippetFindPositions(Fts3Expr *pExpr, int iPhrase, void *ctx){
     int iFirst = 0;
     pPhrase->pList = pCsr;
     fts3GetDeltaPosition(&pCsr, &iFirst);
+    assert( iFirst>=0 );
     pPhrase->pHead = pCsr;
     pPhrase->pTail = pCsr;
     pPhrase->iHead = iFirst;
@@ -1409,7 +1410,7 @@ void sqlite3Fts3Offsets(
 
       if( !pTerm ){
         /* All offsets for this column have been gathered. */
-        break;
+        rc = SQLITE_DONE;
       }else{
         assert( iCurrent<=iMinPos );
         if( 0==(0xFE&*pTerm->pList) ){
@@ -1426,7 +1427,7 @@ void sqlite3Fts3Offsets(
               "%d %d %d %d ", iCol, pTerm-sCtx.aTerm, iStart, iEnd-iStart
           );
           rc = fts3StringAppend(&res, aBuffer, -1);
-        }else if( rc==SQLITE_DONE ){
+        }else if( rc==SQLITE_DONE && pTab->zContentTbl==0 ){
           rc = FTS_CORRUPT_VTAB;
         }
       }

ext/fts3/fts3_write.c

@@ -256,7 +256,7 @@ static int fts3SqlStmt(
 /* 4  */  "DELETE FROM %Q.'%q_segdir'",
 /* 5  */  "DELETE FROM %Q.'%q_docsize'",
 /* 6  */  "DELETE FROM %Q.'%q_stat'",
-/* 7  */  "SELECT %s FROM %Q.'%q_content' AS x WHERE rowid=?",
+/* 7  */  "SELECT %s WHERE rowid=?",
 /* 8  */  "SELECT (SELECT max(idx) FROM %Q.'%q_segdir' WHERE level = ?) + 1",
 /* 9  */  "INSERT INTO %Q.'%q_segments'(blockid, block) VALUES(?, ?)",
 /* 10 */  "SELECT coalesce((SELECT max(blockid) FROM %Q.'%q_segments') + 1, 1)",
@@ -298,7 +298,7 @@ static int fts3SqlStmt(
     if( eStmt==SQL_CONTENT_INSERT ){
       zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName, p->zWriteExprlist);
     }else if( eStmt==SQL_SELECT_CONTENT_BY_ROWID ){
-      zSql = sqlite3_mprintf(azSql[eStmt], p->zReadExprlist, p->zDb, p->zName);
+      zSql = sqlite3_mprintf(azSql[eStmt], p->zReadExprlist);
     }else{
       zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName);
     }
@@ -409,17 +409,24 @@ static void fts3SqlExec(
 ** not what users expect when they get SQLITE_LOCKED_SHAREDCACHE. It can
 ** still happen if the user reads data directly from the %_segments or
 ** %_segdir tables instead of going through FTS3 though.
+**
+** This reasoning does not apply to a content=xxx table.
 */
 int sqlite3Fts3ReadLock(Fts3Table *p){
   int rc;                         /* Return code */
   sqlite3_stmt *pStmt;            /* Statement used to obtain lock */
 
+  if( p->zContentTbl==0 ){
     rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pStmt, 0);
     if( rc==SQLITE_OK ){
       sqlite3_bind_null(pStmt, 1);
       sqlite3_step(pStmt);
       rc = sqlite3_reset(pStmt);
     }
+  }else{
+    rc = SQLITE_OK;
+  }
+
   return rc;
 }
 
@@ -780,6 +787,18 @@ static int fts3InsertData(
   int rc;                         /* Return code */
   sqlite3_stmt *pContentInsert;   /* INSERT INTO %_content VALUES(...) */
 
+  if( p->zContentTbl ){
+    sqlite3_value *pRowid = apVal[p->nColumn+3];
+    if( sqlite3_value_type(pRowid)==SQLITE_NULL ){
+      pRowid = apVal[1];
+    }
+    if( sqlite3_value_type(pRowid)!=SQLITE_INTEGER ){
+      return SQLITE_CONSTRAINT;
+    }
+    *piDocid = sqlite3_value_int64(pRowid);
+    return SQLITE_OK;
+  }
+
   /* Locate the statement handle used to insert data into the %_content
   ** table. The SQL for this statement is:
   **
@@ -830,14 +849,16 @@ static int fts3InsertData(
 ** Remove all data from the FTS3 table. Clear the hash table containing
 ** pending terms.
 */
-static int fts3DeleteAll(Fts3Table *p){
+static int fts3DeleteAll(Fts3Table *p, int bContent){
   int rc = SQLITE_OK;             /* Return code */
 
   /* Discard the contents of the pending-terms hash table. */
   sqlite3Fts3PendingTermsClear(p);
 
-  /* Delete everything from the %_content, %_segments and %_segdir tables. */
+  /* Delete everything from the shadow tables. Except, leave %_content as
-  fts3SqlExec(&rc, p, SQL_DELETE_ALL_CONTENT, 0);
+  ** is if bContent is false.  */
+  assert( p->zContentTbl==0 || bContent==0 );
+  if( bContent ) fts3SqlExec(&rc, p, SQL_DELETE_ALL_CONTENT, 0);
   fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGMENTS, 0);
   fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGDIR, 0);
   if( p->bHasDocsize ){
@@ -2125,6 +2146,11 @@ static void fts3SegWriterFree(SegmentWriter *pWriter){
 static int fts3IsEmpty(Fts3Table *p, sqlite3_value *pRowid, int *pisEmpty){
   sqlite3_stmt *pStmt;
   int rc;
+  if( p->zContentTbl ){
+    /* If using the content=xxx option, assume the table is never empty */
+    *pisEmpty = 0;
+    rc = SQLITE_OK;
+  }else{
     rc = fts3SqlStmt(p, SQL_IS_EMPTY, &pStmt, &pRowid);
     if( rc==SQLITE_OK ){
       if( SQLITE_ROW==sqlite3_step(pStmt) ){
@@ -2132,6 +2158,7 @@ static int fts3IsEmpty(Fts3Table *p, sqlite3_value *pRowid, int *pisEmpty){
       }
       rc = sqlite3_reset(pStmt);
     }
+  }
   return rc;
 }
 
@@ -2482,6 +2509,7 @@ int sqlite3Fts3SegReaderStep(
   int isColFilter =    (pCsr->pFilter->flags & FTS3_SEGMENT_COLUMN_FILTER);
   int isPrefix =       (pCsr->pFilter->flags & FTS3_SEGMENT_PREFIX);
   int isScan =         (pCsr->pFilter->flags & FTS3_SEGMENT_SCAN);
+  int isFirst =        (pCsr->pFilter->flags & FTS3_SEGMENT_FIRST);
 
   Fts3SegReader **apSegment = pCsr->apSegment;
   int nSegment = pCsr->nSegment;
@@ -2541,6 +2569,7 @@ int sqlite3Fts3SegReaderStep(
     assert( isIgnoreEmpty || (isRequirePos && !isColFilter) );
     if( nMerge==1 
      && !isIgnoreEmpty 
+     && !isFirst 
      && (p->bDescIdx==0 || fts3SegReaderIsPending(apSegment[0])==0)
     ){
       pCsr->nDoclist = apSegment[0]->nDoclist;
@@ -2606,6 +2635,17 @@ int sqlite3Fts3SegReaderStep(
             }
             pCsr->aBuffer = aNew;
           }
+
+          if( isFirst ){
+            char *a = &pCsr->aBuffer[nDoclist];
+            int nWrite;
+           
+            nWrite = sqlite3Fts3FirstFilter(iDelta, pList, nList, a);
+            if( nWrite ){
+              iPrev = iDocid;
+              nDoclist += nWrite;
+            }
+          }else{
             nDoclist += sqlite3Fts3PutVarint(&pCsr->aBuffer[nDoclist], iDelta);
             iPrev = iDocid;
             if( isRequirePos ){
@@ -2614,6 +2654,7 @@ int sqlite3Fts3SegReaderStep(
               pCsr->aBuffer[nDoclist++] = '\0';
             }
           }
+        }
 
         fts3SegReaderSort(apSegment, nMerge, j, xCmp);
       }
@@ -2789,7 +2830,7 @@ static void fts3DecodeIntArray(
 static void fts3InsertDocsize(
   int *pRC,                       /* Result code */
   Fts3Table *p,                   /* Table into which to insert */
-  u32 *aSz          /* Sizes of each column */
+  u32 *aSz                        /* Sizes of each column, in tokens */
 ){
   char *pBlob;             /* The BLOB encoding of the document size */
   int nBlob;               /* Number of bytes in the BLOB */
@@ -2911,6 +2952,86 @@ static int fts3DoOptimize(Fts3Table *p, int bReturnDone){
   return (rc==SQLITE_OK && bReturnDone && bSeenDone) ? SQLITE_DONE : rc;
 }
 
+/*
+** This function is called when the user executes the following statement:
+**
+**     INSERT INTO <tbl>(<tbl>) VALUES('rebuild');
+**
+** The entire FTS index is discarded and rebuilt. If the table is one 
+** created using the content=xxx option, then the new index is based on
+** the current contents of the xxx table. Otherwise, it is rebuilt based
+** on the contents of the %_content table.
+*/
+static int fts3DoRebuild(Fts3Table *p){
+  int rc;                         /* Return Code */
+
+  rc = fts3DeleteAll(p, 0);
+  if( rc==SQLITE_OK ){
+    u32 *aSz = 0;
+    u32 *aSzIns;
+    u32 *aSzDel;
+    sqlite3_stmt *pStmt = 0;
+    int nEntry = 0;
+
+    /* Compose and prepare an SQL statement to loop through the content table */
+    char *zSql = sqlite3_mprintf("SELECT %s" , p->zReadExprlist);
+    if( !zSql ){
+      rc = SQLITE_NOMEM;
+    }else{
+      rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
+      sqlite3_free(zSql);
+    }
+
+    if( rc==SQLITE_OK ){
+      int nByte = sizeof(u32) * (p->nColumn+1)*3;
+      aSz = (u32 *)sqlite3_malloc(nByte);
+      if( aSz==0 ){
+        rc = SQLITE_NOMEM;
+      }else{
+        memset(aSz, 0, nByte);
+        aSzIns = &aSz[p->nColumn+1];
+        aSzDel = &aSzIns[p->nColumn+1];
+      }
+    }
+
+    while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
+      int iCol;
+      rc = fts3PendingTermsDocid(p, sqlite3_column_int64(pStmt, 0));
+      aSz[p->nColumn] = 0;
+      for(iCol=0; rc==SQLITE_OK && iCol<p->nColumn; iCol++){
+        const char *z = (const char *) sqlite3_column_text(pStmt, iCol+1);
+        rc = fts3PendingTermsAdd(p, z, iCol, &aSz[iCol]);
+        aSz[p->nColumn] += sqlite3_column_bytes(pStmt, iCol+1);
+      }
+      if( p->bHasDocsize ){
+        fts3InsertDocsize(&rc, p, aSz);
+      }
+      if( rc!=SQLITE_OK ){
+        sqlite3_finalize(pStmt);
+        pStmt = 0;
+      }else{
+        nEntry++;
+        for(iCol=0; iCol<=p->nColumn; iCol++){
+          aSzIns[iCol] += aSz[iCol];
+        }
+      }
+    }
+    if( p->bHasStat ){
+      fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nEntry);
+    }
+    sqlite3_free(aSz);
+
+    if( pStmt ){
+      int rc2 = sqlite3_finalize(pStmt);
+      if( rc==SQLITE_OK ){
+        rc = rc2;
+      }
+    }
+  }
+
+  return rc;
+}
+
 /*
 ** Handle a 'special' INSERT of the form:
 **
@@ -2928,6 +3049,8 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){
     return SQLITE_NOMEM;
   }else if( nVal==8 && 0==sqlite3_strnicmp(zVal, "optimize", 8) ){
     rc = fts3DoOptimize(p, 0);
+  }else if( nVal==7 && 0==sqlite3_strnicmp(zVal, "rebuild", 7) ){
+    rc = fts3DoRebuild(p);
 #ifdef SQLITE_TEST
   }else if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){
     p->nNodeSize = atoi(&zVal[9]);
@@ -3008,6 +3131,7 @@ int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *pCsr){
         for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){
           Fts3PhraseToken *pPT = pDef->pToken;
           if( (pDef->iCol>=p->nColumn || pDef->iCol==i)
+           && (pPT->bFirst==0 || iPos==0)
            && (pPT->n==nToken || (pPT->isPrefix && pPT->n<nToken))
            && (0==memcmp(zToken, pPT->z, pPT->n))
           ){
@@ -3099,14 +3223,18 @@ static int fts3DeleteByRowid(
       /* Deleting this row means the whole table is empty. In this case
       ** delete the contents of all three tables and throw away any
       ** data in the pendingTerms hash table.  */
-      rc = fts3DeleteAll(p);
+      rc = fts3DeleteAll(p, 1);
       *pnDoc = *pnDoc - 1;
     }else{
       sqlite3_int64 iRemove = sqlite3_value_int64(pRowid);
       rc = fts3PendingTermsDocid(p, iRemove);
       fts3DeleteTerms(&rc, p, pRowid, aSzDel);
+      if( p->zContentTbl==0 ){
         fts3SqlExec(&rc, p, SQL_DELETE_CONTENT, &pRowid);
         if( sqlite3_changes(p->db) ) *pnDoc = *pnDoc - 1;
+      }else{
+        *pnDoc = *pnDoc - 1;
+      }
       if( p->bHasDocsize ){
         fts3SqlExec(&rc, p, SQL_DELETE_DOCSIZE, &pRowid);
       }
@@ -3166,7 +3294,7 @@ int sqlite3Fts3UpdateMethod(
   ** detect the conflict and return SQLITE_CONSTRAINT before beginning to
   ** modify the database file.
   */
-  if( nArg>1 ){
+  if( nArg>1 && p->zContentTbl==0 ){
     /* Find the value object that holds the new rowid value. */
     sqlite3_value *pNewRowid = apVal[3+p->nColumn];
     if( sqlite3_value_type(pNewRowid)==SQLITE_NULL ){
@@ -3217,7 +3345,9 @@ int sqlite3Fts3UpdateMethod(
   if( nArg>1 && rc==SQLITE_OK ){
     if( bInsertDone==0 ){
       rc = fts3InsertData(p, apVal, pRowid);
-      if( rc==SQLITE_CONSTRAINT ) rc = FTS_CORRUPT_VTAB;
+      if( rc==SQLITE_CONSTRAINT && p->zContentTbl==0 ){
+        rc = FTS_CORRUPT_VTAB;
+      }
     }
     if( rc==SQLITE_OK && (!isRemove || *pRowid!=p->iPrevDocid ) ){
       rc = fts3PendingTermsDocid(p, *pRowid);

manifest

@@ -1,5 +1,5 @@
-C Fix\sthe\svirtual\stable\srename\slogic\sso\sthat\sit\sworks\seven\sif\sthe\sdatabase\nencoding\sis\ssomething\sother\sthan\sUTF8.\nTicket\s[8290242b2a9a81683]
+C Merge\sthe\sfts4-content\sbranch\swith\sthe\strunk.
-D 2011-10-18T22:07:47.722
+D 2011-10-19T16:20:40.613
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in a162fe39e249b8ed4a65ee947c30152786cfe897
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -62,22 +62,22 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0
 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a
 F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9
 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
-F ext/fts3/fts3.c 19f36945148cfd4ee3655b5fac0879ba4e0f3117
+F ext/fts3/fts3.c 9c125699baf1c03a3d8d53fb7d8d27ab4ad7d6eb
 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
-F ext/fts3/fts3Int.h 59c5a9475fed5d76c70a4763103b3c8e60424a68
+F ext/fts3/fts3Int.h def7a900f98c5ab5fa4772e922bfa219d5097f05
 F ext/fts3/fts3_aux.c 0ebfa7b86cf8ff6a0861605fcc63b83ec1b70691
-F ext/fts3/fts3_expr.c 61ceee7c9698c7ae1ba55ab6ff7961456a8b465a
+F ext/fts3/fts3_expr.c f5df26bddf46a5916b2a5f80c4027996e92b7b15
 F ext/fts3/fts3_hash.c 8dd2d06b66c72c628c2732555a32bc0943114914
 F ext/fts3/fts3_hash.h 8331fb2206c609f9fc4c4735b9ab5ad6137c88ec
 F ext/fts3/fts3_icu.c 6c8f395cdf9e1e3afa7fadb7e523dbbf381c6dfa
 F ext/fts3/fts3_porter.c 8d946908f4812c005d3d33fcbe78418b1f4eb70c
-F ext/fts3/fts3_snippet.c 19a906f8ed73ad8b670dfc271ceae7b3338c157e
+F ext/fts3/fts3_snippet.c 1f9ee6a8e0e242649645968dcec4deb253d86c2a
 F ext/fts3/fts3_term.c a5457992723455a58804cb75c8cbd8978db5c2ef
 F ext/fts3/fts3_test.c 24fa13f330db011500acb95590da9eee24951894
 F ext/fts3/fts3_tokenizer.c 9ff7ec66ae3c5c0340fa081958e64f395c71a106
 F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3
 F ext/fts3/fts3_tokenizer1.c 0dde8f307b8045565cf63797ba9acfaff1c50c68
-F ext/fts3/fts3_write.c ffe13acc3867ea6b0fc8b9cfbf904bfae64eac84
+F ext/fts3/fts3_write.c f2545f59a4cc2eb6739acb3d026b8a91a1f3d429
 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9
 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100
 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9
@@ -459,7 +459,7 @@ F test/fts3ak.test bd14deafe9d1586e8e9bf032411026ac4f8c925d
 F test/fts3al.test 07d64326e79bbdbab20ee87fc3328fbf01641c9f
 F test/fts3am.test 218aa6ba0dfc50c7c16b2022aac5c6be593d08d8
 F test/fts3an.test a49ccadc07a2f7d646ec1b81bc09da2d85a85b18
-F test/fts3ao.test 60a15590d3c8578e943e4a149524b16b9bc1be92
+F test/fts3ao.test e7b80272efcced57d1d087a9da5c690dd7c21fd9
 F test/fts3atoken.test 402ef2f7c2fb4b3d4fa0587df6441c1447e799b3
 F test/fts3auto.test c1a30b37002b7c764a96937fbc71065b73d69494
 F test/fts3aux1.test 0b02743955d56fc0d4d66236a26177bd1b726de0
@@ -471,23 +471,25 @@ F test/fts3corrupt.test 7b0f91780ca36118d73324ec803187208ad33b32
 F test/fts3corrupt2.test 6d96efae2f8a6af3eeaf283aba437e6d0e5447ba
 F test/fts3cov.test e0fb00d8b715ddae4a94c305992dfc3ef70353d7
 F test/fts3d.test bf640d79722b720fa1c81834c48cdaa45d531b1a
-F test/fts3defer.test ffd4e07f79a09660d4b3e2613b041ab9b6100d91
+F test/fts3defer.test 2ea3fa028f8d9523f9c33dd8acc4555d567ea4ac
 F test/fts3defer2.test 35867d33ba6db03f6c73bd6f5fc333ae14f68c81
 F test/fts3e.test 1f6c6ac9cc8b772ca256e6b22aaeed50c9350851
 F test/fts3expr.test 5e745b2b6348499d9ef8d59015de3182072c564c
 F test/fts3expr2.test 18da930352e5693eaa163a3eacf96233b7290d1a
 F test/fts3fault.test f83e556465bb69dc8bc676339eca408dce4ca246
-F test/fts3fault2.test dc96203af6ba31ce20163fc35460e1556e8edf4d
+F test/fts3fault2.test b62a2bc843c20414405f80e5eeb78e39bc68fe53
-F test/fts3malloc.test 9c8cc3f885bb4dfc66d0460c52f68f45e4710d1b
+F test/fts3first.test dbdedd20914c8d539aa3206c9b34a23775644641
-F test/fts3matchinfo.test 08a82d18cc08abb28aec41d412b4c2ef25ba6a5f
+F test/fts3malloc.test b86ea33db9e8c58c0c2f8027a9fcadaf6a1568be
+F test/fts3matchinfo.test 6507fe1c342e542300d65ea637d4110eccf894e6
 F test/fts3near.test 2e318ee434d32babd27c167142e2b94ddbab4844
-F test/fts3prefix.test 36246609111ec1683f7ea5ed27666ce2cefb5676
+F test/fts3prefix.test b36d4f00b128a51e7b386cc013a874246d9d7dc1
 F test/fts3query.test ef79d31fdb355d094baec1c1b24b60439a1fb8a2
 F test/fts3rnd.test 1320d8826a845e38a96e769562bf83d7a92a15d0
 F test/fts3shared.test 8bb266521d7c5495c0ae522bb4d376ad5387d4a2
 F test/fts3snippet.test 8e956051221a34c7daeb504f023cb54d5fa5a8b2
-F test/fts3sort.test 9a5176c9317bb545ec5f144d62e6fedb4da6c66e
+F test/fts3sort.test 95be0b19d7e41c44b29014f13ea8bddd495fd659
 F test/fts4aa.test 6e7f90420b837b2c685f3bcbe84c868492d40a68
+F test/fts4content.test c5f531ecfc3d446b90032cae212549dbbb18dd78
 F test/func.test 6c5ce11e3a0021ca3c0649234e2d4454c89110ca
 F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f
 F test/func3.test 001021e5b88bd02a3b365a5c5fd8f6f49d39744a
@@ -622,7 +624,7 @@ F test/pageropt.test 9191867ed19a2b3db6c42d1b36b6fbc657cd1ab0
 F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0
 F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16
 F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025
-F test/permutations.test ad17319066a90e2db71823c3ff104795ffc71b31
+F test/permutations.test 522823b47238cb1754198f80817fe9f9158ede55
 F test/pragma.test 1ea0c85be853135bb7468e6eed48ee12b04794d4
 F test/pragma2.test 3a55f82b954242c642f8342b17dffc8b47472947
 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552
@@ -970,7 +972,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06
 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
 F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a
 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381
-P 54aecd929867606d14a062b501abbfb6f5f05e37
+P d65f63531c3f8e3e55e656f049240714a3d7433f df36ac948179f37b432a88701b6c79299e073ce8
-R dccf2fe03db8f585fac4df0bc3670756
+R 3877ede50d6923a62a70ae0714da94fb
-U drh
+U dan
-Z aaf98f5abdd914fb531f49ab6760c2d4
+Z e6c406cc06dcce98e86f6b11f422dc50

manifest.uuid

@@ -1 +1 @@
-d65f63531c3f8e3e55e656f049240714a3d7433f
+8a4077057ddeb08e8edc5f20a75abaaba7a278ba

test/fts3ao.test

@@ -200,6 +200,9 @@ do_test fts3ao-4.7 {
     SELECT * FROM t5;
   }
 } {{the quick brown fox} {jumped over the} {lazy dog}}
+do_execsql_test fts3ao-4.8 {
+  SELECT snippet(t5, '[', ']') FROM t5 WHERE t5 MATCH 'the'
+} {{[the] quick brown fox} {jumped over [the]}}
 
 # Test that it is possible to rename an FTS4 table. Renaming an FTS4 table
 # involves renaming the extra %_docsize and %_stat tables.

test/fts3defer.test

@@ -426,6 +426,18 @@ foreach {tn setup} {
       SELECT rowid FROM t1 WHERE t1 MATCH '"jk xduvfhk" OR "zm azavwm"'
     } {8 15 26 92 96}
   }
+
+  if {$tn>1} {
+    # These tests will not work with $tn==1, as in this case table t1 is
+    # created using FTS3. The ^ syntax is only available with FTS4 tables.
+    #
+    do_select_test 7.1 {
+      SELECT rowid FROM t1 WHERE t1 MATCH '^zm mjpavjuhw'
+    } {56 62}
+    do_select_test 7.2 {
+      SELECT rowid FROM t1 WHERE t1 MATCH '^azavwm zm'
+    } {43}
+  }
 }
 
 set testprefix fts3defer

test/fts3fault2.test

@@ -82,4 +82,53 @@ do_faultsim_test 2.1 -prep {
   faultsim_test_result {0 {a * 1 1 a 0 1 1 b * 1 1 b 0 1 1 c * 1 1 c 0 1 1 x * 1 1 x 1 1 1 y * 1 1 y 1 1 1 z * 1 1 z 1 1 1}}
 }
 
+do_faultsim_test 3.0 -faults oom* -prep {
+  faultsim_delete_and_reopen
+  db eval { CREATE TABLE 'xx yy'(a, b); }
+} -body {
+  execsql {
+    CREATE VIRTUAL TABLE tt USING fts4(content="xx yy");
+  }
+} -test {
+  faultsim_test_result {0 {}}
+}
+
+do_faultsim_test 3.1 -faults oom* -prep {
+  faultsim_delete_and_reopen
+  db func zip zip
+  db func unzip unzip
+} -body {
+  execsql {
+    CREATE VIRTUAL TABLE tt USING fts4(compress=zip, uncompress=unzip);
+  }
+} -test {
+  faultsim_test_result {0 {}}
+}
+
+do_test 4.0 {
+  faultsim_delete_and_reopen
+  execsql {
+    CREATE VIRTUAL TABLE ft USING fts4(a, b);
+    INSERT INTO ft VALUES('U U T C O', 'F N D E S');
+    INSERT INTO ft VALUES('P H X G B', 'I D M R U');
+    INSERT INTO ft VALUES('P P X D M', 'Y V N T C');
+    INSERT INTO ft VALUES('Z L Q O W', 'D F U N Q');
+    INSERT INTO ft VALUES('A J D U P', 'C H M Q E');
+    INSERT INTO ft VALUES('P S A O H', 'S Z C W D');
+    INSERT INTO ft VALUES('T B N L W', 'C A K T I');
+    INSERT INTO ft VALUES('K E Z L O', 'L L Y C E');
+    INSERT INTO ft VALUES('C R E S V', 'Q V F W P');
+    INSERT INTO ft VALUES('S K H G W', 'R W Q F G');
+  }
+  faultsim_save_and_close
+} {}
+do_faultsim_test 4.1 -prep {
+  faultsim_restore_and_reopen
+  db eval {SELECT * FROM sqlite_master}
+} -body {
+  execsql { INSERT INTO ft(ft) VALUES('rebuild') }
+} -test {
+  faultsim_test_result {0 {}}
+}
+
 finish_test

test/fts3first.test

@@ -0,0 +1,163 @@
+# 2011 October 18
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/malloc_common.tcl
+
+ifcapable !fts3 {
+  finish_test
+  return
+}
+
+set testprefix fts3first
+
+proc lreverse {L} {
+  set res [list]
+  for {set ii [expr [llength $L]-1]} {$ii>=0} {incr ii -1} {
+    lappend res [lindex $L $ii]
+  }
+  set res
+}
+
+proc mit {blob} {
+  set scan(littleEndian) i*
+  set scan(bigEndian) I*
+  binary scan $blob $scan($::tcl_platform(byteOrder)) r
+  return $r
+}
+db func mit mit
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE x1 USING FTS4(a, b, c);
+  INSERT INTO x1(docid,a,b,c) VALUES(0, 'K H D S T', 'V M N Y K', 'S Z N Q S');
+  INSERT INTO x1(docid,a,b,c) VALUES(1, 'K N J L W', 'S Z W J Q', 'D U W S E');
+  INSERT INTO x1(docid,a,b,c) VALUES(2, 'B P M O I', 'R P H W S', 'R J L L E');
+  INSERT INTO x1(docid,a,b,c) VALUES(3, 'U R Q M L', 'M J K A V', 'Q W J T J');
+  INSERT INTO x1(docid,a,b,c) VALUES(4, 'N J C Y N', 'R U D X V', 'B O U A Q');
+  INSERT INTO x1(docid,a,b,c) VALUES(5, 'Q L X L U', 'I F N X S', 'U Q A N Y');
+  INSERT INTO x1(docid,a,b,c) VALUES(6, 'M R G U T', 'U V I Q P', 'X Y D L S');
+  INSERT INTO x1(docid,a,b,c) VALUES(7, 'D Y P O I', 'X J P K R', 'V O T H V');
+  INSERT INTO x1(docid,a,b,c) VALUES(8, 'R Y D L R', 'U U E S J', 'N W L M R');
+  INSERT INTO x1(docid,a,b,c) VALUES(9, 'Z P F N P', 'W A X D U', 'V A E Q A');
+  INSERT INTO x1(docid,a,b,c) VALUES(10, 'Q I A Q M', 'N D K H C', 'A H T Q Z');
+  INSERT INTO x1(docid,a,b,c) VALUES(11, 'T E R Q B', 'C I B C B', 'F Z U W R');
+  INSERT INTO x1(docid,a,b,c) VALUES(12, 'E S V U W', 'T P F W H', 'A M D J Q');
+  INSERT INTO x1(docid,a,b,c) VALUES(13, 'X S B X Y', 'U D N D P', 'X Z Y G F');
+  INSERT INTO x1(docid,a,b,c) VALUES(14, 'K H A B L', 'S R C C Z', 'D W E H J');
+  INSERT INTO x1(docid,a,b,c) VALUES(15, 'C E U C C', 'W F M N M', 'T Z U X T');
+  INSERT INTO x1(docid,a,b,c) VALUES(16, 'Q G C G H', 'H N N B H', 'B Q I H Y');
+  INSERT INTO x1(docid,a,b,c) VALUES(17, 'Q T S K B', 'W B D Y N', 'V J P E C');
+  INSERT INTO x1(docid,a,b,c) VALUES(18, 'A J M O Q', 'L G Y Y A', 'G N M R N');
+  INSERT INTO x1(docid,a,b,c) VALUES(19, 'T R Y P Y', 'N V Y B X', 'L Z T N T');
+
+  CREATE VIRTUAL TABLE x2 USING FTS4(a, b, c, order=DESC);
+  INSERT INTO x2(docid, a, b, c) SELECT docid, a, b, c FROM x1;
+}
+
+
+# Test queries.
+#
+foreach x {1 2} {
+  foreach {tn match res} {
+    1  "^K"              {0 1 14}
+    2  "^S"              {0 1 14}
+    3  "^W"              {9 15 17}
+    4  "^J"              {}
+    5  "^E"              {12}
+    6  "V ^-E"           {0 3 4 6 7 9 17 19}
+    7  "V -^E"           {0 3 4 6 7 9 17 19}
+    8  "^-E V"           {0 3 4 6 7 9 17 19}
+    9  "-^E V"           {0 3 4 6 7 9 17 19}
+    10 "V"               {0 3 4 6 7 9 12 17 19}
+
+    11 {"^K H"}          {0 14}
+    12 {"K H"}           {0 10 14}
+    13 {"K ^H"}          {}
+  } {
+    set rev [lreverse $res]
+    do_execsql_test 1.$x.$tn.1 {SELECT docid FROM x1 WHERE x1 MATCH $match} $res
+    do_execsql_test 1.$x.$tn.2 {SELECT docid FROM x2 WHERE x2 MATCH $match} $rev
+  }
+
+  do_execsql_test 1.$x.[expr $tn+1] { 
+    INSERT INTO x1(x1) VALUES('optimize');
+    INSERT INTO x2(x2) VALUES('optimize');
+  } {}
+}
+
+# Test the snippet() function.
+#
+foreach {tn match res} {
+  1  {^K}    {{[K] H D S T} {[K] N J L W} {[K] H A B L}}
+  2  {^X}    {{[X] Y D L S} {[X] J P K R} {[X] S B X Y}}
+  3  {^X Y}  {{[X] [Y] D L S} {D [Y] P O I...[X] J P K R} {[X] S B X [Y]}}
+} {
+  set rev [lreverse $res]
+
+  do_execsql_test 1.3.$tn.1 {
+    SELECT snippet(x1, '[', ']', '...') FROM x1 WHERE x1 MATCH $match
+  } $res
+
+  do_execsql_test 1.3.$tn.2 {
+    SELECT snippet(x2, '[', ']', '...') FROM x2 WHERE x2 MATCH $match
+  } $rev
+}
+
+# Test matchinfo().
+#
+foreach {tn match res} {
+  1  {^K}    {
+                {1 3 3 0 0 0 0 0 0}
+                {1 3 3 0 0 0 0 0 0}
+                {1 3 3 0 0 0 0 0 0}
+             }
+  2  {^X}    {
+                {0 1 1 0 1 1 1 2 2}
+                {0 1 1 1 1 1 0 2 2}
+                {1 1 1 0 1 1 1 2 2}
+             }
+  3  {^X Y}  {
+                {0 1 1 0 1 1 1 2 2 0 6 5 0 5 4 1 4 4} 
+                {0 1 1 1 1 1 0 2 2 1 6 5 0 5 4 0 4 4} 
+                {1 1 1 0 1 1 1 2 2 1 6 5 0 5 4 1 4 4}
+             }
+} {
+  set rev [lreverse $res]
+
+  do_execsql_test 1.3.$tn.1 {
+    SELECT mit(matchinfo(x1, 'x')) FROM x1 WHERE x1 MATCH $match
+  } $res
+  do_execsql_test 1.3.$tn.2 {
+    SELECT mit(matchinfo(x2, 'x')) FROM x2 WHERE x2 MATCH $match
+  } $rev
+}
+
+# Test that ^ is ignored for FTS3 tables.
+#
+do_execsql_test 2.1 {
+  CREATE VIRTUAL TABLE x3 USING fts3;
+  INSERT INTO x3 VALUES('A B C');
+  INSERT INTO x3 VALUES('B A C');
+
+  CREATE VIRTUAL TABLE x4 USING fts4;
+  INSERT INTO x4 VALUES('A B C');
+  INSERT INTO x4 VALUES('B A C');
+}
+
+do_execsql_test 2.2.1 {
+  SELECT * FROM x3 WHERE x3 MATCH '^A';
+} {{A B C} {B A C}}
+do_execsql_test 2.2.2 {
+  SELECT * FROM x4 WHERE x4 MATCH '^A';
+} {{A B C}}
+
+finish_test

test/fts3malloc.test

@@ -294,6 +294,7 @@ do_write_test fts3_malloc-5.1 ft_content {
 do_test fts3_malloc-5.2 {
   execsql { CREATE VIRTUAL TABLE ft8 USING fts3(x, tokenize porter) }
 } {}
+
 do_write_test fts3_malloc-5.3 ft_content {
   INSERT INTO ft8 VALUES('short alongertoken reallyquitealotlongerimeanit andthistokenisjustsolongthatonemightbeforgivenforimaginingthatitwasmerelyacontrivedexampleandnotarealtoken')
 }

test/fts3matchinfo.test

@@ -19,6 +19,7 @@ source $testdir/tester.tcl
 ifcapable !fts3 { finish_test ; return }
 
 set testprefix fts3matchinfo
+set sqlite_fts3_enable_parentheses 0
 
 proc mit {blob} {
   set scan(littleEndian) i*
@@ -57,6 +58,9 @@ do_catchsql_test 2.0 {
 do_catchsql_test 2.1 {
   CREATE VIRTUAL TABLE x2 USING fts4(mtchinfo=fts3);
 } {1 {unrecognized parameter: mtchinfo=fts3}}
+do_catchsql_test 2.2 {
+  CREATE VIRTUAL TABLE x2 USING fts4(matchinfo=fts5);
+} {1 {unrecognized matchinfo: fts5}}
 
 # Check that with fts3, the "=" character is permitted in column definitions.
 #
@@ -224,6 +228,18 @@ do_matchinfo_test 4.1.3 t4 {t4 MATCH 'a b'}     { s {{2 0} {0 2}} }
 do_matchinfo_test 4.1.4 t4 {t4 MATCH '"a b" c'} { s {{2 0} {0 2}} }
 do_matchinfo_test 4.1.5 t4 {t4 MATCH 'a "b c"'} { s {{2 0} {0 2}} }
 do_matchinfo_test 4.1.6 t4 {t4 MATCH 'd d'}     { s {{1 0} {0 1}} }
+do_matchinfo_test 4.1.7 t4 {t4 MATCH 'f OR abcd'} {
+  x { 
+    {0 1 1  1 1 1  0 0 0  0 0 0} 
+    {1 1 1  0 1 1  0 0 0  0 0 0}
+  }
+}
+do_matchinfo_test 4.1.8 t4 {t4 MATCH 'f -abcd'} {
+  x { 
+    {0 1 1  1 1 1}
+    {1 1 1  0 1 1}
+  }
+}
 
 do_execsql_test 4.2.0 {
   CREATE VIRTUAL TABLE t5 USING fts4;

test/fts3prefix.test

@@ -200,4 +200,14 @@ do_execsql_test 4.6 {
   SELECT * FROM t3 WHERE t3 MATCH 'one*'
 } {{one two three}}
 
+#-------------------------------------------------------------------------
+# Syntax tests.
+#
+do_catchsql_test 5.1 {
+  CREATE VIRTUAL TABLE t4 USING fts4(prefix="abc");
+} {1 {error parsing prefix parameter: abc}}
+do_catchsql_test 5.2 {
+  CREATE VIRTUAL TABLE t4 USING fts4(prefix="");
+} {0 {}}
+
 finish_test

test/fts3sort.test

@@ -138,6 +138,8 @@ foreach {tn param res} {
   3 "order=dec"             {1 {unrecognized order: dec}}
   4 "order=xxx, order=asc"  {1 {unrecognized order: xxx}}
   5 "order=desc, order=asc" {0 {}}
+  6 "order=xxxx, order=asc" {1 {unrecognized order: xxxx}}
+  7 "order=desk"            {1 {unrecognized order: desk}}
 } {
   execsql { DROP TABLE IF EXISTS t1 }
   do_catchsql_test 2.1.$tn "
@@ -157,6 +159,9 @@ do_execsql_test 2.2 {
 do_execsql_test 2.3 {
   SELECT docid FROM t2 WHERE t2 MATCH 'aa';
 } {3 1}
+do_execsql_test 2.4 {
+  SELECT docid FROM t2 WHERE t2 MATCH 'aa' ORDER BY content;
+} {1 3}
 
 #-------------------------------------------------------------------------
 # Test that ticket [56be976859] has been fixed.

test/fts4content.test

@@ -0,0 +1,478 @@
+# 2011 October 03
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this script is testing the content=xxx FTS4 option.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set ::testprefix fts4content
+
+# If SQLITE_ENABLE_FTS3 is defined, omit this file.
+ifcapable !fts3 {
+  finish_test
+  return
+}
+
+#-------------------------------------------------------------------------
+# Test organization:
+#   
+#   1.* - Warm-body tests.
+#
+#   2.* - Querying a content=xxx FTS table.
+#
+#   3.* - Writing to a content=xxx FTS table.
+#
+#   4.* - The "INSERT INTO fts(fts) VALUES('rebuild')" command.
+#
+#   5.* - Check that CREATE TABLE, DROP TABLE and ALTER TABLE correctly
+#         ignore any %_content table when used with the content=xxx option.
+#
+#   6.* - Test the effects of messing with the schema of table xxx after
+#         creating a content=xxx FTS index.
+#
+
+do_execsql_test 1.1.1 {
+  CREATE TABLE t1(a, b, c);
+  INSERT INTO t1 VALUES('w x', 'x y', 'y z');
+  CREATE VIRTUAL TABLE ft1 USING fts4(content=t1);
+}
+
+do_execsql_test 1.1.2 {
+  PRAGMA table_info(ft1);
+} {
+  0 a {} 0 {} 0 
+  1 b {} 0 {} 0 
+  2 c {} 0 {} 0
+}
+
+do_execsql_test 1.1.3 { SELECT *, rowid FROM ft1 } {{w x} {x y} {y z} 1}
+do_execsql_test 1.1.4 { SELECT a, c FROM ft1 WHERE rowid=1 } {{w x} {y z}}
+
+do_execsql_test 1.1.5 { INSERT INTO ft1(ft1) VALUES('rebuild') } {}
+do_execsql_test 1.1.6 { SELECT rowid FROM ft1 WHERE ft1 MATCH 'x' } {1}
+do_execsql_test 1.1.7 { SELECT rowid FROM ft1 WHERE ft1 MATCH 'a' } {}
+
+do_execsql_test 1.2.1 {
+  DROP TABLE ft1;
+  CREATE VIRTUAL TABLE ft1 USING fts4(content=t1, b);
+  PRAGMA table_info(ft1);
+} {
+  0 b {} 0 {} 0 
+}
+do_execsql_test 1.2.2 { 
+  SELECT *, rowid FROM ft1 
+} {{x y} 1}
+
+#-------------------------------------------------------------------------
+# The following block of tests - 2.* - test that a content=xxx FTS table
+# can be queried. Also tested are cases where rows identified in the FTS
+# are missing from the content table, and cases where the index is 
+# inconsistent with the content table.
+# 
+do_execsql_test 2.0 {
+  CREATE TABLE t2(x);
+  INSERT INTO t2 VALUES('O S W W F U C R Q I C N P Z Y Y E Y Y E');  -- 1
+  INSERT INTO t2 VALUES('Y X U V L B E H Y J C Y A I A P V F V K');  -- 2
+  INSERT INTO t2 VALUES('P W I N J H I I N I F B K D U Q B Z S F');  -- 3
+  INSERT INTO t2 VALUES('N R O R H J R H G M D I U U B O M P A U');  -- 4
+  INSERT INTO t2 VALUES('Y O V O G T P N G T N F I V B U M J M G');  -- 5
+  INSERT INTO t2 VALUES('J O B N K N E C H Z R K J O U G M K L S');  -- 6
+  INSERT INTO t2 VALUES('S Z S R I Q U A P W R X H K C Z U L S P');  -- 7
+  INSERT INTO t2 VALUES('J C H N R C K R V N M O F Z M Z A I H W');  -- 8
+  INSERT INTO t2 VALUES('O Y G I S J U U W O D Z F J K N R P R L');  -- 9
+  INSERT INTO t2 VALUES('B G L K U R U P V X Z I H V R W C Q A S');  -- 10
+  INSERT INTO t2 VALUES('T F T J F F Y V F W N X K Q A Y L X W G');  -- 11
+  INSERT INTO t2 VALUES('C J U H B Q X L C M M Y E G V F W V Z C');  -- 12
+  INSERT INTO t2 VALUES('B W L T F S G X D P H N G M R I O A X I');  -- 13
+  INSERT INTO t2 VALUES('N G Y O K Q K Z N M H U J E D H U W R K');  -- 14
+  INSERT INTO t2 VALUES('U D T R U Y F J D S J X E H Q G V A S Z');  -- 15
+  INSERT INTO t2 VALUES('M I W P J S H R J D Q I C G P C T P H R');  -- 16
+  INSERT INTO t2 VALUES('J M N I S L X Q C A B F C B Y D H V R J');  -- 17
+  INSERT INTO t2 VALUES('F V Z W J Q L P X Y E W B U Q N H X K T');  -- 18
+  INSERT INTO t2 VALUES('R F S R Y O F Q E I E G H C B H R X Y N');  -- 19
+  INSERT INTO t2 VALUES('U Q Q Q T E P D M F X P J G H X C Q D L');  -- 20
+}
+
+do_execsql_test 2.1 {
+  CREATE VIRTUAL TABLE ft2 USING fts4(content=t2);
+  INSERT INTO ft2(ft2) VALUES('rebuild');
+
+  -- Modify the backing table a bit: Row 17 is missing and the contents 
+  -- of row 20 do not match the FTS index contents. 
+  DELETE FROM t2 WHERE rowid = 17;
+  UPDATE t2 SET x = 'a b c d e f g h i j' WHERE rowid = 20;
+}
+
+foreach {tn match rowidlist} {
+  1   {S}        {1 3 6 7 9 10 13 15 16 17 19}
+  2   {"S R"}    {7 19}
+  3   {"N K N"}  {6}
+  4   {"Q Q"}    {20}
+  5   {"B Y D"}  {17}
+} {
+  do_execsql_test 2.2.1.$tn {
+    SELECT rowid FROM ft2 WHERE ft2 MATCH $match
+  } $rowidlist
+
+  do_execsql_test 2.2.2.$tn {
+    SELECT docid FROM ft2 WHERE ft2 MATCH $match
+  } $rowidlist
+}
+
+foreach {tn match result} {
+  1   {"N K N"}  {{J O B N K N E C H Z R K J O U G M K L S}}
+  2   {"Q Q"}    {{a b c d e f g h i j}}
+  3   {"B Y D"}  {{}}
+} {
+  do_execsql_test 2.3.$tn {
+    SELECT * FROM ft2 WHERE ft2 MATCH $match
+  } $result
+}
+
+foreach {tn match result} {
+  1   {"N K N"}  {{..O B [N] [K] [N] E..}}
+  2   {"B Y D"}  {{}}
+  3   {"Q Q"}    {{a [b] [c] [d] e f..}}
+} {
+  do_execsql_test 2.4.$tn {
+    SELECT snippet(ft2, '[', ']', '..', -1, 6) FROM ft2 WHERE ft2 MATCH $match
+  } $result
+}
+
+foreach {tn match result} {
+  1   {"N K N"}  {{0 0 6 1 0 1 8 1 0 2 10 1}}
+  2   {"B Y D"}  {{}}
+  3   {"Q Q"}    {{0 0 2 1 0 0 4 1 0 1 4 1 0 1 6 1}}
+  4   {"Q D L"}  {{}}
+} {
+  do_execsql_test 2.5.$tn {
+    SELECT offsets(ft2) FROM ft2 WHERE ft2 MATCH $match
+  } $result
+}
+
+#-------------------------------------------------------------------------
+# The following block of tests - 3.* - test that the FTS index can be
+# modified by writing to the table. But that this has no effect on the 
+# content table.
+# 
+
+do_execsql_test 3.1 {
+  CREATE TABLE t3(x, y);
+  CREATE VIRTUAL TABLE ft3 USING fts4(content=t3);
+}
+
+do_catchsql_test 3.1.1 {
+  INSERT INTO ft3 VALUES('a b c', 'd e f');
+} {1 {constraint failed}}
+do_execsql_test 3.1.2 {
+  INSERT INTO ft3(docid, x, y) VALUES(21, 'a b c', 'd e f');
+  SELECT rowid FROM ft3 WHERE ft3 MATCH '"a b c"';
+} {21}
+do_execsql_test 3.1.3 { SELECT * FROM t3 } {}
+
+# This DELETE does not work, since there is no row in [t3] to base the
+# DELETE on. So the SELECT on [ft3] still returns rowid 21.
+do_execsql_test 3.1.4 { 
+  DELETE FROM ft3;
+  SELECT rowid FROM ft3 WHERE ft3 MATCH '"a b c"';
+} {21}
+
+# If the row is added to [t3] before the DELETE on [ft3], it works.
+do_execsql_test 3.1.5 {
+  INSERT INTO t3(rowid, x, y) VALUES(21, 'a b c', 'd e f');
+  DELETE FROM ft3;
+  SELECT rowid FROM ft3 WHERE ft3 MATCH '"a b c"';
+} {}
+do_execsql_test 3.1.6 { SELECT rowid FROM t3 } {21}
+
+do_execsql_test 3.2.1 {
+  INSERT INTO ft3(rowid, x, y) VALUES(0, 'R T M S M', 'A F O K H');
+  INSERT INTO ft3(rowid, x, y) VALUES(1, 'C Z J O X', 'U S Q D K');
+  INSERT INTO ft3(rowid, x, y) VALUES(2, 'N G H P O', 'N O P O C');
+  INSERT INTO ft3(rowid, x, y) VALUES(3, 'V H S D R', 'K N G E C');
+  INSERT INTO ft3(rowid, x, y) VALUES(4, 'J T R V U', 'U X S L C');
+  INSERT INTO ft3(rowid, x, y) VALUES(5, 'N A Y N G', 'X D G P Y');
+  INSERT INTO ft3(rowid, x, y) VALUES(6, 'I Q I S P', 'D R O Q B');
+  INSERT INTO ft3(rowid, x, y) VALUES(7, 'T K T Z J', 'B W D G O');
+  INSERT INTO ft3(rowid, x, y) VALUES(8, 'Y K F X T', 'D F G V G');
+  INSERT INTO ft3(rowid, x, y) VALUES(9, 'E L E T L', 'P W N F Z');
+  INSERT INTO ft3(rowid, x, y) VALUES(10, 'O G J G X', 'G J F E P');
+  INSERT INTO ft3(rowid, x, y) VALUES(11, 'O L N N Z', 'K E Z F D');
+  INSERT INTO ft3(rowid, x, y) VALUES(12, 'R Z M R J', 'X G I M Z');
+  INSERT INTO ft3(rowid, x, y) VALUES(13, 'L X N N X', 'R R N S T');
+  INSERT INTO ft3(rowid, x, y) VALUES(14, 'F L B J H', 'K W F L C');
+  INSERT INTO ft3(rowid, x, y) VALUES(15, 'P E B M V', 'E A A B U');
+  INSERT INTO ft3(rowid, x, y) VALUES(16, 'V E C F P', 'L U T V K');
+  INSERT INTO ft3(rowid, x, y) VALUES(17, 'T N O Z N', 'T P Q X N');
+  INSERT INTO ft3(rowid, x, y) VALUES(18, 'V W U W R', 'H O A A V');
+  INSERT INTO ft3(rowid, x, y) VALUES(19, 'A H N L F', 'I G H B O');
+}
+
+foreach {tn match rowidlist} {
+  1   "N A"    {5 19}
+  2   "x:O"    {1 2 10 11 17}
+  3   "y:O"    {0 2 6 7 18 19}
+} {
+  set res [list]
+  foreach rowid $rowidlist { lappend res $rowid {} {} }
+
+  do_execsql_test 3.2.2.$tn {
+    SELECT rowid, * FROM ft3 WHERE ft3 MATCH $match
+  } $res
+  do_execsql_test 3.2.3.$tn {
+    SELECT docid, * FROM ft3 WHERE ft3 MATCH $match
+  } $res
+}
+
+do_execsql_test 3.3.1 {
+  INSERT INTO t3(rowid, x, y) VALUES(0, 'R T M S M', 'A F O K H');
+  INSERT INTO t3(rowid, x, y) VALUES(1, 'C Z J O X', 'U S Q D K');
+  INSERT INTO t3(rowid, x, y) VALUES(2, 'N G H P O', 'N O P O C');
+  INSERT INTO t3(rowid, x, y) VALUES(3, 'V H S D R', 'K N G E C');
+  INSERT INTO t3(rowid, x, y) VALUES(4, 'J T R V U', 'U X S L C');
+  INSERT INTO t3(rowid, x, y) VALUES(5, 'N A Y N G', 'X D G P Y');
+  UPDATE ft3 SET x = y, y = x;
+  DELETE FROM t3;
+}
+
+foreach {tn match rowidlist} {
+  1   "N A"    {5 19}
+  2   "x:O"    {0 2 10 11 17}
+  3   "y:O"    {1 2 6 7 18 19}
+} {
+  set res [list]
+  foreach rowid $rowidlist { lappend res $rowid {} {} }
+
+  do_execsql_test 3.3.2.$tn {
+    SELECT rowid, * FROM ft3 WHERE ft3 MATCH $match
+  } $res
+  do_execsql_test 3.3.3.$tn {
+    SELECT docid, * FROM ft3 WHERE ft3 MATCH $match
+  } $res
+}
+
+do_execsql_test 3.3.1 {
+  INSERT INTO t3(rowid, x, y) VALUES(15, 'P E B M V', 'E A A B U');
+  INSERT INTO t3(rowid, x, y) VALUES(16, 'V E C F P', 'L U T V K');
+  INSERT INTO t3(rowid, x, y) VALUES(17, 'T N O Z N', 'T P Q X N');
+  INSERT INTO t3(rowid, x, y) VALUES(18, 'V W U W R', 'H O A A V');
+  INSERT INTO t3(rowid, x, y) VALUES(19, 'A H N L F', 'I G H B O');
+  DELETE FROM ft3;
+}
+
+foreach {tn match rowidlist} {
+  1   "N A"    {5}
+  2   "x:O"    {0 2 10 11}
+  3   "y:O"    {1 2 6 7}
+} {
+  set res [list]
+  foreach rowid $rowidlist { lappend res $rowid {} {} }
+
+  do_execsql_test 3.3.2.$tn {
+    SELECT rowid, * FROM ft3 WHERE ft3 MATCH $match
+  } $res
+  do_execsql_test 3.3.3.$tn {
+    SELECT docid, * FROM ft3 WHERE ft3 MATCH $match
+  } $res
+}
+
+
+#-------------------------------------------------------------------------
+# Test cases 4.* test the 'rebuild' command. On content=xxx and regular
+# FTS tables.
+# 
+do_execsql_test 4.0 {
+  CREATE TABLE t4(x);
+  CREATE VIRTUAL TABLE ft4 USING fts4(content=t4);
+  CREATE VIRTUAL TABLE ft4x USING fts4(x);
+}
+
+do_execsql_test 4.1.1 {
+  INSERT INTO ft4x(ft4x) VALUES('rebuild');
+  INSERT INTO ft4(ft4) VALUES('rebuild');
+} {}
+do_execsql_test 4.1.2 {
+  SELECT id, quote(value) FROM ft4_stat
+} {0 X'000000'}
+do_execsql_test 4.1.3 {
+  SELECT id, quote(value) FROM ft4x_stat
+} {0 X'000000'}
+
+do_execsql_test 4.2.1 {
+  INSERT INTO ft4x VALUES('M G M F T');
+  INSERT INTO ft4x VALUES('Z Q C A U');
+  INSERT INTO ft4x VALUES('N L L V');
+  INSERT INTO ft4x VALUES('T F D X D');
+  INSERT INTO ft4x VALUES('Z H I S D');
+
+  SELECT id, quote(value) FROM ft4x_stat
+} {0 X'05182B'}
+
+do_execsql_test 4.2.2 {
+  INSERT INTO ft4(rowid, x) SELECT rowid, * FROM ft4x;
+  SELECT id, quote(value) FROM ft4_stat
+} {0 X'05182B'}
+
+do_execsql_test 4.2.3 {
+  SELECT docid, quote(size) FROM ft4_docsize
+} {1 X'05' 2 X'05' 3 X'04' 4 X'05' 5 X'05'}
+
+do_execsql_test 4.2.4 {
+  INSERT INTO ft4x(ft4x) VALUES('rebuild');
+  SELECT id, quote(value) FROM ft4x_stat;
+  SELECT docid, quote(size) FROM ft4x_docsize
+} {0 X'05182B' 1 X'05' 2 X'05' 3 X'04' 4 X'05' 5 X'05'}
+
+do_execsql_test 4.2.5 {
+  INSERT INTO ft4(ft4) VALUES('rebuild');
+  SELECT id, quote(value) FROM ft4_stat;
+  SELECT docid, quote(size) FROM ft4_docsize
+} {0 X'000000'}
+
+do_execsql_test 4.2.6 {
+  INSERT INTO t4(rowid, x) SELECT rowid, x FROM ft4x;
+  INSERT INTO ft4(ft4) VALUES('rebuild');
+  SELECT id, quote(value) FROM ft4_stat;
+  SELECT docid, quote(size) FROM ft4_docsize
+} {0 X'05182B' 1 X'05' 2 X'05' 3 X'04' 4 X'05' 5 X'05'}
+
+
+#-------------------------------------------------------------------------
+# Test cases 5.* test that the following commands do not create/move or
+# delete a %_content table when used with a content=xxx FTS table.
+# 
+do_execsql_test 5.1.1 {
+  CREATE TABLE t5(a, b, c, d);
+  CREATE VIRTUAL TABLE ft5 USING fts4(content=t5);
+  SELECT name FROM sqlite_master WHERE name LIKE '%t5%';
+} {
+  t5 ft5 ft5_segments ft5_segdir 
+  sqlite_autoindex_ft5_segdir_1 ft5_docsize ft5_stat
+}
+do_execsql_test 5.1.2 {
+  ALTER TABLE ft5 RENAME TO ft6;
+  SELECT name FROM sqlite_master WHERE name LIKE '%t5%';
+} {
+  t5
+}
+do_execsql_test 5.1.3 {
+  SELECT name FROM sqlite_master WHERE name LIKE '%t6%';
+} {
+  ft6 ft6_segments ft6_segdir 
+  sqlite_autoindex_ft6_segdir_1 ft6_docsize ft6_stat
+}
+do_execsql_test 5.1.4 {
+  INSERT INTO t5 VALUES('a', 'b', 'c', 'd');
+  INSERT INTO ft6(ft6) VALUES('rebuild');
+  SELECT rowid FROM ft6 WHERE ft6 MATCH 'b';
+} {1}
+do_execsql_test 5.1.5 {
+  DROP TABLE ft6;
+  SELECT * FROM t5;
+} {a b c d}
+do_execsql_test 5.1.6 {
+  SELECT name FROM sqlite_master WHERE name LIKE '%t6%';
+} {
+}
+do_execsql_test 5.1.7 {
+  CREATE VIRTUAL TABLE ft5 USING fts4(content=t5);
+  CREATE TABLE t5_content(a, b);
+  DROP TABLE ft5;
+  SELECT name FROM sqlite_master WHERE name LIKE '%t5%';
+} {
+  t5 t5_content
+}
+
+#-------------------------------------------------------------------------
+# Test cases 6.* test 
+# 
+do_catchsql_test 6.1.1 {
+  CREATE VIRTUAL TABLE ft7 USING fts4(content=t7);
+} {1 {vtable constructor failed: ft7}}
+
+do_execsql_test 6.2.1 {
+  CREATE TABLE t7(one, two);
+  CREATE VIRTUAL TABLE ft7 USING fts4(content=t7);
+  INSERT INTO t7 VALUES('A B', 'B A');
+  INSERT INTO t7 VALUES('C D', 'A A');
+  SELECT * FROM ft7;
+} {
+  {A B} {B A} {C D} {A A}
+}
+
+do_catchsql_test 6.2.2 {
+  DROP TABLE t7;
+  SELECT * FROM ft7;
+} {1 {SQL logic error or missing database}}
+
+db close
+sqlite3 db test.db
+do_execsql_test 6.2.3 {
+  SELECT name FROM sqlite_master WHERE name LIKE '%t7%'
+} {
+  ft7 ft7_segments ft7_segdir sqlite_autoindex_ft7_segdir_1 
+  ft7_docsize ft7_stat
+}
+do_catchsql_test 6.2.4 {
+  SELECT * FROM ft7;
+} {1 {vtable constructor failed: ft7}}
+do_execsql_test 6.2.5 {
+  CREATE TABLE t7(x, y);
+  INSERT INTO t7 VALUES('A B', 'B A');
+  INSERT INTO t7 VALUES('C D', 'A A');
+  SELECT * FROM ft7;
+} {
+  {A B} {B A} {C D} {A A}
+}
+
+do_execsql_test 6.2.6 {
+  INSERT INTO ft7(ft7) VALUES('rebuild');
+  SELECT rowid FROM ft7 WHERE ft7 MATCH '"A A"';
+} {2}
+
+do_execsql_test 6.2.7 {
+  DROP TABLE t7;
+  CREATE TABLE t7(x);
+}
+do_catchsql_test 6.2.8 {
+  SELECT * FROM ft7 WHERE ft7 MATCH '"A A"';
+} {1 {SQL logic error or missing database}}
+do_catchsql_test 6.2.9 {
+  SELECT * FROM ft7 WHERE ft7 MATCH '"A A"';
+} {1 {SQL logic error or missing database}}
+
+db close
+sqlite3 db test.db
+do_catchsql_test 6.2.10 {
+  SELECT rowid FROM ft7 WHERE ft7 MATCH '"A A"';
+} {0 2}
+do_catchsql_test 6.2.11 {
+  SELECT rowid, * FROM ft7 WHERE ft7 MATCH '"A A"';
+} {0 {2 {}}}
+
+#-------------------------------------------------------------------------
+# Test cases 7.*
+# 
+do_execsql_test 7.1.1 {
+  CREATE VIRTUAL TABLE ft8 USING fts4(content=nosuchtable, x);
+  INSERT INTO ft8(docid, x) VALUES(13, 'U O N X G');
+  INSERT INTO ft8(docid, x) VALUES(14, 'C J J U B');
+  INSERT INTO ft8(docid, x) VALUES(15, 'N J Y G X');
+  INSERT INTO ft8(docid, x) VALUES(16, 'R Y D O R');
+  INSERT INTO ft8(docid, x) VALUES(17, 'I Y T Q O');
+}
+
+do_execsql_test 7.1.2 {
+  SELECT docid FROM ft8 WHERE ft8 MATCH 'N';
+} {13 15}
+
+finish_test

test/permutations.test

@@ -180,10 +180,12 @@ test_suite "fts3" -prefix "" -description {
   fts3defer.test fts3defer2.test fts3e.test fts3expr.test fts3expr2.test 
   fts3near.test fts3query.test fts3shared.test fts3snippet.test 
   fts3sort.test
-
   fts3fault.test fts3malloc.test fts3matchinfo.test
-
   fts3aux1.test fts3comp1.test fts3auto.test
+  fts4aa.test fts4content.test
+  fts3conf.test fts3prefix.test fts3fault2.test fts3corrupt.test
+  fts3corrupt2.test
+  fts3first.test
 }