Further fixes and test cases related to external content tables.

FossilOrigin-Name: ce6a899baff7265a60c880098a9a57ea352b5415

ext/fts5/fts5_config.c

@@ -216,12 +216,13 @@ static char *fts5EscapeName(int *pRc, const char *z){
   char *pRet = 0;
   if( *pRc==SQLITE_OK ){
     int n = strlen(z);
-    pRet = (char*)sqlite3_malloc(2 * 2*n + 1);
+    pRet = (char*)sqlite3_malloc(2 + 2*n + 1);
     if( pRet==0 ){
       *pRc = SQLITE_NOMEM;
     }else{
       int i;
       char *p = pRet;
+      *p++ = '`';
       for(i=0; i<n; i++){
         if( z[i]=='`' ) *p++ = '`';
         *p++ = z[i];

ext/fts5/fts5_index.c

@@ -406,6 +406,7 @@ struct Fts5SegWriter {
 struct Fts5MultiSegIter {
   int nSeg;                       /* Size of aSeg[] array */
   int bRev;                       /* True to iterate in reverse order */
+  int bSkipEmpty;                 /* True to skip deleted entries */
   Fts5SegIter *aSeg;              /* Array of segment iterators */
   u16 *aFirst;                    /* Current merge state (see above) */
 };
@@ -1601,6 +1602,33 @@ static void fts5SegIterReverseNewPage(Fts5Index *p, Fts5SegIter *pIter){
   }
 }
 
+/*
+** Return true if the iterator passed as the second argument currently
+** points to a delete marker. A delete marker is an entry with a 0 byte
+** position-list.
+*/
+static int fts5SegIterIsDelete(
+  Fts5Index *p,                   /* FTS5 backend object */
+  Fts5SegIter *pIter              /* Iterator to advance */
+){
+  int bRet = 0;
+  Fts5Data *pLeaf = pIter->pLeaf;
+  if( p->rc==SQLITE_OK && pLeaf ){
+    if( pIter->iLeafOffset<pLeaf->n ){
+      bRet = (pLeaf->p[pIter->iLeafOffset]==0x00);
+    }else{
+      Fts5Data *pNew = fts5DataRead(p, FTS5_SEGMENT_ROWID(
+            pIter->iIdx, pIter->pSeg->iSegid, 0, pIter->iLeafPgno
+      ));
+      if( pNew ){
+        bRet = (pNew->p[4]==0x00);
+        fts5DataRelease(pNew);
+      }
+    }
+  }
+  return bRet;
+}
+
 /*
 ** Advance iterator pIter to the next entry. 
 **
@@ -2094,14 +2122,20 @@ static void fts5MultiIterNext(
   i64 iFrom                       /* Advance at least as far as this */
 ){
   if( p->rc==SQLITE_OK ){
+    int bUseFrom = bFrom;
+    do {
       int iFirst = pIter->aFirst[1];
       Fts5SegIter *pSeg = &pIter->aSeg[iFirst];
-    if( bFrom && pSeg->pDlidx ){
+      if( bUseFrom && pSeg->pDlidx ){
         fts5SegIterNextFrom(p, pSeg, iFrom);
       }else{
         fts5SegIterNext(p, pSeg);
       }
       fts5MultiIterAdvanced(p, pIter, iFirst, 1);
+      bUseFrom = 0;
+    }while( pIter->bSkipEmpty 
+         && fts5SegIterIsDelete(p, &pIter->aSeg[pIter->aFirst[1]])
+    );
   }
 }
 
@@ -2120,6 +2154,7 @@ static void fts5MultiIterNew(
   Fts5Index *p,                   /* FTS5 backend to iterate within */
   Fts5Structure *pStruct,         /* Structure of specific index */
   int iIdx,                       /* Config.aHash[] index of FTS index */
+  int bSkipEmpty,
   int flags,                      /* True for >= */
   const u8 *pTerm, int nTerm,     /* Term to seek to (or NULL/0) */
   int iLevel,                     /* Level to iterate (-1 for all) */
@@ -2152,6 +2187,7 @@ static void fts5MultiIterNew(
   pNew->aSeg = (Fts5SegIter*)&pNew[1];
   pNew->aFirst = (u16*)&pNew->aSeg[nSlot];
   pNew->bRev = (0!=(flags & FTS5INDEX_QUERY_ASC));
+  pNew->bSkipEmpty = bSkipEmpty;
 
   /* Initialize each of the component segment iterators. */
   if( iLevel<0 ){
@@ -2187,6 +2223,12 @@ static void fts5MultiIterNew(
         fts5MultiIterAdvanced(p, pNew, iEq, iIter);
       }
     }
+
+    if( pNew->bSkipEmpty 
+     && fts5SegIterIsDelete(p, &pNew->aSeg[pNew->aFirst[1]]) 
+    ){
+      fts5MultiIterNext(p, pNew, 0, 0);
+    }
   }else{
     fts5MultiIterFree(p, pNew);
     *ppOut = 0;
@@ -2958,7 +3000,7 @@ fprintf(stdout, "merging %d segments from level %d!", nInput, iLvl);
 fflush(stdout);
 #endif
 
-  for(fts5MultiIterNew(p, pStruct, iIdx, 0, 0, 0, iLvl, nInput, &pIter);
+  for(fts5MultiIterNew(p, pStruct, iIdx, 0, 0, 0, 0, iLvl, nInput, &pIter);
       fts5MultiIterEof(p, pIter)==0;
       fts5MultiIterNext(p, pIter, 0, 0)
   ){
@@ -3689,7 +3731,7 @@ static void fts5SetupPrefixIter(
     Fts5Buffer doclist;
 
     memset(&doclist, 0, sizeof(doclist));
-    for(fts5MultiIterNew(p, pStruct, 0, 1, pToken, nToken, -1, 0, &p1);
+    for(fts5MultiIterNew(p, pStruct, 0, 1, 1, pToken, nToken, -1, 0, &p1);
         fts5MultiIterEof(p, p1)==0;
         fts5MultiIterNext(p, p1, 0, 0)
     ){
@@ -3770,7 +3812,7 @@ int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){
   for(iIdx=0; iIdx<=pConfig->nPrefix; iIdx++){
     Fts5MultiSegIter *pIter;
     Fts5Structure *pStruct = fts5StructureRead(p, iIdx);
-    for(fts5MultiIterNew(p, pStruct, iIdx, 0, 0, 0, -1, 0, &pIter);
+    for(fts5MultiIterNew(p, pStruct, iIdx, 0, 0, 0, 0, -1, 0, &pIter);
         fts5MultiIterEof(p, pIter)==0;
         fts5MultiIterNext(p, pIter, 0, 0)
     ){
@@ -4031,7 +4073,7 @@ int sqlite3Fts5IndexQuery(
       pRet->pStruct = fts5StructureRead(p, iIdx);
       if( pRet->pStruct ){
         fts5MultiIterNew(p, pRet->pStruct, 
-            iIdx, flags, (const u8*)pToken, nToken, -1, 0, &pRet->pMulti
+            iIdx, 1, flags, (const u8*)pToken, nToken, -1, 0, &pRet->pMulti
         );
       }
     }else{

ext/fts5/fts5_storage.c

@@ -62,8 +62,8 @@ static int fts5StorageGetStmt(
   assert( eStmt>=0 && eStmt<ArraySize(p->aStmt) );
   if( p->aStmt[eStmt]==0 ){
     const char *azStmt[] = {
-      "SELECT * FROM %s ORDER BY id ASC",               /* SCAN_ASC */
+      "SELECT * FROM %s ORDER BY %s ASC",               /* SCAN_ASC */
-      "SELECT * FROM %s ORDER BY id DESC",              /* SCAN_DESC */
+      "SELECT * FROM %s ORDER BY %s DESC",              /* SCAN_DESC */
       "SELECT * FROM %s WHERE %s=?",                    /* LOOKUP  */
 
       "INSERT INTO %Q.'%q_content' VALUES(%s)",         /* INSERT_CONTENT  */
@@ -82,9 +82,6 @@ static int fts5StorageGetStmt(
     switch( eStmt ){
       case FTS5_STMT_SCAN_ASC:
       case FTS5_STMT_SCAN_DESC:
-        zSql = sqlite3_mprintf(azStmt[eStmt], pC->zContent);
-        break;
-
       case FTS5_STMT_LOOKUP:
         zSql = sqlite3_mprintf(azStmt[eStmt], pC->zContent, pC->zContentRowid);
         break;
@@ -725,7 +722,7 @@ int sqlite3Fts5StorageIntegrity(Fts5Storage *p){
 
   /* Check that the %_docsize and %_content tables contain the expected
   ** number of rows.  */
-  if( rc==SQLITE_OK ){
+  if( rc==SQLITE_OK && pConfig->eContent==FTS5_CONTENT_NORMAL ){
     i64 nRow;
     rc = fts5StorageCount(p, "content", &nRow);
     if( rc==SQLITE_OK && nRow!=p->nTotalRow ) rc = SQLITE_CORRUPT_VTAB;

ext/fts5/test/fts5aa.test

@@ -303,5 +303,26 @@ do_test 12.3 {
   string is integer $res
 } {1}
 
+#-------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 13.1 {
+  CREATE VIRTUAL TABLE t1 USING fts5(x);
+  INSERT INTO t1(rowid, x) VALUES(1, 'o n e'), (2, 't w o');
+} {}
+
+do_execsql_test 13.2 {
+  SELECT rowid FROM t1 WHERE t1 MATCH 'o';
+} {2 1}
+
+do_execsql_test 13.4 {
+  DELETE FROM t1 WHERE rowid=2;
+} {}
+
+do_execsql_test 13.5 {
+  SELECT rowid FROM t1 WHERE t1 MATCH 'o';
+} {1}
+
 finish_test
 
+

ext/fts5/test/fts5content.test

@@ -17,6 +17,9 @@ if {![info exists testdir]} {
 source $testdir/tester.tcl
 set testprefix fts5content
 
+#-------------------------------------------------------------------------
+# Contentless tables
+#
 do_execsql_test 1.1 {
   CREATE VIRTUAL TABLE f1 USING fts5(a, b, content='');
   INSERT INTO f1(rowid, a, b) VALUES(1, 'one',   'o n e');
@@ -83,17 +86,67 @@ do_execsql_test 1.15 {
   INSERT INTO f1(f1, rowid, a, b) VALUES('delete', 2, 'two', 't w o');
 } {}
 
-db eval { SELECT fts5_decode(id, block) AS d FROM f1_data } { puts $d }
-
-breakpoint
 do_execsql_test 1.16 {
   SELECT rowid FROM f1 WHERE f1 MATCH 'o';
 } {4 1}
+
 do_execsql_test 1.17 {
   SELECT rowid FROM f1;
 } {4 3 1}
 
+#-------------------------------------------------------------------------
+# External content tables
+#
+reset_db
+do_execsql_test 2.1 {
+  -- Create a table. And an external content fts5 table to index it.
+  CREATE TABLE tbl(a INTEGER PRIMARY KEY, b, c);
+  CREATE VIRTUAL TABLE fts_idx USING fts5(b, c, content='tbl', content_rowid='a');
 
+  -- Triggers to keep the FTS index up to date.
+  CREATE TRIGGER tbl_ai AFTER INSERT ON tbl BEGIN
+    INSERT INTO fts_idx(rowid, b, c) VALUES (new.a, new.b, new.c);
+  END;
+  CREATE TRIGGER tbl_ad AFTER DELETE ON tbl BEGIN
+    INSERT INTO fts_idx(fts_idx, rowid, b, c) 
+        VALUES('delete', old.a, old.b, old.c);
+  END;
+  CREATE TRIGGER tbl_au AFTER UPDATE ON tbl BEGIN
+    INSERT INTO fts_idx(fts_idx, rowid, b, c) 
+        VALUES('delete', old.a, old.b, old.c);
+    INSERT INTO fts_idx(rowid, b, c) VALUES (new.a, new.b, new.c);
+  END;
+}
+
+do_execsql_test 2.2 {
+  INSERT INTO tbl VALUES(1, 'one', 'o n e');
+  INSERT INTO tbl VALUES(NULL, 'two', 't w o');
+  INSERT INTO tbl VALUES(3, 'three', 't h r e e');
+}
+
+do_execsql_test 2.3 {
+  INSERT INTO fts_idx(fts_idx) VALUES('integrity-check');
+}
+
+do_execsql_test 2.4 {
+  DELETE FROM tbl WHERE rowid=2;
+  INSERT INTO fts_idx(fts_idx) VALUES('integrity-check');
+}
+
+do_execsql_test 2.5 {
+  UPDATE tbl SET c = c || ' x y z';
+  INSERT INTO fts_idx(fts_idx) VALUES('integrity-check');
+}
+
+do_execsql_test 2.6 {
+  SELECT * FROM fts_idx WHERE fts_idx MATCH 't AND x';
+} {three {t h r e e x y z}}
+
+do_execsql_test 2.7 {
+  SELECT highlight(fts_idx, 1, '[', ']') FROM fts_idx 
+  WHERE fts_idx MATCH 't AND x';
+} {{[t] h r e e [x] y z}}
 
 
 finish_test
+

manifest

@@ -1,5 +1,5 @@
-C Tests\sand\sfixes\sfor\sfts5\sexternal\scontent\stables.
+C Further\sfixes\sand\stest\scases\srelated\sto\sexternal\scontent\stables.
-D 2015-01-05T20:41:39.791
+D 2015-01-06T14:38:34.378
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 7cd23e4fc91004a6bd081623e1bc6932e44828c0
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -109,16 +109,16 @@ F ext/fts5/fts5.h 4f9d2c477c0ee1907164642471329a82cb6b203b
 F ext/fts5/fts5Int.h 9aafe97064e9c3380991abad4f51bee51021d18d
 F ext/fts5/fts5_aux.c a74523025a553f57c99c699b9e2d83c4506503b4
 F ext/fts5/fts5_buffer.c 1bc5c762bb2e9b4a40b2e8a820a31b809e72eec1
-F ext/fts5/fts5_config.c 630f92bb0a301c0b4e37a05ec4e38dc51ceeba37
+F ext/fts5/fts5_config.c ecd2f2efca1cda58525087a1a0e0bc1d34aad7a0
 F ext/fts5/fts5_expr.c 317093f00a2ccdaaee0a5290f9f228c600189c41
 F ext/fts5/fts5_hash.c 63fa8379c5f2ac107d47c2b7d9ac04c95ef8a279
-F ext/fts5/fts5_index.c 4a8e8535b4303400ddb5f6fb08152da0d88ebf6f
+F ext/fts5/fts5_index.c a0f370b7843183c040dbbf724e1080a615ee05cc
-F ext/fts5/fts5_storage.c 68ce8ec98b009cbd350ff73df06a97b1a012e122
+F ext/fts5/fts5_storage.c 9b6b8afde63ccc7e8f2f37252bf47a0ea00f468c
 F ext/fts5/fts5_tcl.c 664e710e2bbeed505cb91848772ca7538623a67f
 F ext/fts5/fts5_tokenize.c 5a0ad46408d09bcda2bf0addb5af42fdb75ebabb
 F ext/fts5/fts5_unicode2.c 9c7dd640d1f014bf5c3ee029759adfbb4d7e95a9
 F ext/fts5/fts5parse.y 777da8e5819f75c217982c79c29d014c293acac9
-F ext/fts5/test/fts5aa.test 01fff9cf4e75c33871dd121d6adae33b609542cf
+F ext/fts5/test/fts5aa.test 2affb47c0efa9cd39e1589ff8d8d78bcc7792952
 F ext/fts5/test/fts5ab.test 7a58a954cae2ae50cef3ee525c57bc8eb3eb50b3
 F ext/fts5/test/fts5ac.test d3de838f48d2ac8c26386832f6d93a3a3dbb5d4b
 F ext/fts5/test/fts5ad.test a8311d6ce46964fa1686937793dd81d284317324
@@ -131,7 +131,7 @@ F ext/fts5/test/fts5aj.test 1a64ab4144f54bd12a520683950bf8460dd74fb3
 F ext/fts5/test/fts5ak.test df2669fb76684f03d03918dfb2cf692012251b1f
 F ext/fts5/test/fts5al.test bc873766fec3baae05ba6e76b379bc2f5e8eaf75
 F ext/fts5/test/fts5auxdata.test fec4c9113176d351e567eab65fe9917e5ea0ab05
-F ext/fts5/test/fts5content.test 0f267ba2086f2dff81484c8ee71fa0d3990c41f7
+F ext/fts5/test/fts5content.test 55f760043ab3b066b9d91a9bf5f518198d31cc1f
 F ext/fts5/test/fts5ea.test 0ef2c89e14c6360ad3905fae44409420d6b5a5c8
 F ext/fts5/test/fts5fault1.test b95ed600b88bbbce5390f9097a5a5b7b01b3b9f7
 F ext/fts5/test/fts5porter.test d8f7591b733bcc1f02ca0dd313bc891a4b289562
@@ -1271,7 +1271,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 17ef5b59f789e9fa35c4f053246d819987fd06f8
+P 047aaf830d1e72f0fdad3832a0b617e769d66468
-R 6bfe2a49f6feaf1db299d5e29da25a24
+R 4c2c7726c7891be9cd96464f52b4b676
 U dan
-Z 3d2200ed8057fd64a39f743bdc333945
+Z b55a8c4b3246d78dc3224ac9cef3d20c

manifest.uuid

@@ -1 +1 @@
-047aaf830d1e72f0fdad3832a0b617e769d66468
+ce6a899baff7265a60c880098a9a57ea352b5415