Change the fts5 content= option so that it matches fts5 columns with the underlying table columns by name, not by their position within the CREATE TABLE statement.

FossilOrigin-Name: e38e2bb637844dae8ae5d5f3e23d8369e1b91e45

ext/fts5/fts5Int.h

@@ -85,6 +85,16 @@ typedef struct Fts5Config Fts5Config;
 **   The minimum number of segments that an auto-merge operation should
 **   attempt to merge together. A value of 1 sets the object to use the 
 **   compile time default. Zero disables auto-merge altogether.
+**
+** zContent:
+**
+** zContentRowid:
+**   The value of the content_rowid= option, if one was specified. Or 
+**   the string "rowid" otherwise. This text is not quoted - if it is
+**   used as part of an SQL statement it needs to be quoted appropriately.
+**
+** zContentExprlist:
+**
 */
 struct Fts5Config {
   sqlite3 *db;                    /* Database handle */
@@ -98,6 +108,7 @@ struct Fts5Config {
   int eContent;                   /* An FTS5_CONTENT value */
   char *zContent;                 /* content table */ 
   char *zContentRowid;            /* "content_rowid=" option value */ 
+  char *zContentExprlist;
   Fts5Tokenizer *pTok;
   fts5_tokenizer *pTokApi;
 

ext/fts5/fts5_config.c

@@ -370,7 +370,7 @@ static int fts5ConfigParseSpecial(
       *pzErr = sqlite3_mprintf("multiple content_rowid=... directives");
       rc = SQLITE_ERROR;
     }else{
-      pConfig->zContentRowid = fts5EscapeName(&rc, zArg);
+      pConfig->zContentRowid = fts5Strdup(&rc, zArg);
     }
     return rc;
   }
@@ -469,6 +469,31 @@ static int fts5ConfigParseColumn(
   return rc;
 }
 
+/*
+** Populate the Fts5Config.zContentExprlist string.
+*/
+static int fts5ConfigMakeExprlist(Fts5Config *p){
+  int i;
+  int rc = SQLITE_OK;
+  Fts5Buffer buf = {0, 0, 0};
+  const char *zSep = "";
+
+  sqlite3Fts5BufferAppendPrintf(&rc, &buf, "T.%Q", p->zContentRowid);
+  if( p->eContent!=FTS5_CONTENT_NONE ){
+    for(i=0; i<p->nCol; i++){
+      if( p->eContent==FTS5_CONTENT_EXTERNAL ){
+        sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", T.%Q", p->azCol[i]);
+      }else{
+        sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", T.c%d", i);
+      }
+    }
+  }
+
+  assert( p->zContentExprlist==0 );
+  p->zContentExprlist = (char*)buf.p;
+  return rc;
+}
+
 /*
 ** Arguments nArg/azArg contain the string arguments passed to the xCreate
 ** or xConnect method of the virtual table. This function attempts to 
@@ -571,6 +596,11 @@ int sqlite3Fts5ConfigParse(
     pRet->zContentRowid = fts5Strdup(&rc, "rowid");
   }
 
+  /* Formulate the zContentExprlist text */
+  if( rc==SQLITE_OK ){
+    rc = fts5ConfigMakeExprlist(pRet);
+  }
+
   if( rc!=SQLITE_OK ){
     sqlite3Fts5ConfigFree(pRet);
     *ppOut = 0;
@@ -598,6 +628,7 @@ void sqlite3Fts5ConfigFree(Fts5Config *pConfig){
     sqlite3_free(pConfig->zRankArgs);
     sqlite3_free(pConfig->zContent);
     sqlite3_free(pConfig->zContentRowid);
+    sqlite3_free(pConfig->zContentExprlist);
     sqlite3_free(pConfig);
   }
 }

ext/fts5/fts5_index.c

@@ -4729,7 +4729,7 @@ static void fts5IndexIntegrityCheckSegment(
 ){
   Fts5BtreeIter iter;             /* Used to iterate through b-tree hierarchy */
 
-  if( pSeg->pgnoFirst==0 && pSeg->pgnoLast==0 ) return;
+  if( pSeg->pgnoFirst==0 ) return;
 
   /* Iterate through the b-tree hierarchy.  */
   for(fts5BtreeIterInit(p, iIdx, pSeg, &iter);
@@ -5148,10 +5148,8 @@ static int fts5DecodeDoclist(int *pRc, Fts5Buffer *pBuf, const u8 *a, int n){
   i64 iDocid;
   int iOff = 0;
 
-  if( iOff<n ){
-    iOff += sqlite3GetVarint(&a[iOff], (u64*)&iDocid);
-    sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " rowid=%lld", iDocid);
-  }
+  iOff = sqlite3GetVarint(&a[iOff], (u64*)&iDocid);
+  sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " rowid=%lld", iDocid);
   while( iOff<n ){
     int nPos;
     int bDummy;

ext/fts5/fts5_storage.c

@@ -65,9 +65,9 @@ static int fts5StorageGetStmt(
   assert( eStmt>=0 && eStmt<ArraySize(p->aStmt) );
   if( p->aStmt[eStmt]==0 ){
     const char *azStmt[] = {
-      "SELECT * FROM %s ORDER BY %s ASC",               /* SCAN_ASC */
-      "SELECT * FROM %s ORDER BY %s DESC",              /* SCAN_DESC */
-      "SELECT * FROM %s WHERE %s=?",                    /* LOOKUP  */
+      "SELECT %s FROM %s T ORDER BY T.%Q ASC",          /* SCAN_ASC */
+      "SELECT %s FROM %s T ORDER BY T.%Q DESC",         /* SCAN_DESC */
+      "SELECT %s FROM %s T WHERE    T.%Q=?",            /* LOOKUP  */
 
       "INSERT INTO %Q.'%q_content' VALUES(%s)",         /* INSERT_CONTENT  */
       "REPLACE INTO %Q.'%q_content' VALUES(%s)",        /* REPLACE_CONTENT */
@@ -86,7 +86,9 @@ static int fts5StorageGetStmt(
       case FTS5_STMT_SCAN_ASC:
       case FTS5_STMT_SCAN_DESC:
       case FTS5_STMT_LOOKUP:
-        zSql = sqlite3_mprintf(azStmt[eStmt], pC->zContent, pC->zContentRowid);
+        zSql = sqlite3_mprintf(azStmt[eStmt], 
+            pC->zContentExprlist, pC->zContent, pC->zContentRowid
+        );
         break;
 
       case FTS5_STMT_INSERT_CONTENT: 

ext/fts5/test/fts5content.test

@@ -9,6 +9,7 @@
 #
 #***********************************************************************
 #
+# This file contains tests for the content= and content_rowid= options.
 #
 
 source [file join [file dirname [info script]] fts5_common.tcl]
@@ -186,5 +187,35 @@ do_catchsql_test 3.8 {
   INSERT INTO t4(t4) VALUES('delete-all');
 } {1 {'delete-all' may only be used with a contentless or external content fts5 table}}
 
+#-------------------------------------------------------------------------
+# Test an external content table with a more interesting schema.
+#
+do_execsql_test 4.1 {
+  CREATE TABLE x2(a, "key col" PRIMARY KEY, b, c) WITHOUT ROWID;
+  INSERT INTO x2 VALUES('a b',   1, 'c d' , 'e f');
+  INSERT INTO x2 VALUES('x y', -40, 'z z' , 'y x');
+
+  CREATE VIRTUAL TABLE t2 USING fts5(a, c, content=x2, content_rowid='key col');
+  INSERT INTO t2(t2) VALUES('rebuild');
+}
+
+do_execsql_test 4.2 { SELECT rowid FROM t2 } {-40 1}
+do_execsql_test 4.3 { SELECT rowid FROM t2 WHERE t2 MATCH 'c'} {}
+do_execsql_test 4.4 { SELECT rowid FROM t2 WHERE t2 MATCH 'a'} {1}
+do_execsql_test 4.5 { SELECT rowid FROM t2 WHERE t2 MATCH 'x'} {-40}
+
+do_execsql_test 4.6 { INSERT INTO t2(t2) VALUES('integrity-check') } {}
+
+do_execsql_test 4.7 { 
+  DELETE FROM x2 WHERE "key col" = 1;
+  INSERT INTO t2(t2, rowid, a, c) VALUES('delete', 1, 'a b', 'e f');
+  INSERT INTO t2(t2) VALUES('integrity-check');
+}
+
+do_execsql_test 4.8 { SELECT rowid FROM t2 WHERE t2 MATCH 'b'} {}
+do_execsql_test 4.9 { SELECT rowid FROM t2 WHERE t2 MATCH 'y'} {-40}
+
+
 finish_test
 
+

ext/fts5/test/fts5fault2.test

@@ -98,5 +98,24 @@ do_faultsim_test 3.1 -faults oom-trans* -prep {
   faultsim_test_result {0 {}}
 }
 
+#-------------------------------------------------------------------------
+# OOM within an 'integrity-check' operation.
+#
+reset_db 
+db func rnddoc fts5_rnddoc
+do_execsql_test 4.0 {
+  CREATE VIRTUAL TABLE zzz USING fts5(z);
+  INSERT INTO zzz(zzz, rank) VALUES('pgsz', 32);
+  WITH ii(i) AS (SELECT 1 UNION SELECT i+1 FROM ii WHERE i<10)
+  INSERT INTO zzz SELECT rnddoc(10) || ' xccc' FROM ii;
+}
+
+do_faultsim_test 4.1 -faults oom-trans* -prep {
+} -body {
+  execsql { INSERT INTO zzz(zzz) VALUES('integrity-check') }
+} -test {
+  faultsim_test_result {0 {}}
+}
+
 finish_test
 

ext/fts5/tool/showfts5.tcl

@@ -0,0 +1,31 @@
+
+
+proc usage {} {
+  puts stderr "usage: $::argv0 database table"
+  puts stderr ""
+  exit 1
+}
+
+set o(vtab)       fts5
+set o(tok)        ""
+set o(limit)      0
+set o(automerge)  -1
+set o(crisismerge)  -1
+
+if {[llength $argv]!=2} usage
+
+set database [lindex $argv 0]
+set tbl [lindex $argv 1]
+
+sqlite3 db $database
+
+db eval "SELECT fts5_decode(rowid, block) AS d FROM ${tbl}_data WHERE id=10" {
+  foreach lvl [lrange $d 1 end] {
+    puts $lvl
+  }
+}
+
+
+
+
+