Fix the xColumnSize() extension API.

FossilOrigin-Name: 19504c4108472d2ad1281221642b8bd06eb69f4e

ext/fts5/fts5.c

@@ -619,8 +619,9 @@ static int fts5ApiColumnCount(Fts5Context *pCtx){
 }
 
 static int fts5ApiColumnAvgSize(Fts5Context *pCtx, int iCol, int *pnToken){
-  assert( 0 );
+  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
-  return 0;
+  Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab);
+  return sqlite3Fts5StorageAvgsize(pTab->pStorage, iCol, pnToken);
 }
 
 static int fts5ApiTokenize(

ext/fts5/fts5Int.h

@@ -267,6 +267,9 @@ int sqlite3Fts5IndexInit(sqlite3*);
 
 void sqlite3Fts5IndexPgsz(Fts5Index *p, int pgsz);
 
+int sqlite3Fts5IndexGetAverages(Fts5Index *p, Fts5Buffer *pBuf);
+int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8*, int);
+
 /*
 ** End of interface to code in fts5_index.c.
 **************************************************************************/
@@ -297,7 +300,7 @@ int sqlite3Fts5StorageStmt(Fts5Storage *p, int eStmt, sqlite3_stmt **);
 void sqlite3Fts5StorageStmtRelease(Fts5Storage *p, int eStmt, sqlite3_stmt*);
 
 int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol);
-int sqlite3Fts5StorageAvgsize(Fts5Storage *p, int *aCol);
+int sqlite3Fts5StorageAvgsize(Fts5Storage *p, int iCol, int *pnAvg);
 
 
 /*

ext/fts5/fts5_aux.c

@@ -43,11 +43,24 @@ static void fts5TestFunction(
   }
 
   memset(&s, 0, sizeof(Fts5Buffer));
+  nCol = pApi->xColumnCount(pFts);
+
+  if( zReq==0 ){
+    sqlite3Fts5BufferAppendPrintf(&rc, &s, "columnavgsize ");
+  }
+  if( 0==zReq || 0==sqlite3_stricmp(zReq, "columnavgsize") ){
+    if( zReq==0 && nCol>1 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, "{");
+    for(i=0; rc==SQLITE_OK && i<nCol; i++){
+      int colsz = 0;
+      rc = pApi->xColumnAvgSize(pFts, i, &colsz);
+      sqlite3Fts5BufferAppendPrintf(&rc, &s, "%s%d", i==0?"":" ", colsz);
+    }
+    if( zReq==0 && nCol>1 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, "}");
+  }
 
   if( zReq==0 ){
     sqlite3Fts5BufferAppendPrintf(&rc, &s, "columncount ");
   }
-  nCol = pApi->xColumnCount(pFts);
   if( 0==zReq || 0==sqlite3_stricmp(zReq, "columncount") ){
     sqlite3Fts5BufferAppendPrintf(&rc, &s, "%d", nCol);
   }

ext/fts5/fts5_index.c

@@ -734,7 +734,7 @@ static void fts5DataReference(Fts5Data *pData){
 /*
 ** INSERT OR REPLACE a record into the %_data table.
 */
-static void fts5DataWrite(Fts5Index *p, i64 iRowid, u8 *pData, int nData){
+static void fts5DataWrite(Fts5Index *p, i64 iRowid, const u8 *pData, int nData){
   if( p->rc!=SQLITE_OK ) return;
 
   if( p->pWriter==0 ){
@@ -2734,6 +2734,7 @@ int sqlite3Fts5IndexOpen(
       }
       rc = p->rc;
     }
+    sqlite3Fts5IndexSetAverages(p, (const u8*)"", 0);
   }
 
   if( rc ){
@@ -3619,3 +3620,22 @@ void sqlite3Fts5IterClose(Fts5IndexIter *pIter){
   }
 }
 
+/*
+** Read the "averages" record into the buffer supplied as the second 
+** argument. Return SQLITE_OK if successful, or an SQLite error code
+** if an error occurs.
+*/
+int sqlite3Fts5IndexGetAverages(Fts5Index *p, Fts5Buffer *pBuf){
+  fts5DataReadOrBuffer(p, pBuf, FTS5_AVERAGES_ROWID);
+  return p->rc;
+}
+
+/*
+** Replace the current "averages" record with the contents of the buffer 
+** supplied as the second argument.
+*/
+int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8 *pData, int nData){
+  fts5DataWrite(p, FTS5_AVERAGES_ROWID, pData, nData);
+  return p->rc;
+}
+

ext/fts5/fts5_storage.c

@@ -17,6 +17,8 @@
 struct Fts5Storage {
   Fts5Config *pConfig;
   Fts5Index *pIndex;
+  i64 nTotalRow;                  /* Total number of rows in FTS table */
+  i64 *aTotalSize;                /* Total sizes of each column */ 
   sqlite3_stmt *aStmt[9];
 };
 
@@ -168,11 +170,15 @@ int sqlite3Fts5StorageOpen(
 ){
   int rc;
   Fts5Storage *p;                 /* New object */
+  int nByte;                      /* Bytes of space to allocate */
 
-  *pp = p = (Fts5Storage*)sqlite3_malloc(sizeof(Fts5Storage));
+  nByte = sizeof(Fts5Storage)               /* Fts5Storage object */
+        + pConfig->nCol * sizeof(i64);      /* Fts5Storage.aTotalSize[] */
+  *pp = p = (Fts5Storage*)sqlite3_malloc(nByte);
   if( !p ) return SQLITE_NOMEM;
 
-  memset(p, 0, sizeof(Fts5Storage));
+  memset(p, 0, nByte);
+  p->aTotalSize = (i64*)&p[1];
   p->pConfig = pConfig;
   p->pIndex = pIndex;
 
@@ -285,7 +291,9 @@ static int fts5StorageDeleteFromIndex(Fts5Storage *p, i64 iDel){
             (void*)&ctx,
             fts5StorageInsertCallback
         );
+        p->aTotalSize[iCol-1] -= (i64)ctx.szCol;
       }
+      p->nTotalRow--;
     }
     rc2 = sqlite3_reset(pSeek);
     if( rc==SQLITE_OK ) rc = rc2;
@@ -315,6 +323,62 @@ static int fts5StorageInsertDocsize(
   return rc;
 }
 
+/*
+** Load the contents of the "averages" record from disk into the
+** p->nTotalRow and p->aTotalSize[] variables.
+**
+** Return SQLITE_OK if successful, or an SQLite error code if an error
+** occurs.
+*/
+static int fts5StorageLoadTotals(Fts5Storage *p){
+  int nCol = p->pConfig->nCol;
+  Fts5Buffer buf;
+  int rc;
+  memset(&buf, 0, sizeof(buf));
+
+  memset(p->aTotalSize, 0, sizeof(i64) * nCol);
+  p->nTotalRow = 0;
+  rc = sqlite3Fts5IndexGetAverages(p->pIndex, &buf);
+  if( rc==SQLITE_OK && buf.n ){
+    int i = 0;
+    int iCol;
+    i += getVarint(&buf.p[i], (u64*)&p->nTotalRow);
+    for(iCol=0; i<buf.n && iCol<nCol; iCol++){
+      i += getVarint(&buf.p[i], (u64*)&p->aTotalSize[iCol]);
+    }
+  }
+  sqlite3_free(buf.p);
+
+  return rc;
+}
+
+/*
+** Store the current contents of the p->nTotalRow and p->aTotalSize[] 
+** variables in the "averages" record on disk.
+**
+** Return SQLITE_OK if successful, or an SQLite error code if an error
+** occurs.
+*/
+static int fts5StorageSaveTotals(Fts5Storage *p){
+  int nCol = p->pConfig->nCol;
+  int i;
+  Fts5Buffer buf;
+  int rc = SQLITE_OK;
+  memset(&buf, 0, sizeof(buf));
+
+  sqlite3Fts5BufferAppendVarint(&rc, &buf, p->nTotalRow);
+  for(i=0; i<nCol; i++){
+    sqlite3Fts5BufferAppendVarint(&rc, &buf, p->aTotalSize[i]);
+  }
+  if( rc==SQLITE_OK ){
+    rc = sqlite3Fts5IndexSetAverages(p->pIndex, buf.p, buf.n);
+  }
+  sqlite3_free(buf.p);
+
+  return rc;
+}
+
+
 /*
 ** Insert a new row into the FTS table.
 */
@@ -333,15 +397,18 @@ int sqlite3Fts5StorageInsert(
   Fts5Buffer buf;                 /* Buffer used to build up %_docsize blob */
 
   memset(&buf, 0, sizeof(Fts5Buffer));
+  rc = fts5StorageLoadTotals(p);
 
   /* Insert the new row into the %_content table. */
-  if( eConflict==SQLITE_REPLACE ){
+  if( rc==SQLITE_OK ){
-    eStmt = FTS5_STMT_REPLACE_CONTENT;
+    if( eConflict==SQLITE_REPLACE ){
-    if( sqlite3_value_type(apVal[1])==SQLITE_INTEGER ){
+      eStmt = FTS5_STMT_REPLACE_CONTENT;
-      rc = fts5StorageDeleteFromIndex(p, sqlite3_value_int64(apVal[1]));
+      if( sqlite3_value_type(apVal[1])==SQLITE_INTEGER ){
+        rc = fts5StorageDeleteFromIndex(p, sqlite3_value_int64(apVal[1]));
+      }
+    }else{
+      eStmt = FTS5_STMT_INSERT_CONTENT;
     }
-  }else{
-    eStmt = FTS5_STMT_INSERT_CONTENT;
   }
   if( rc==SQLITE_OK ){
     rc = fts5StorageGetStmt(p, eStmt, &pInsert);
@@ -367,7 +434,9 @@ int sqlite3Fts5StorageInsert(
         fts5StorageInsertCallback
     );
     sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol);
+    p->aTotalSize[ctx.iCol] += (i64)ctx.szCol;
   }
+  p->nTotalRow++;
 
   /* Write the %_docsize record */
   if( rc==SQLITE_OK ){
@@ -375,6 +444,11 @@ int sqlite3Fts5StorageInsert(
   }
   sqlite3_free(buf.p);
 
+  /* Write the averages record */
+  if( rc==SQLITE_OK ){
+    rc = fts5StorageSaveTotals(p);
+  }
+
   return rc;
 }
 
@@ -538,7 +612,16 @@ int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol){
   return rc;
 }
 
-int sqlite3Fts5StorageAvgsize(Fts5Storage *p, int *aCol){
+int sqlite3Fts5StorageAvgsize(Fts5Storage *p, int iCol, int *pnAvg){
-  return 0;
+  int rc = fts5StorageLoadTotals(p);
+  if( rc==SQLITE_OK ){
+    int nAvg = 1;
+    if( p->nTotalRow ){
+      nAvg = (int)((p->aTotalSize[iCol] + (p->nTotalRow/2)) / p->nTotalRow);
+      if( nAvg<1 ) nAvg = 1;
+      *pnAvg = nAvg;
+    }
+  }
+  return rc;
 }
 

manifest

@@ -1,5 +1,5 @@
-C Add\ssimple\stests\sfor\sthe\sxColumnText()\sextension\sapi.
+C Fix\sthe\sxColumnSize()\sextension\sAPI.
-D 2014-07-19T20:27:54.153
+D 2014-07-21T11:44:47.659
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in b03432313a3aad96c706f8164fb9f5307eaf19f5
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -103,15 +103,15 @@ F ext/fts3/tool/fts3view.c 6cfc5b67a5f0e09c0d698f9fd012c784bfaa9197
 F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c
 F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7
 F ext/fts3/unicode/mkunicode.tcl dc6f268eb526710e2c6e496c372471d773d0c368
-F ext/fts5/fts5.c fbd94670336eb95a26c705c1e4c188818720c888
+F ext/fts5/fts5.c 5bf93402f9bafa55181dfa70c3a785a41af31025
 F ext/fts5/fts5.h 84060c2fe91aa03a783cc993f313d8a5b22cbe11
-F ext/fts5/fts5Int.h 5105506a180942811aa7104867518bc84d36c17a
+F ext/fts5/fts5Int.h 12d03496152b716e63a5380e396b776fbefa2065
-F ext/fts5/fts5_aux.c 75f9abf7a7ccc7712357f04eeb6297c64095528d
+F ext/fts5/fts5_aux.c 9f0487715cd9933f2268620b41fb47f78a389297
 F ext/fts5/fts5_buffer.c 00361d4a70040ebd2c32bc349ab708ff613a1749
 F ext/fts5/fts5_config.c 94f1b4cb4de6a7cd5780c14adb0198e289df8cef
 F ext/fts5/fts5_expr.c 288b3e016253eab69ea8cefbff346a4697b44291
-F ext/fts5/fts5_index.c 9ff3008e903aa9077b0a7a7aa76ab6080eb07a36
+F ext/fts5/fts5_index.c 68d2d41b5c6d2f8838c3d6ebdc8b242718b8e997
-F ext/fts5/fts5_storage.c fcf66173e55927cee0675ecfb1038d0000e4fa10
+F ext/fts5/fts5_storage.c f722b080b9794f9e49cc4d36f0d9fb516cb7f309
 F ext/fts5/fts5parse.y 777da8e5819f75c217982c79c29d014c293acac9
 F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43
 F ext/icu/icu.c d415ccf984defeb9df2c0e1afcfaa2f6dc05eacb
@@ -595,11 +595,11 @@ F test/fts4merge3.test aab02a09f50fe6baaddc2e159c3eabc116d45fc7
 F test/fts4merge4.test d895b1057a7798b67e03455d0fa50e9ea836c47b
 F test/fts4noti.test 524807f0c36d49deea7920cdd4cd687408b58849
 F test/fts4unicode.test 01ec3fe2a7c3cfff3b4c0581b83caa11b33efa36
-F test/fts5aa.test c8d3b9694f6b2864161c7437408464a535d19343
+F test/fts5aa.test 0f5d29bf0a86b9dff0906c9e166d624c591d3437
 F test/fts5ab.test dc04ed48cf93ca957d174406e6c192f2ff4f3397
 F test/fts5ac.test 9be418d037763f4cc5d86f4239db41fc86bb4f85
 F test/fts5ad.test 2ed38bbc865678cb2905247120d02ebba7f20e07
-F test/fts5ae.test c95b106236ea2f9f151715311ad0a2e45c49ccc1
+F test/fts5ae.test b856782549ae0c56628e0980444fc38cf20b41a0
 F test/fts5ea.test ff43b40f8879ba50b82def70f2ab67c195d1a1d4
 F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d
 F test/func.test ae97561957aba6ca9e3a7b8a13aac41830d701ef
@@ -1195,7 +1195,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 43fcb844726cfeeb1c8a0dbfaa0d2ca22e6ac16c
+P 1e9053abdaf5e128d44504ee00dfd909dc25f378
-R fe613ccf4d230813074cd2e3b5b79d5f
+R 962590916dedf53a395813418f70a86f
 U dan
-Z 057df6fbedf27e3f40e741066d4f0911
+Z 393e4d18f784466676fdeef71a737c83

manifest.uuid

@@ -1 +1 @@
-1e9053abdaf5e128d44504ee00dfd909dc25f378
+19504c4108472d2ad1281221642b8bd06eb69f4e

test/fts5aa.test

@@ -129,6 +129,7 @@ foreach {i x y} {
 
 #-------------------------------------------------------------------------
 #
+breakpoint
 reset_db
 do_execsql_test 6.0 {
   CREATE VIRTUAL TABLE t1 USING fts5(x,y);

test/fts5ae.test

@@ -128,7 +128,7 @@ do_execsql_test 4.1 {
 }
 
 #-------------------------------------------------------------------------
-# Test that the xColumnSize() API works.
+# Test that the xColumnSize() and xColumnAvgsize() APIs work.
 #
 
 reset_db
@@ -156,6 +156,26 @@ do_execsql_test 5.2 {
   1 {{a b c d} {e f g h i j}}
 }
 
+do_execsql_test 5.3 {
+  SELECT rowid, fts5_test(t5, 'columnavgsize') FROM t5 WHERE t5 MATCH 'a'
+  ORDER BY rowid DESC;
+} {
+  3 {2 2}
+  2 {2 2}
+  1 {2 2}
+}
+
+do_execsql_test 5.4 {
+  INSERT INTO t5 VALUES('x y z', 'v w x y z');
+  SELECT rowid, fts5_test(t5, 'columnavgsize') FROM t5 WHERE t5 MATCH 'a'
+  ORDER BY rowid DESC;
+} {
+  3 {2 3}
+  2 {2 3}
+  1 {2 3}
+}
+
+
 
 finish_test