Add the 'hashsize' configuration option to fts5, for configuring the amount of memory allocated to the in-memory hash table while writing.

FossilOrigin-Name: 445480095e6877cce8220b1c095f334bbb04c1c3

ext/fts5/fts5Int.h

@@ -160,6 +160,7 @@ struct Fts5Config {
   int pgsz;                       /* Approximate page size used in %_data */
   int nAutomerge;                 /* 'automerge' setting */
   int nCrisisMerge;               /* Maximum allowed segments per level */
+  int nHashSize;                  /* Bytes of memory for in-memory hash */
   char *zRank;                    /* Name of rank function */
   char *zRankArgs;                /* Arguments to rank function */
 

ext/fts5/fts5_config.c

@@ -20,6 +20,7 @@
 #define FTS5_DEFAULT_PAGE_SIZE   4050
 #define FTS5_DEFAULT_AUTOMERGE      4
 #define FTS5_DEFAULT_CRISISMERGE   16
+#define FTS5_DEFAULT_HASHSIZE    (1024*1024)
 
 /* Maximum allowed page size */
 #define FTS5_MAX_PAGE_SIZE (128*1024)
@@ -767,6 +768,18 @@ int sqlite3Fts5ConfigSetValue(
     }
   }
 
+  else if( 0==sqlite3_stricmp(zKey, "hashsize") ){
+    int nHashSize = -1;
+    if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){
+      nHashSize = sqlite3_value_int(pVal);
+    }
+    if( nHashSize<=0 ){
+      *pbBadkey = 1;
+    }else{
+      pConfig->nHashSize = nHashSize;
+    }
+  }
+
   else if( 0==sqlite3_stricmp(zKey, "automerge") ){
     int nAutomerge = -1;
     if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){
@@ -827,6 +840,7 @@ int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){
   pConfig->pgsz = FTS5_DEFAULT_PAGE_SIZE;
   pConfig->nAutomerge = FTS5_DEFAULT_AUTOMERGE;
   pConfig->nCrisisMerge = FTS5_DEFAULT_CRISISMERGE;
+  pConfig->nHashSize = FTS5_DEFAULT_HASHSIZE;
 
   zSql = sqlite3Fts5Mprintf(&rc, zSelect, pConfig->zDb, pConfig->zName);
   if( zSql ){

ext/fts5/fts5_index.c

@@ -288,7 +288,6 @@ struct Fts5Index {
   ** in-memory hash tables before they are flushed to disk.
   */
   Fts5Hash *pHash;                /* Hash table for in-memory data */
-  int nMaxPendingData;            /* Max pending data before flush to disk */
   int nPendingData;               /* Current bytes of pending data */
   i64 iWriteRowid;                /* Rowid for current doc being written */
   int bDelete;                    /* Current write is a delete */
@@ -4448,12 +4447,15 @@ int sqlite3Fts5IndexBeginWrite(Fts5Index *p, int bDelete, i64 iRowid){
   /* Allocate the hash table if it has not already been allocated */
   if( p->pHash==0 ){
     p->rc = sqlite3Fts5HashNew(&p->pHash, &p->nPendingData);
+
+    /* Force the configuration to be loaded */
+    fts5StructureRelease(fts5StructureRead(p));
   }
 
   /* Flush the hash table to disk if required */
   if( iRowid<p->iWriteRowid 
    || (iRowid==p->iWriteRowid && p->bDelete==0)
-   || (p->nPendingData > p->nMaxPendingData) 
+   || (p->nPendingData > p->pConfig->nHashSize) 
   ){
     fts5IndexFlush(p);
   }
@@ -4519,7 +4521,6 @@ int sqlite3Fts5IndexOpen(
   if( rc==SQLITE_OK ){
     p->pConfig = pConfig;
     p->nWorkUnit = FTS5_WORK_UNIT;
-    p->nMaxPendingData = 1024*1024;
     p->zDataTbl = sqlite3Fts5Mprintf(&rc, "%s_data", pConfig->zName);
     if( p->zDataTbl && bCreate ){
       rc = sqlite3Fts5CreateTable(

ext/fts5/test/fts5simple.test

@@ -18,7 +18,7 @@ ifcapable !fts5 {
   finish_test
   return
 }
-
+ 
 #-------------------------------------------------------------------------
 #
 set doc "x x [string repeat {y } 50]z z"
@@ -322,5 +322,19 @@ do_execsql_test 13.3 {
   INSERT INTO xy(xy) VALUES('integrity-check');
 }
 
+#-------------------------------------------------------------------------
+#
+do_execsql_test 14.1 {
+  CREATE VIRTUAL TABLE ttt USING fts5(x);
+  BEGIN;
+    INSERT INTO ttt(rowid, x) VALUES(1, 'a b c');
+    INSERT INTO ttt(rowid, x) VALUES(2, 'a b c');
+    INSERT INTO ttt(rowid, x) VALUES(3, 'a b c');
+  COMMIT;
+}
+do_test 14.2 { 
+  fts5_level_segs ttt 
+} {1}
+
 finish_test
 

ext/fts5/tool/loadfts5.tcl

@@ -48,6 +48,7 @@ proc usage {} {
   puts stderr "  -crisismerge N (set the crisismerge parameter to N)"
   puts stderr "  -prefix PREFIX (comma separated prefix= argument)"
   puts stderr "  -trans N     (commit after N inserts - 0 == never)"
+  puts stderr "  -hashsize N  (set the fts5 hashsize parameteger to N)"
   exit 1
 }
 
@@ -59,6 +60,7 @@ set O(automerge)  -1
 set O(crisismerge)  -1
 set O(prefix)     ""
 set O(trans)      0
+set O(hashsize)   -1
 
 if {[llength $argv]<2} usage
 set nOpt [expr {[llength $argv]-2}]
@@ -106,6 +108,11 @@ for {set i 0} {$i < $nOpt} {incr i} {
       set O(prefix) [lindex $argv $i]
     }
 
+    -hashsize {
+      if { [incr i]>=$nOpt } usage
+      set O(hashsize) [lindex $argv $i]
+    }
+
     default {
       usage
     }
@@ -126,6 +133,14 @@ db eval BEGIN
     db eval "CREATE VIRTUAL TABLE t1 USING $O(vtab) (path, content$O(tok)$pref)"
     db eval "INSERT INTO t1(t1, rank) VALUES('pgsz', 4050);"
   }
+
+  if {$O(hashsize)>=0} {
+    catch {
+      db eval "INSERT INTO t1(t1, rank) VALUES('hashsize', $O(hashsize));"
+    }
+  }
+
+
   if {$O(automerge)>=0} {
     if {$O(vtab) == "fts5"} {
       db eval { INSERT INTO t1(t1, rank) VALUES('automerge', $O(automerge)) }
@@ -141,6 +156,7 @@ db eval BEGIN
   }
   load_hierachy [lindex $argv end]
 db eval COMMIT
+puts ""
 
 
 

manifest

@@ -1,5 +1,5 @@
-C Remove\sa\s#pragma\sused\sto\swork\saround\san\sissues\swith\sMSVC\s2012\sthat\shas\sbeen\novercome\sbut\ssubsequent\schanges.
-D 2015-11-05T11:47:48.838
+C Add\sthe\s'hashsize'\sconfiguration\soption\sto\sfts5,\sfor\sconfiguring\sthe\samount\sof\smemory\sallocated\sto\sthe\sin-memory\shash\stable\swhile\swriting.
+D 2015-11-05T18:09:16.776
 F Makefile.in 3a705bb4bd12e194212ddbdbf068310d17153cdb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 702d3e98f3afc6587a78481257f3c4c900efc3a4
@@ -103,13 +103,13 @@ F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252
 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95
 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
 F ext/fts5/fts5.h 8b9a13b309b180e9fb88ea5666c0d8d73c6102d9
-F ext/fts5/fts5Int.h 06594fd3e5a3c74da6df9141e165975dc0ea6ef4
+F ext/fts5/fts5Int.h acf968e43d57b6b1caf7554d34ec35d6ed3b4fe8
 F ext/fts5/fts5_aux.c 1f384972d606375b8fa078319f25ab4b5feb1b35
 F ext/fts5/fts5_buffer.c 1e49512a535045e621246dc7f4f65f3593fa0fc2
-F ext/fts5/fts5_config.c 88a77f5d5e4dfbb2355b8f6cc9969b7f02d94685
+F ext/fts5/fts5_config.c 81ec07cb644e33df4cd58d46990016b2ce7c175c
 F ext/fts5/fts5_expr.c 28b15c9ae296204bc0a2e5cf7a667d840a9d2900
 F ext/fts5/fts5_hash.c a9d4c1efebc2a91d26ad7ebdfcbf2678ceac405f
-F ext/fts5/fts5_index.c 356481ce027cc2ede8e462c316b578260cda29d2
+F ext/fts5/fts5_index.c 967d797282e4595f8671d152ed8ecf6cc7e640b5
 F ext/fts5/fts5_main.c 39358d3d8f0d6ea3757c40e0ddcbb6bc435604c3
 F ext/fts5/fts5_storage.c 9ea3d92178743758b6c54d9fe8836bbbdcc92e3b
 F ext/fts5/fts5_tcl.c 3bf445e66de32137d4693694ff7b1fd6074e32bd
@@ -174,7 +174,7 @@ F ext/fts5/test/fts5rank.test 11dcebba31d822f7e99685b4ea2c2ae3ec0b16f1
 F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b
 F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17
 F ext/fts5/test/fts5rowid.test 400384798349d658eaf06aefa1e364957d5d4821
-F ext/fts5/test/fts5simple.test 41333e267c6145efc3620342af53dfe65d5676b7
+F ext/fts5/test/fts5simple.test ec1c6a6f51890ba864f866387794878ff2994de9
 F ext/fts5/test/fts5synonym.test cf88c0a56d5ea9591e3939ef1f6e294f7f2d0671
 F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89
 F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841
@@ -184,7 +184,7 @@ F ext/fts5/test/fts5unindexed.test e9539d5b78c677315e7ed8ea911d4fd25437c680
 F ext/fts5/test/fts5version.test 978f59541d8cef7e8591f8be2115ec5ccb863e2e
 F ext/fts5/test/fts5vocab.test c88a5554d0409494da95ba647bbdb4879b2624b0
 F ext/fts5/tool/fts5txt2db.tcl c374c4c4797e8cdfadabdfaeeb5412dcd6686e84
-F ext/fts5/tool/loadfts5.tcl 58e90407cc5c2b1770460119488fd7c0090d4dd3
+F ext/fts5/tool/loadfts5.tcl 84d6f1b11096e984259765874dae7d119e3cfb54
 F ext/fts5/tool/mkfts5c.tcl d1c2a9ab8e0ec690a52316f33dd9b1d379942f45
 F ext/fts5/tool/showfts5.tcl d54da0e067306663e2d5d523965ca487698e722c
 F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43
@@ -1398,8 +1398,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 2aa50f6f2029e3c91a4cb91224df8ec1194f215e b9b22fae69000ca506656f9cf847e713c6304b75
-R 8f915e54b493516d25c62a2e6e65196c
-T +closed b9b22fae69000ca506656f9cf847e713c6304b75
-U drh
-Z 3e3a650cde9c3d0b9f2c0083b06276a0
+P 8303e4cfed715464c710668c2bdc58a8e3d4cd9f
+R 430b0b0a24aaa42e32648f0a371f94b9
+U dan
+Z 6707a1116c6926e8f57d8eea48f7af2a

manifest.uuid

@@ -1 +1 @@
-8303e4cfed715464c710668c2bdc58a8e3d4cd9f
+445480095e6877cce8220b1c095f334bbb04c1c3