Fix problems in fts5 found by ASAN.

FossilOrigin-Name: c564bf870106faef297594a51995619c80311d06bd5f8a0c7644f666f22ba576

ext/fts3/fts3_unicode2.c

@@ -200,7 +200,7 @@ static int remove_diacritic(int c, int bComplex){
     'w',       'x',       'y',       'z',       'h',       't',       
     'w',       'y',       'a',       'a'|HIBIT, 'a'|HIBIT, 'a'|HIBIT, 
     'e',       'e'|HIBIT, 'e'|HIBIT, 'i',       'o',       'o'|HIBIT, 
-    'o'|HIBIT, 'o'|HIBIT, 'u',       'u'|HIBIT, 'u'|HIBIT, 'y',  
+    'o'|HIBIT, 'o'|HIBIT, 'u',       'u'|HIBIT, 'u'|HIBIT, 'y',       
   };
 
   unsigned int key = (((unsigned int)c)<<3) | 0x00000007;
@@ -231,8 +231,8 @@ int sqlite3FtsUnicodeIsdiacritic(int c){
   unsigned int mask1 = 0x000361F8;
   if( c<768 || c>817 ) return 0;
   return (c < 768+32) ?
-      (mask0 & (1 << (c-768))) :
+      (mask0 & ((unsigned int)1 << (c-768))) :
-      (mask1 & (1 << (c-768-32)));
+      (mask1 & ((unsigned int)1 << (c-768-32)));
 }
 
 

ext/fts3/unicode/mkunicode.tcl

@@ -63,14 +63,15 @@ proc print_rd {map} {
   }
   puts ""
   puts "  \};"
+  puts "#define HIBIT ((char)0x80)"
   puts "  char aChar\[\] = \{"
   puts -nonewline "    '\\0',      "
   set i 1
   foreach c $aChar f $aFlag {
     if { $f } {
-      set str "'$c'|0x80,  "
+      set str "'$c'|HIBIT, "
     } else {
-      set str "'$c'|0x00,  "
+      set str "'$c',       "
     }
     if {$c == ""} { set str "'\\0',      " }
 
@@ -134,8 +135,8 @@ proc print_isdiacritic {zFunc map} {
 
   puts "  if( c<$iFirst || c>$iLast ) return 0;"
   puts "  return (c < $iFirst+32) ?"
-  puts "      (mask0 & (1 << (c-$iFirst))) :"
+  puts "      (mask0 & ((unsigned int)1 << (c-$iFirst))) :"
-  puts "      (mask1 & (1 << (c-$iFirst-32)));"
+  puts "      (mask1 & ((unsigned int)1 << (c-$iFirst-32)));"
   puts "\}"
 }
 
@@ -699,7 +700,7 @@ proc print_categories {lMap} {
     static u16 aFts5UnicodeMap[] = {$aMapArray};
     static u16 aFts5UnicodeData[] = {$aDataArray};
 
-    int sqlite3Fts5UnicodeCategory(int iCode) { 
+    int sqlite3Fts5UnicodeCategory(u32 iCode) { 
       int iRes = -1;
       int iHi;
       int iLo;
@@ -782,7 +783,7 @@ proc print_test_categories {lMap} {
           aArray[0] = 1;
         }
 
-        c = sqlite3Fts5UnicodeCategory(i);
+        c = sqlite3Fts5UnicodeCategory((u32)i);
         if( aArray[c]==0 ){
           *piCode = i;
           return 1;

ext/fts5/fts5Int.h

@@ -788,7 +788,7 @@ int sqlite3Fts5UnicodeIsdiacritic(int c);
 int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic);
 
 int sqlite3Fts5UnicodeCatParse(const char*, u8*);
-int sqlite3Fts5UnicodeCategory(int iCode);
+int sqlite3Fts5UnicodeCategory(u32 iCode);
 void sqlite3Fts5UnicodeAscii(u8*, u8*);
 /*
 ** End of interface to code in fts5_unicode2.c.

ext/fts5/fts5_expr.c

@@ -2553,7 +2553,7 @@ static void fts5ExprIsAlnum(
   sqlite3Fts5UnicodeCatParse("N*", aArr);
   sqlite3Fts5UnicodeCatParse("Co", aArr);
   iCode = sqlite3_value_int(apVal[0]);
-  sqlite3_result_int(pCtx, aArr[sqlite3Fts5UnicodeCategory(iCode)]);
+  sqlite3_result_int(pCtx, aArr[sqlite3Fts5UnicodeCategory((u32)iCode)]);
 }
 
 static void fts5ExprFold(

ext/fts5/fts5_index.c

@@ -3577,14 +3577,14 @@ static int fts5AllocateSegid(Fts5Index *p, Fts5Structure *pStruct){
         for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){
           int iId = pStruct->aLevel[iLvl].aSeg[iSeg].iSegid;
           if( iId<=FTS5_MAX_SEGMENT ){
-            aUsed[(iId-1) / 32] |= 1 << ((iId-1) % 32);
+            aUsed[(iId-1) / 32] |= (u32)1 << ((iId-1) % 32);
           }
         }
       }
 
       for(i=0; aUsed[i]==0xFFFFFFFF; i++);
       mask = aUsed[i];
-      for(iSegid=0; mask & (1 << iSegid); iSegid++);
+      for(iSegid=0; mask & ((u32)1 << iSegid); iSegid++);
       iSegid += 1 + i*32;
 
 #ifdef SQLITE_DEBUG
@@ -6282,8 +6282,7 @@ static void fts5DecodeFunction(
   nSpace = n + FTS5_DATA_ZERO_PADDING;
   a = (u8*)sqlite3Fts5MallocZero(&rc, nSpace);
   if( a==0 ) goto decode_out;
-  memcpy(a, aBlob, n);
+  if( n>0 ) memcpy(a, aBlob, n);
-
 
   fts5DecodeRowid(iRowid, &iSegid, &bDlidx, &iHeight, &iPgno);
 

ext/fts5/fts5_test_tok.c

@@ -378,7 +378,7 @@ static int fts5tokFilterMethod(
     if( pCsr->zInput==0 ){
       rc = SQLITE_NOMEM;
     }else{
-      memcpy(pCsr->zInput, zByte, nByte);
+      if( nByte>0 ) memcpy(pCsr->zInput, zByte, nByte);
       pCsr->zInput[nByte] = 0;
       rc = pTab->tok.xTokenize(
           pTab->pTok, (void*)pCsr, 0, zByte, nByte, fts5tokCb

ext/fts5/fts5_tokenize.c

@@ -268,7 +268,7 @@ static int fts5UnicodeAddExceptions(
         if( iCode<128 ){
           p->aTokenChar[iCode] = (unsigned char)bTokenChars;
         }else{
-          bToken = p->aCategory[sqlite3Fts5UnicodeCategory(iCode)];
+          bToken = p->aCategory[sqlite3Fts5UnicodeCategory((u32)iCode)];
           assert( (bToken==0 || bToken==1) ); 
           assert( (bTokenChars==0 || bTokenChars==1) );
           if( bToken!=bTokenChars && sqlite3Fts5UnicodeIsdiacritic(iCode)==0 ){
@@ -429,7 +429,7 @@ static int fts5UnicodeCreate(
 */
 static int fts5UnicodeIsAlnum(Unicode61Tokenizer *p, int iCode){
   return (
-    p->aCategory[sqlite3Fts5UnicodeCategory(iCode)]
+    p->aCategory[sqlite3Fts5UnicodeCategory((u32)iCode)]
     ^ fts5UnicodeIsException(p, iCode)
   );
 }

ext/fts5/fts5_unicode2.c

@@ -69,7 +69,7 @@ static int fts5_remove_diacritic(int c, int bComplex){
     'w',       'x',       'y',       'z',       'h',       't',       
     'w',       'y',       'a',       'a'|HIBIT, 'a'|HIBIT, 'a'|HIBIT, 
     'e',       'e'|HIBIT, 'e'|HIBIT, 'i',       'o',       'o'|HIBIT, 
-    'o'|HIBIT, 'o'|HIBIT, 'u',       'u'|HIBIT, 'u'|HIBIT, 'y',  
+    'o'|HIBIT, 'o'|HIBIT, 'u',       'u'|HIBIT, 'u'|HIBIT, 'y',       
   };
 
   unsigned int key = (((unsigned int)c)<<3) | 0x00000007;
@@ -100,8 +100,8 @@ int sqlite3Fts5UnicodeIsdiacritic(int c){
   unsigned int mask1 = 0x000361F8;
   if( c<768 || c>817 ) return 0;
   return (c < 768+32) ?
-      (mask0 & (1 << (c-768))) :
+      (mask0 & ((unsigned int)1 << (c-768))) :
-      (mask1 & (1 << (c-768-32)));
+      (mask1 & ((unsigned int)1 << (c-768-32)));
 }
 
 
@@ -249,6 +249,7 @@ int sqlite3Fts5UnicodeFold(int c, int eRemoveDiacritic){
   return ret;
 }
 
+
 int sqlite3Fts5UnicodeCatParse(const char *zCat, u8 *aArray){ 
   aArray[0] = 1;
   switch( zCat[0] ){
@@ -730,7 +731,7 @@ static u16 aFts5UnicodeData[] = {
     34,    3074,  7692,  63,    63,    
   };
 
-int sqlite3Fts5UnicodeCategory(int iCode) { 
+int sqlite3Fts5UnicodeCategory(u32 iCode) { 
   int iRes = -1;
   int iHi;
   int iLo;
@@ -773,3 +774,4 @@ void sqlite3Fts5UnicodeAscii(u8 *aArray, u8 *aAscii){
     iTbl++;
   }
 }
+

ext/fts5/test/fts5unicode3.test

@@ -21,7 +21,7 @@ ifcapable !fts5 {
 }
 
 proc fts3_unicode_path {file} {
-  file join [file dirname [info script]] .. .. fts3 unicode $file
+  file join .. [file dirname [info script]] .. .. fts3 unicode $file
 }
 
 source [fts3_unicode_path parseunicode.tcl]

manifest

@@ -1,5 +1,5 @@
-C Fix\sanother\sproblem\swith\scorrupt\sdatabase\shandling\sin\sfts5.
+C Fix\sproblems\sin\sfts5\sfound\sby\sASAN.
-D 2018-12-27T20:12:02.887
+D 2018-12-28T07:37:22.836
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in d8b254f8bb81bab43c340d70d17dc3babab40fcc8a348c8255881f780a45fee6
@@ -97,7 +97,7 @@ F ext/fts3/fts3_tokenizer.c a22bf311a71f3efa9d7012d8cc48fc9b0f3dace7
 F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3
 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004
 F ext/fts3/fts3_unicode.c b1902e9ad47a6569fbb8ecb5ce52f20fe59b590d5c5e3bbdd56b10b03bdf632b
-F ext/fts3/fts3_unicode2.c e49f9e015f239bf5faf2f4fa483bbf1b08a9978f0ad1f31159d952f8b8a10d08
+F ext/fts3/fts3_unicode2.c faf0c750a1d0e9c9902f947212fc88f84e9450cddb3d1ffdbacbd0084b91d647
 F ext/fts3/fts3_write.c e36d2f7e8f180e8030e92a5c2d09ccf87021afedcc5148a9d823b496667bf2f2
 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9
 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100
@@ -105,24 +105,24 @@ F ext/fts3/tool/fts3cov.sh c331d006359456cf6f8f953e37f2b9c7d568f3863f00bb5f7eb87
 F ext/fts3/tool/fts3view.c 202801a2056995b763864d60c2dee744d46f1677
 F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c
 F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7
-F ext/fts3/unicode/mkunicode.tcl 2ea30d8122ccf1e33142c9cc8913d8cad9eb6668db359a228f10aeb37e2ab863
+F ext/fts3/unicode/mkunicode.tcl 2315b3f8c83dcb09091009db01b321fb401d98d8c5489c147c1a7c5a54d21e20
 F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb
 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
 F ext/fts5/fts5.h 4f5d19b7973dae23de368728f06d3eb1fe9f5cca2990366b40e9379996f35e61
-F ext/fts5/fts5Int.h 39f12034b598df4e0f59bbe6cf03af03a905a534b04f182d155641a04e1eb797
+F ext/fts5/fts5Int.h 8590e08a485c98a7e3075ed72abe3452fe944a9e58e63dfa51b732cced344cab
 F ext/fts5/fts5_aux.c ca666a3bbe07c5a3bbe9fffaea19c935a1efaf337333e28bad7bdd1971ffd093
 F ext/fts5/fts5_buffer.c 1dd1ec0446b3acfc2d7d407eb894762a461613e2695273f48e449bfd13e973ff
 F ext/fts5/fts5_config.c 5af9c360e99669d29f06492c370892394aba0857
-F ext/fts5/fts5_expr.c 5aef080ba3c8947e22f38ce1ff9fe548e4a740e72b77241f35ed941ae128d2c7
+F ext/fts5/fts5_expr.c 0eff5bad90838be83c5cc16837ca215a961abe05cfbf741a285ba62a79ad586f
 F ext/fts5/fts5_hash.c 32be400cf761868c9db33efe81a06eb19a17c5402ad477ee9efb51301546dd55
-F ext/fts5/fts5_index.c f3601579efeca9a859bf4cc82cbe99106fa8e3277a96b96c44ce35c7ceffb6d0
+F ext/fts5/fts5_index.c c67a3bf1c185ff05937a9e3a01b8a6981f5a6add8da93911f461d59e57eea8d5
 F ext/fts5/fts5_main.c 287a1a56580df304d7fa2fc1890f85b9cb6ac6b9e7c8af7dfa2151528db4b059
 F ext/fts5/fts5_storage.c 5862f1b785a983acb8420281340f3f424896ab48f396f6fd8540787be7459139
 F ext/fts5/fts5_tcl.c 39bcbae507f594aad778172fa914cad0f585bf92fd3b078c686e249282db0d95
 F ext/fts5/fts5_test_mi.c 65864ba1e5c34a61d409c4c587e0bbe0466eb4f8f478d85dc42a92caad1338e6
-F ext/fts5/fts5_test_tok.c 80de1a4b1a3caa216c3be8862440f0117a8357dd9b7cfc5a2a2ce11fe6eb64ae
+F ext/fts5/fts5_test_tok.c 6a5574354ce61a98737e150fd4f7a002000db0cb4bcd655dc8694cb3e035381d
-F ext/fts5/fts5_tokenize.c ca2b6a033794945ac505241a86b0aa978709c23aa2e6121984d3e3ede96003c8
+F ext/fts5/fts5_tokenize.c 240f849c9136103f77a67818deb172d8a7564cbefaf00ec7b2fd04ab17d6d66c
-F ext/fts5/fts5_unicode2.c 03bc8ebf44d825d0b7ac519f7b16139cf8e879ae7a27e62fb6326f0d68e8bdd3
+F ext/fts5/fts5_unicode2.c 3f1bad6f2924ad13e25408f00301edd12d4216388a11daa518cb28f79b337f5e
 F ext/fts5/fts5_varint.c a5aceacda04dafcbae725413d7a16818ecd65738
 F ext/fts5/fts5_vocab.c fbe38044889b2d2d99babeeef239c620fb0332bb928a84506ac748d81500b354
 F ext/fts5/fts5parse.y eb526940f892ade5693f22ffd6c4f2702543a9059942772526eac1fde256bb05
@@ -211,7 +211,7 @@ F ext/fts5/test/fts5tokenizer.test ac3c9112b263a639fb0508ae73a3ee886bf4866d21537
 F ext/fts5/test/fts5umlaut.test a42fe2fe6387c40c49ab27ccbd070e1ae38e07f38d05926482cc0bccac9ad602
 F ext/fts5/test/fts5unicode.test 17056f4efe6b0a5d4f41fdf7a7dc9af2873004562eaa899d40633b93dc95f5a9
 F ext/fts5/test/fts5unicode2.test 9b3df486de05fb4bde4aa7ee8de2e6dae1df6eb90e3f2e242c9383b95d314e3e
-F ext/fts5/test/fts5unicode3.test 9cbc82e2b02e2e3b7504103580c90f095e07fe8230b1951a9ed7558717b5feb7
+F ext/fts5/test/fts5unicode3.test 590c72e18195bda2446133f9d82d04a4e89d094bba58c75ae10f4afc6faa0744
 F ext/fts5/test/fts5unicode4.test 6463301d669f963c83988017aa354108be0b947d325aef58d3abddf27147b687
 F ext/fts5/test/fts5unindexed.test 9021af86a0fb9fc616f7a69a996db0116e7936d0db63892db6bafabbec21af4d
 F ext/fts5/test/fts5update.test 0737876e20e97a6a6abf45de19fc99315727bcee6a83fadcada1cc080b9aa8f0
@@ -1794,7 +1794,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P d9e4f9574659c6285deba3dca68eca9989e615971cea3a901ffcda12fb4ee36f
+P fb0d7fba07865ff557bfcdacfda7b625d099ac8b4dcbe0c08cd2d4b84a068bb9
-R 308e6395f3ea79d50b990f4e574e128e
+R 31f0c322d87dce0c17eac6b9b39fdaff
 U dan
-Z d6ac7bbdbfe101fd5a8ad83d11fdc195
+Z f209c1d4c450aa64c337c8ad21da7144

manifest.uuid

@@ -1 +1 @@
-fb0d7fba07865ff557bfcdacfda7b625d099ac8b4dcbe0c08cd2d4b84a068bb9
+c564bf870106faef297594a51995619c80311d06bd5f8a0c7644f666f22ba576