From 512795dfeae9dd816eefda3eee3222aedcd6d2b5 Mon Sep 17 00:00:00 2001 From: drh Date: Sun, 24 Dec 2017 18:56:28 +0000 Subject: [PATCH 1/5] Fix a NULL pointer dereference after a syntax error that can occur as a result of check-in [6b2ff26c25bb9da3] yesterday. This problem was discovered by the OSSFuzz. FossilOrigin-Name: d49afb8f9804e96662d1e3cadc4c6643908706d848a53d5ed019919c98f2ccba --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/build.c | 1 + test/colname.test | 10 ++++++++++ 4 files changed, 19 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index f187cc0290..1c622c1102 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Simplification\sto\sthe\serror\shandling\slogic\sin\sthe\sextension\sloader. -D 2017-12-23T14:39:36.160 +C Fix\sa\sNULL\spointer\sdereference\safter\sa\ssyntax\serror\sthat\scan\soccur\sas\sa\nresult\sof\scheck-in\s[6b2ff26c25bb9da3]\syesterday.\s\sThis\sproblem\swas\ndiscovered\sby\sthe\sOSSFuzz. +D 2017-12-24T18:56:28.786 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69 @@ -423,7 +423,7 @@ F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca F src/btree.c b83a6b03f160528020bb965f0c3a40af5286cd4923c3870fd218177f03a120a7 F src/btree.h 32ef5d3f25dc70ef1ee9cecf84a023c21378f06a57cd701d2e866e141b150f09 F src/btreeInt.h 55b702efce17e5d1941865464227d3802cfc9c7c832fac81d4c94dced47a71fc -F src/build.c ed567f088edbc305dad33a6b14e08f8216a3860f6bad1d180450d5a5414bf346 +F src/build.c ab5bdf955c85bcd56acbf310a48bbd50b4b92079efa40d997a7e4246f8e03741 F src/callback.c fe677cb5f5abb02f7a772a62a98c2f516426081df68856e8f2d5f950929b966a F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0 @@ -682,7 +682,7 @@ F test/collate9.test 3adcc799229545940df2f25308dd1ad65869145a F test/collateA.test b8218ab90d1fa5c59dcf156efabb1b2599c580d6 F test/collateB.test 1e68906951b846570f29f20102ed91d29e634854ee47454d725f2151ecac0b95 F test/colmeta.test 2c765ea61ee37bc43bbe6d6047f89004e6508eb1 -F test/colname.test a7ecb8f1d6d8b30a6cf8fa84a2cd6f6e91cad8296376fabe485cf93cd5eb6229 +F test/colname.test 36da785927822ecd0de979459e27e9be63f458dd08d3edde41af3af37a337d58 F test/conflict.test 029faa2d81a0d1cafb5f88614beb663d972c01db F test/conflict2.test bb0b94cf7196c64a3cbd815c66d3ee98c2fecd9c F test/conflict3.test a83db76a6c3503b2fa057c7bfb08c318d8a422202d8bc5b86226e078e5b49ff9 @@ -1687,7 +1687,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 05fee1a21ea398f1e4d6f1cf361657eff25ed6cd8f85ab398262dcfd30da57e9 -R 7fbfe3e61eca395fa5baf5f121a4d2ad +P 07c773148d8db185fa54991df09298b64f4fef28879e6c9395759265e8183977 +R 2e4222d820aa06549d33319e9e33c627 U drh -Z d82d9c01768cefc5beb206b92e1398bf +Z 67298e45eb2d4d992f0cdf5678deffdf diff --git a/manifest.uuid b/manifest.uuid index 8454418cd5..d58d2c0dd5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -07c773148d8db185fa54991df09298b64f4fef28879e6c9395759265e8183977 \ No newline at end of file +d49afb8f9804e96662d1e3cadc4c6643908706d848a53d5ed019919c98f2ccba \ No newline at end of file diff --git a/src/build.c b/src/build.c index 01d8972415..390dae5d3b 100644 --- a/src/build.c +++ b/src/build.c @@ -1965,6 +1965,7 @@ void sqlite3EndTable( pParse->nTab = 2; addrTop = sqlite3VdbeCurrentAddr(v) + 1; sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, addrTop); + if( pParse->nErr ) return; pSelTab = sqlite3ResultSetOfSelect(pParse, pSelect); if( pSelTab==0 ) return; assert( p->aCol==0 ); diff --git a/test/colname.test b/test/colname.test index 5a40286773..cacf91e4e3 100644 --- a/test/colname.test +++ b/test/colname.test @@ -398,6 +398,16 @@ do_execsql_test colname-9.320 { SELECT name FROM pragma_table_info('t2'); } {Bbb} +# Issue detected by clusterfuzz on 2017-12-24 (Christmas Eve) +# caused by check-in https://sqlite.org/src/info/6b2ff26c25 +# +# Prior to being fixed, the following CREATE TABLE was dereferencing +# a NULL pointer and segfaulting. +# +do_catchsql_test colname-9.400 { + CREATE TABLE t4 AS SELECT #0; +} {1 {near "#0": syntax error}} + # Make sure the quotation marks get removed from the column names # when constructing a new table from an aggregate SELECT. From 5060a67c4d825b0286966ec9df37e1105a734c2c Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 25 Dec 2017 13:43:54 +0000 Subject: [PATCH 2/5] Fix an assertion fault on a syntax error input caused by check-in [6b2ff26c25bb9da3]. Problem discovered by OSSFuzz. FossilOrigin-Name: 90d6e4f10d3055a776d24854c442a2a68e726af8fc382cdb6241a834082e4c4c --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/build.c | 1 + test/colname.test | 11 ++++++++++- 4 files changed, 19 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 1c622c1102..d9715194d3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sNULL\spointer\sdereference\safter\sa\ssyntax\serror\sthat\scan\soccur\sas\sa\nresult\sof\scheck-in\s[6b2ff26c25bb9da3]\syesterday.\s\sThis\sproblem\swas\ndiscovered\sby\sthe\sOSSFuzz. -D 2017-12-24T18:56:28.786 +C Fix\san\sassertion\sfault\son\sa\ssyntax\serror\sinput\scaused\sby\scheck-in\n[6b2ff26c25bb9da3].\s\sProblem\sdiscovered\sby\sOSSFuzz. +D 2017-12-25T13:43:54.974 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69 @@ -423,7 +423,7 @@ F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca F src/btree.c b83a6b03f160528020bb965f0c3a40af5286cd4923c3870fd218177f03a120a7 F src/btree.h 32ef5d3f25dc70ef1ee9cecf84a023c21378f06a57cd701d2e866e141b150f09 F src/btreeInt.h 55b702efce17e5d1941865464227d3802cfc9c7c832fac81d4c94dced47a71fc -F src/build.c ab5bdf955c85bcd56acbf310a48bbd50b4b92079efa40d997a7e4246f8e03741 +F src/build.c 6ab114a9a4717622df3745d26f2c55e9113f356441a8f76fa2b1d7e1f2472c20 F src/callback.c fe677cb5f5abb02f7a772a62a98c2f516426081df68856e8f2d5f950929b966a F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0 @@ -682,7 +682,7 @@ F test/collate9.test 3adcc799229545940df2f25308dd1ad65869145a F test/collateA.test b8218ab90d1fa5c59dcf156efabb1b2599c580d6 F test/collateB.test 1e68906951b846570f29f20102ed91d29e634854ee47454d725f2151ecac0b95 F test/colmeta.test 2c765ea61ee37bc43bbe6d6047f89004e6508eb1 -F test/colname.test 36da785927822ecd0de979459e27e9be63f458dd08d3edde41af3af37a337d58 +F test/colname.test 101aa39392a1f6883278f588836a3ab99178f8103f78032433400475cc05109f F test/conflict.test 029faa2d81a0d1cafb5f88614beb663d972c01db F test/conflict2.test bb0b94cf7196c64a3cbd815c66d3ee98c2fecd9c F test/conflict3.test a83db76a6c3503b2fa057c7bfb08c318d8a422202d8bc5b86226e078e5b49ff9 @@ -1687,7 +1687,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 07c773148d8db185fa54991df09298b64f4fef28879e6c9395759265e8183977 -R 2e4222d820aa06549d33319e9e33c627 +P d49afb8f9804e96662d1e3cadc4c6643908706d848a53d5ed019919c98f2ccba +R 22dc7c511444b2944fcb40dc7656150e U drh -Z 67298e45eb2d4d992f0cdf5678deffdf +Z df3951acae9acd7f479ba37833a7cfe6 diff --git a/manifest.uuid b/manifest.uuid index d58d2c0dd5..55584bef0a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d49afb8f9804e96662d1e3cadc4c6643908706d848a53d5ed019919c98f2ccba \ No newline at end of file +90d6e4f10d3055a776d24854c442a2a68e726af8fc382cdb6241a834082e4c4c \ No newline at end of file diff --git a/src/build.c b/src/build.c index 390dae5d3b..ae2a92c79b 100644 --- a/src/build.c +++ b/src/build.c @@ -1976,6 +1976,7 @@ void sqlite3EndTable( sqlite3DeleteTable(db, pSelTab); sqlite3SelectDestInit(&dest, SRT_Coroutine, regYield); sqlite3Select(pParse, pSelect, &dest); + if( pParse->nErr ) return; sqlite3VdbeEndCoroutine(v, regYield); sqlite3VdbeJumpHere(v, addrTop - 1); addrInsLoop = sqlite3VdbeAddOp1(v, OP_Yield, dest.iSDParm); diff --git a/test/colname.test b/test/colname.test index cacf91e4e3..358ea77e07 100644 --- a/test/colname.test +++ b/test/colname.test @@ -398,7 +398,7 @@ do_execsql_test colname-9.320 { SELECT name FROM pragma_table_info('t2'); } {Bbb} -# Issue detected by clusterfuzz on 2017-12-24 (Christmas Eve) +# Issue detected by OSSFuzz on 2017-12-24 (Christmas Eve) # caused by check-in https://sqlite.org/src/info/6b2ff26c25 # # Prior to being fixed, the following CREATE TABLE was dereferencing @@ -408,6 +408,15 @@ do_catchsql_test colname-9.400 { CREATE TABLE t4 AS SELECT #0; } {1 {near "#0": syntax error}} +# Issue detected by OSSFuzz on 2017-12-25 (Christmas Day) +# also caused by check-in https://sqlite.org/src/info/6b2ff26c25 +# +# Prior to being fixed, the following CREATE TABLE caused an +# assertion fault. +# +do_catchsql_test colname-9.410 { + CREATE TABLE t5 AS SELECT RAISE(abort,a); +} {1 {RAISE() may only be used within a trigger-program}} # Make sure the quotation marks get removed from the column names # when constructing a new table from an aggregate SELECT. From f6c37dbc8c9ae15689f91e1d9c1e43e7b8737120 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 26 Dec 2017 14:30:44 +0000 Subject: [PATCH 3/5] Add options to speedtest1.c and speed-check.sh for testing performance of floating-point computatations. FossilOrigin-Name: ebfea8728fec955b1d74b1d0a3de498fd1a32e8b39572a8fdab606ed87b169b4 --- manifest | 17 +++++----- manifest.uuid | 2 +- test/speedtest1.c | 76 +++++++++++++++++++++++++++++++++++++++++++-- tool/speed-check.sh | 6 ++++ 4 files changed, 90 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index d9715194d3..aa2114d805 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sassertion\sfault\son\sa\ssyntax\serror\sinput\scaused\sby\scheck-in\n[6b2ff26c25bb9da3].\s\sProblem\sdiscovered\sby\sOSSFuzz. -D 2017-12-25T13:43:54.974 +C Add\soptions\sto\sspeedtest1.c\sand\sspeed-check.sh\sfor\stesting\sperformance\sof\nfloating-point\scomputatations. +D 2017-12-26T14:30:44.117 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69 @@ -1248,7 +1248,7 @@ F test/speed3.test 694affeb9100526007436334cf7d08f3d74b85ef F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa F test/speed4p.test 0e51908951677de5a969b723e03a27a1c45db38b -F test/speedtest1.c e44c5fccddcfe916c3bf7fe2f87dcc4b4fd66a0d923eb83515f311212670f267 +F test/speedtest1.c a5faf4cbe5769eee4b721b3875cb3f12520a9b99d9026b1063b47c39603375b8 F test/spellfix.test f9c1f431e2c096c8775fec032952320c0e4700db F test/spellfix2.test dfc8f519a3fc204cb2dfa8b4f29821ae90f6f8c3 F test/spellfix3.test 0f9efaaa502a0e0a09848028518a6fb096c8ad33 @@ -1647,7 +1647,7 @@ F tool/showstat4.c 0682ebea7abf4d3657f53c4a243f2e7eab48eab344ed36a94bb75dcd19a5c F tool/showwal.c ad9d768f96ca6199ad3a8c9562d679680bd032dd01204ea3e5ea6fb931d81847 F tool/soak1.tcl 8d407956e1a45b485a8e072470a3e629a27037fe F tool/spaceanal.tcl 4bfd19aad7eb3ce0372ef0255f58035e0bba4ff5e9acfd763a10c6fb365c8dec -F tool/speed-check.sh a97ae367e9172a706101901e7caef48f1a14fc8a49053b25e79f6a67296b3412 +F tool/speed-check.sh 9ae425da8819e54e780cf494fc6d8175dfb16e109ae3214a45a5c9bb2b74e2c4 F tool/speedtest.tcl 06c76698485ccf597b9e7dbb1ac70706eb873355 F tool/speedtest16.c ecb6542862151c3e6509bbc00509b234562ae81e F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff @@ -1687,7 +1687,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d49afb8f9804e96662d1e3cadc4c6643908706d848a53d5ed019919c98f2ccba -R 22dc7c511444b2944fcb40dc7656150e -U drh -Z df3951acae9acd7f479ba37833a7cfe6 +P 90d6e4f10d3055a776d24854c442a2a68e726af8fc382cdb6241a834082e4c4c +Q +b3c6105181d22c8a53f5bf662e3d182bba962ee8afd1ad972ebd10094b20fc17 +R c2447707ab323a591ab8e05be0f0cae9 +U dan +Z 1002781326ca68c114cfc46396055176 diff --git a/manifest.uuid b/manifest.uuid index 55584bef0a..aaa3417a9c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -90d6e4f10d3055a776d24854c442a2a68e726af8fc382cdb6241a834082e4c4c \ No newline at end of file +ebfea8728fec955b1d74b1d0a3de498fd1a32e8b39572a8fdab606ed87b169b4 \ No newline at end of file diff --git a/test/speedtest1.c b/test/speedtest1.c index b92801a208..9342da79bf 100644 --- a/test/speedtest1.c +++ b/test/speedtest1.c @@ -32,7 +32,7 @@ static const char zHelp[] = " --size N Relative test size. Default=100\n" " --stats Show statistics at the end\n" " --temp N N from 0 to 9. 0: no temp table. 9: all temp tables\n" - " --testset T Run test-set T (main, cte, rtree, orm, debug)\n" + " --testset T Run test-set T (main, cte, rtree, orm, fp, debug)\n" " --trace Turn on SQL tracing\n" " --threads N Use up to N threads for sorting\n" " --utf16be Set text encoding to UTF-16BE\n" @@ -1120,7 +1120,77 @@ void testset_cte(void){ ); speedtest1_run(); speedtest1_end_test(); +} +/* +** Compute a pseudo-random floating point ascii number. +*/ +void speedtest1_random_ascii_fp(char *zFP){ + int x = speedtest1_random(); + int y = speedtest1_random(); + int z; + z = y%10; + if( z<0 ) z = -z; + y /= 10; + sqlite3_snprintf(100,zFP,"%d.%de%d",y,z,x%200); +} + +/* +** A testset for floating-point numbers. +*/ +void testset_fp(void){ + int n; + int i; + char zFP1[100]; + char zFP2[100]; + + n = g.szTest*5000; + speedtest1_begin_test(100, "Fill a table with %d FP values", n*2); + speedtest1_exec("BEGIN"); + speedtest1_exec("CREATE%s TABLE t1(a REAL %s, b REAL %s);", + isTemp(1), g.zNN, g.zNN); + speedtest1_prepare("INSERT INTO t1 VALUES(?1,?2); -- %d times", n); + for(i=1; i<=n; i++){ + speedtest1_random_ascii_fp(zFP1); + speedtest1_random_ascii_fp(zFP2); + sqlite3_bind_text(g.pStmt, 1, zFP1, -1, SQLITE_STATIC); + sqlite3_bind_text(g.pStmt, 2, zFP2, -1, SQLITE_STATIC); + speedtest1_run(); + } + speedtest1_exec("COMMIT"); + speedtest1_end_test(); + + n = g.szTest/25 + 2; + speedtest1_begin_test(110, "%d range queries", n); + speedtest1_prepare("SELECT sum(b) FROM t1 WHERE a BETWEEN ?1 AND ?2"); + for(i=1; i<=n; i++){ + speedtest1_random_ascii_fp(zFP1); + speedtest1_random_ascii_fp(zFP2); + sqlite3_bind_text(g.pStmt, 1, zFP1, -1, SQLITE_STATIC); + sqlite3_bind_text(g.pStmt, 2, zFP2, -1, SQLITE_STATIC); + speedtest1_run(); + } + speedtest1_end_test(); + + speedtest1_begin_test(120, "CREATE INDEX three times"); + speedtest1_exec("BEGIN;"); + speedtest1_exec("CREATE INDEX t1a ON t1(a);"); + speedtest1_exec("CREATE INDEX t1b ON t1(b);"); + speedtest1_exec("CREATE INDEX t1ab ON t1(a,b);"); + speedtest1_exec("COMMIT;"); + speedtest1_end_test(); + + n = g.szTest/3 + 2; + speedtest1_begin_test(130, "%d indexed range queries", n); + speedtest1_prepare("SELECT sum(b) FROM t1 WHERE a BETWEEN ?1 AND ?2"); + for(i=1; i<=n; i++){ + speedtest1_random_ascii_fp(zFP1); + speedtest1_random_ascii_fp(zFP2); + sqlite3_bind_text(g.pStmt, 1, zFP1, -1, SQLITE_STATIC); + sqlite3_bind_text(g.pStmt, 2, zFP2, -1, SQLITE_STATIC); + speedtest1_run(); + } + speedtest1_end_test(); } #ifdef SQLITE_ENABLE_RTREE @@ -1873,6 +1943,8 @@ int main(int argc, char **argv){ testset_orm(); }else if( strcmp(zTSet,"cte")==0 ){ testset_cte(); + }else if( strcmp(zTSet,"fp")==0 ){ + testset_fp(); }else if( strcmp(zTSet,"rtree")==0 ){ #ifdef SQLITE_ENABLE_RTREE testset_rtree(6, 147); @@ -1881,7 +1953,7 @@ int main(int argc, char **argv){ "the R-Tree tests\n"); #endif }else{ - fatal_error("unknown testset: \"%s\"\nChoices: main debug1 cte rtree\n", + fatal_error("unknown testset: \"%s\"\nChoices: main debug1 cte rtree fp\n", zTSet); } speedtest1_final(); diff --git a/tool/speed-check.sh b/tool/speed-check.sh index 2cda5c8078..6cc3018981 100644 --- a/tool/speed-check.sh +++ b/tool/speed-check.sh @@ -117,6 +117,12 @@ while test "$1" != ""; do --orm) SPEEDTEST_OPTS="$SPEEDTEST_OPTS --testset orm" ;; + --cte) + SPEEDTEST_OPTS="$SPEEDTEST_OPTS --testset cte" + ;; + --fp) + SPEEDTEST_OPTS="$SPEEDTEST_OPTS --testset fp" + ;; -*) CC_OPTS="$CC_OPTS $1" ;; From c8f44d245c5034c8f96361026a49b748ac02a700 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 26 Dec 2017 14:32:25 +0000 Subject: [PATCH 4/5] Fix crashes that could occur if SQL NULL values were passed to the built-in FTS5 snippet function. Edit: breaks amalgamation builds. FossilOrigin-Name: 6a790b67a0a5c698526db16ea262b13ecdd1b6ca74e80bdccfcad88ddbdc933a --- ext/fts5/fts5_aux.c | 16 +++++++++++++--- ext/fts5/test/fts5af.test | 10 ++++++++++ manifest | 16 ++++++++-------- manifest.uuid | 2 +- 4 files changed, 32 insertions(+), 12 deletions(-) diff --git a/ext/fts5/fts5_aux.c b/ext/fts5/fts5_aux.c index 219ea6fff8..594b981dda 100644 --- a/ext/fts5/fts5_aux.c +++ b/ext/fts5/fts5_aux.c @@ -358,6 +358,16 @@ static int fts5SnippetScore( return rc; } +/* +** Return the value in pVal interpreted as utf-8 text. Except, if pVal +** contains a NULL value, return a pointer to a static string zero +** bytes in length instead of a NULL pointer. +*/ +static const char *fts5ValueToText(sqlite3_value *pVal){ + const char *zRet = (const char*)sqlite3_value_text(pVal); + return zRet ? zRet : ""; +} + /* ** Implementation of snippet() function. */ @@ -393,9 +403,9 @@ static void fts5SnippetFunction( nCol = pApi->xColumnCount(pFts); memset(&ctx, 0, sizeof(HighlightContext)); iCol = sqlite3_value_int(apVal[0]); - ctx.zOpen = (const char*)sqlite3_value_text(apVal[1]); - ctx.zClose = (const char*)sqlite3_value_text(apVal[2]); - zEllips = (const char*)sqlite3_value_text(apVal[3]); + ctx.zOpen = fts5ValueToText(apVal[1]); + ctx.zClose = fts5ValueToText(apVal[2]); + zEllips = fts5ValueToText(apVal[3]); nToken = sqlite3_value_int(apVal[4]); iBestCol = (iCol>=0 ? iCol : 0); diff --git a/ext/fts5/test/fts5af.test b/ext/fts5/test/fts5af.test index fa4ebd2955..86c8f753fa 100644 --- a/ext/fts5/test/fts5af.test +++ b/ext/fts5/test/fts5af.test @@ -175,6 +175,16 @@ do_execsql_test 5.1 { SELECT snippet(p1, 0, '[', ']', '...', 6) FROM p1('x'); } {{[x] a a a a a...}} +do_execsql_test 5.2 { + SELECT snippet(p1, 0, '[', ']', NULL, 6) FROM p1('x'); +} {{[x] a a a a a}} +do_execsql_test 5.3 { + SELECT snippet(p1, 0, NULL, ']', '...', 6) FROM p1('x'); +} {{x] a a a a a...}} +do_execsql_test 5.4 { + SELECT snippet(p1, 0, '[', NULL, '...', 6) FROM p1('x'); +} {{[x a a a a a...}} + } ;# foreach_detail_mode finish_test diff --git a/manifest b/manifest index aa2114d805..f3f08454e9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\soptions\sto\sspeedtest1.c\sand\sspeed-check.sh\sfor\stesting\sperformance\sof\nfloating-point\scomputatations. -D 2017-12-26T14:30:44.117 +C Fix\scrashes\sthat\scould\soccur\sif\sSQL\sNULL\svalues\swere\spassed\sto\sthe\sbuilt-in\nFTS5\ssnippet\sfunction.\sEdit:\sbreaks\samalgamation\sbuilds. +D 2017-12-26T14:32:25.807 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69 @@ -106,7 +106,7 @@ F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0 F ext/fts5/fts5.h 62f3e33ceeb9a428db139f9c012186b371da1cc7 F ext/fts5/fts5Int.h eda28e3a0a5d87c412e8355fe35da875b04cb389908c8eb0d867ad662adbc491 -F ext/fts5/fts5_aux.c 67acf8d51723cf28ffc3828210ba662df4b8d267 +F ext/fts5/fts5_aux.c ca666a3bbe07c5a3bbe9fffaea19c935a1efaf337333e28bad7bdd1971ffd093 F ext/fts5/fts5_buffer.c 1dd1ec0446b3acfc2d7d407eb894762a461613e2695273f48e449bfd13e973ff F ext/fts5/fts5_config.c 5af9c360e99669d29f06492c370892394aba0857 F ext/fts5/fts5_expr.c 01048018d21524e2c302b063ff5c3cdcf546e03297215e577205d85b47499deb @@ -129,7 +129,7 @@ F ext/fts5/test/fts5ab.test 9205c839332c908aaad2b01ab8670ece8b161e8f2ec8a9fabf18 F ext/fts5/test/fts5ac.test a7aa7e1fefc6e1918aa4d3111d5c44a09177168e962c5fd2cca9620de8a7ed6d F ext/fts5/test/fts5ad.test e8cf959dfcd57c8e46d6f5f25665686f3b6627130a9a981371dafdf6482790de F ext/fts5/test/fts5ae.test 1142d16d9cc193894dc13cc8f9c7a8a21411ac61b5567a878514df6f9f0d7bb7 -F ext/fts5/test/fts5af.test aa635947bda31ac87fbe99483eef4d9a8571f58ad89c75dfb63312a35688eceb +F ext/fts5/test/fts5af.test 724247405b13f8f06cc6ce464dc4f152dc5dd4e86b12c2099685d8f19747bf7b F ext/fts5/test/fts5ag.test 7816f25a0707578f08145ab539fc0ca025f8951e788b28a6a18a06b2099469dd F ext/fts5/test/fts5ah.test 27b5a33bfd0363ca8a4dc659e6e2a5df3dea1c3c5b04bc51ca6aeb1277bd9b21 F ext/fts5/test/fts5ai.test d837c42249c0d8ad1a2912270e22cf2f303790a611f85c0be3a58e42a3696e3d @@ -1687,8 +1687,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 90d6e4f10d3055a776d24854c442a2a68e726af8fc382cdb6241a834082e4c4c -Q +b3c6105181d22c8a53f5bf662e3d182bba962ee8afd1ad972ebd10094b20fc17 -R c2447707ab323a591ab8e05be0f0cae9 +P ebfea8728fec955b1d74b1d0a3de498fd1a32e8b39572a8fdab606ed87b169b4 +Q +553a3ad32498ddda920216cd44a376a439a58fbb326d2d3800528867db1ffa9d +R 742f62796704fae3ae9dc15618c72290 U dan -Z 1002781326ca68c114cfc46396055176 +Z ecb93df9cf8c6f4bd1c8f7a6ca97d753 diff --git a/manifest.uuid b/manifest.uuid index aaa3417a9c..9c54144fee 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ebfea8728fec955b1d74b1d0a3de498fd1a32e8b39572a8fdab606ed87b169b4 \ No newline at end of file +6a790b67a0a5c698526db16ea262b13ecdd1b6ca74e80bdccfcad88ddbdc933a \ No newline at end of file From 02a43f6defda835b5baaaee274923f1b80d31ceb Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 26 Dec 2017 14:46:20 +0000 Subject: [PATCH 5/5] Faster and smaller implementation of sqlite3AtoF() based on a suggestion from Cezary H. Noweta. FossilOrigin-Name: fd2e0e7a770c2ce9355068aad1024c3d2861c104fd3be304a91c55ca742155fa --- manifest | 15 +++++++-------- manifest.uuid | 2 +- src/util.c | 26 ++++++++++++++++++++------ 3 files changed, 28 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index f3f08454e9..3d3fdb11e9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\scrashes\sthat\scould\soccur\sif\sSQL\sNULL\svalues\swere\spassed\sto\sthe\sbuilt-in\nFTS5\ssnippet\sfunction.\sEdit:\sbreaks\samalgamation\sbuilds. -D 2017-12-26T14:32:25.807 +C Faster\sand\ssmaller\simplementation\sof\ssqlite3AtoF()\sbased\son\sa\ssuggestion\nfrom\sCezary\sH.\sNoweta. +D 2017-12-26T14:46:20.800 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69 @@ -545,7 +545,7 @@ F src/treeview.c eae35972ff44f67064de2eaf35f04afe94e7aea3271a8b3bcebb3f954880fec F src/trigger.c 775053eecf6b73062e243404b56f5064446254d5cce17d8704d5cdffd72a546a F src/update.c 961bd1265d4d1e5cd65c9a54fa5122fb7aefcb003fcf2de0c092fceb7e58972c F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5 -F src/util.c d01fa6f45bfad3b65fb2490513aa2e0676412c61b4b094340b513cf72c3704a4 +F src/util.c 7315e97a8dc2c8e19ca64196c652cf0a65d13fd0a211b2cec082062372dc6261 F src/vacuum.c 90839322fd5f00df9617eb21b68beda9b6e2a2937576b0d65985e4aeb1c53739 F src/vdbe.c 3393b508d9ad084ffce232a7c53e375ef5ac99b50b685c5131fcdfce97a9d534 F src/vdbe.h d50cadf12bcf9fb99117ef392ce1ea283aa429270481426b6e8b0280c101fd97 @@ -1687,8 +1687,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ebfea8728fec955b1d74b1d0a3de498fd1a32e8b39572a8fdab606ed87b169b4 -Q +553a3ad32498ddda920216cd44a376a439a58fbb326d2d3800528867db1ffa9d -R 742f62796704fae3ae9dc15618c72290 -U dan -Z ecb93df9cf8c6f4bd1c8f7a6ca97d753 +P 6a790b67a0a5c698526db16ea262b13ecdd1b6ca74e80bdccfcad88ddbdc933a +R c3240df89b3aa2504611bdd745085460 +U drh +Z 3dc20c6e2c72c6078d443025934f152b diff --git a/manifest.uuid b/manifest.uuid index 9c54144fee..bdfe92339c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6a790b67a0a5c698526db16ea262b13ecdd1b6ca74e80bdccfcad88ddbdc933a \ No newline at end of file +fd2e0e7a770c2ce9355068aad1024c3d2861c104fd3be304a91c55ca742155fa \ No newline at end of file diff --git a/src/util.c b/src/util.c index a4dbe8fdaf..75de4b3b30 100644 --- a/src/util.c +++ b/src/util.c @@ -320,6 +320,24 @@ int sqlite3_strnicmp(const char *zLeft, const char *zRight, int N){ return N<0 ? 0 : UpperToLower[*a] - UpperToLower[*b]; } +/* +** Compute 10 to the E-th power. Examples: E==1 results in 10. +** E==2 results in 100. E==50 results in 1.0e50. +** +** This routine only works for values of E between 1 and 341. +*/ +static LONGDOUBLE_TYPE sqlite3Pow10(int E){ + LONGDOUBLE_TYPE x = 10.0; + LONGDOUBLE_TYPE r = 1.0; + while(1){ + if( E & 1 ) r *= x; + E >>= 1; + if( E==0 ) break; + x *= x; + } + return r; +} + /* ** The string z[] is an text representation of a real number. ** Convert this string to a double and write it into *pResult. @@ -475,11 +493,10 @@ do_atof_calc: if( e==0 ){ /*OPTIMIZATION-IF-TRUE*/ result = (double)s; }else{ - LONGDOUBLE_TYPE scale = 1.0; /* attempt to handle extremely small/large numbers better */ if( e>307 ){ /*OPTIMIZATION-IF-TRUE*/ if( e<342 ){ /*OPTIMIZATION-IF-TRUE*/ - while( e%308 ) { scale *= 1.0e+1; e -= 1; } + LONGDOUBLE_TYPE scale = sqlite3Pow10(e-308); if( esign<0 ){ result = s / scale; result /= 1.0e+308; @@ -499,10 +516,7 @@ do_atof_calc: } } }else{ - /* 1.0e+22 is the largest power of 10 than can be - ** represented exactly. */ - while( e%22 ) { scale *= 1.0e+1; e -= 1; } - while( e>0 ) { scale *= 1.0e+22; e -= 22; } + LONGDOUBLE_TYPE scale = sqlite3Pow10(e); if( esign<0 ){ result = s / scale; }else{