From 22cd1eec5ae516facb39a9ac3efbf6a28f170237 Mon Sep 17 00:00:00 2001 From: larrybr Date: Tue, 3 Sep 2024 02:09:13 +0000 Subject: [PATCH 01/36] Cause cfGets() (under SQLITE_USE_ONLY_WIN32) to better emulate fgets(). FossilOrigin-Name: 2d783524d1671d988ebb1b21c83de99c9f335963b6c20cf4df612f58c13da913 --- ext/consio/console_io.c | 9 ++++++++- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/ext/consio/console_io.c b/ext/consio/console_io.c index 6b837cbafc..75324a34fd 100755 --- a/ext/consio/console_io.c +++ b/ext/consio/console_io.c @@ -655,6 +655,10 @@ cfWrite(const void *buf, size_t osz, size_t ocnt, FILE *pf){ return rv; } +/* An fgets() equivalent, using Win32 file API for actual input. +** Input ends when given buffer is filled or a newline is read. +** If the FILE object is in text mode, swallows 0x0D. (ASCII CR) +*/ SQLITE_INTERNAL_LINKAGE char * cfGets(char *cBuf, int n, FILE *pf){ int nci = 0; @@ -665,7 +669,10 @@ cfGets(char *cBuf, int n, FILE *pf){ while( nci < n-1 ){ DWORD nr; if( !ReadFile(fai.fh, cBuf+nci, 1, &nr, 0) || nr==0 ) break; - if( nr>0 && (!eatCR || cBuf[nci]!='\r') ) nci += nr; + if( nr>0 && (!eatCR || cBuf[nci]!='\r') ){ + nci += nr; + if( cBuf[nci-nr]=='\n' ) break; + } } if( nci < n ) cBuf[nci] = 0; return (nci>0)? cBuf : 0; diff --git a/manifest b/manifest index ddb1f8bfb9..55e9207dbb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Only\savoid\sall\sC-runtime\sI/O\sif\sthe\sSQLITE_USE_ONLY_WIN32\sflag\sis\sdefined. -D 2024-08-27T14:25:52.592 +C Cause\scfGets()\s(under\sSQLITE_USE_ONLY_WIN32)\sto\sbetter\semulate\sfgets(). +D 2024-09-03T02:09:13.849 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -53,7 +53,7 @@ F ext/README.md fd5f78013b0a2bc6f0067afb19e6ad040e89a10179b4f6f03eee58fac5f169bd F ext/async/README.txt e12275968f6fde133a80e04387d0e839b0c51f91 F ext/async/sqlite3async.c 6f247666b495c477628dd19364d279c78ea48cd90c72d9f9b98ad1aff3294f94 F ext/async/sqlite3async.h 46b47c79357b97ad85d20d2795942c0020dc20c532114a49808287f04aa5309a -F ext/consio/console_io.c 6e02dea912a49f55785b0027fe77960aafee6c236307c23f82ec86a69f1a2001 x +F ext/consio/console_io.c d2b74afae8d301de2e8447b1045fcd33eb59df13bf581d906d99c74fe5d2b13f x F ext/consio/console_io.h b5ebe34aa15b357621ebbea3d3f2e2b24750d4280b5802516409e23947fd9ee5 F ext/expert/README.md b321c2762bb93c18ea102d5a5f7753a4b8bac646cb392b3b437f633caf2020c3 F ext/expert/expert.c d548d603a4cc9e61f446cc179c120c6713511c413f82a4a32b1e1e69d3f086a4 @@ -2210,8 +2210,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2d52db98f47fbcda0622c034b21c2fb19bf4345b88c5c565ae9e6f7128642e43 -R fe2336ed930d3088aea807bfa7c9e690 -U drh -Z dffceaa70fb90b81619d76d49bca0c7c +P 164b1e1962aa1e16bdf52e9e86d4cf9c9e09220c0821932ac8e390e82074185f +R 0ea7d1c943863a7b1da4b295016a8f50 +U larrybr +Z 8cf571eafc534d90679b77113d98b6b5 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 5430ab55f6..47b76c643e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -164b1e1962aa1e16bdf52e9e86d4cf9c9e09220c0821932ac8e390e82074185f +2d783524d1671d988ebb1b21c83de99c9f335963b6c20cf4df612f58c13da913 From e6e49bfef29b29e75fe86451b8c40a6ef114412f Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 3 Sep 2024 12:41:21 +0000 Subject: [PATCH 02/36] Fix harmless compiler warnings in the percentile extension. FossilOrigin-Name: c5557f281c6294b6db6682349d245feac7c6ce7f4f61356f486afdf186c566c4 --- ext/misc/percentile.c | 6 +++--- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ext/misc/percentile.c b/ext/misc/percentile.c index 1d69a49398..1c6191d42e 100644 --- a/ext/misc/percentile.c +++ b/ext/misc/percentile.c @@ -307,7 +307,7 @@ static void percentStep(sqlite3_context *pCtx, int argc, sqlite3_value **argv){ }else if( p->bKeepSorted ){ int i; i = percentBinarySearch(p, y, 0); - if( inUsed ){ + if( i<(int)p->nUsed ){ memmove(&p->a[i+1], &p->a[i], (p->nUsed-i)*sizeof(p->a[0])); } p->a[i] = y; @@ -423,7 +423,7 @@ static void percentInverse(sqlite3_context *pCtx,int argc,sqlite3_value **argv){ i = percentBinarySearch(p, y, 1); if( i>=0 ){ p->nUsed--; - if( inUsed ){ + if( i<(int)p->nUsed ){ memmove(&p->a[i], &p->a[i+1], (p->nUsed - i)*sizeof(p->a[0])); } } @@ -483,7 +483,7 @@ int sqlite3_percentile_init( const sqlite3_api_routines *pApi ){ int rc = SQLITE_OK; - int i; + unsigned int i; #if defined(SQLITE3_H) || defined(SQLITE_STATIC_PERCENTILE) (void)pApi; /* Unused parameter */ #else diff --git a/manifest b/manifest index bec335f8c6..9d2a9f8cea 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\stestrunner.tcl,\sallow\ssetting\snjob\sto\szero,\swhich\scauses\sno\snew\sjobs\nto\sbe\slaunched\sand\sfor\sthe\sprocess\sto\sshut\sdown\sonce\sall\scurrent\sjobs\sare\ncompleted. -D 2024-09-03T10:53:32.903 +C Fix\sharmless\scompiler\swarnings\sin\sthe\spercentile\sextension. +D 2024-09-03T12:41:21.618 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -411,7 +411,7 @@ F ext/misc/nextchar.c 7877914c2a80c2f181dd04c3dbef550dfb54c93495dc03da2403b5dd58 F ext/misc/noop.c f1a21cc9b7a4e667e5c8458d80ba680b8bd4315a003f256006046879f679c5a0 F ext/misc/normalize.c bd84355c118e297522aba74de34a4fd286fc775524e0499b14473918d09ea61f F ext/misc/pcachetrace.c f4227ce03fb16aa8d6f321b72dd051097419d7a028a9853af048bee7645cb405 -F ext/misc/percentile.c b37f01f559cec6ed3c4bcf5079e355cfe8a64eb32cc15c3b0f531980fd632b67 +F ext/misc/percentile.c 42eb041edab407e512aaa087f99ed9287fe0b3224f7a6d194456c00fc1454312 F ext/misc/prefixes.c 82645f79229877afab08c8b08ca1e7fa31921280906b90a61c294e4f540cd2a6 F ext/misc/qpvtab.c fc189e127f68f791af90a487f4460ec91539a716daf45a0c357e963fd47cc06c F ext/misc/randomjson.c ef835fc64289e76ac4873b85fe12f9463a036168d7683cf2b773e36e6262c4ed @@ -2212,8 +2212,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7891a266c4425722ae8b9231397ef9e42e2432be9e6b70632dfaf9ff15300d2c -R 6a86d12ea82ce01298969ad043e5cc65 +P 0ef65fd4ba17def4c13986365b3af300c4024725af4bc314845d1af8be568ab4 +R 2b10d334a3671a793c7868d6c8e12ad1 U drh -Z 5112afd2e4dab18f4b12bbba01825cee +Z 339ed9f3d932cc4c3af3add15bcda94b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 9843b7b310..d62eb99bea 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0ef65fd4ba17def4c13986365b3af300c4024725af4bc314845d1af8be568ab4 +c5557f281c6294b6db6682349d245feac7c6ce7f4f61356f486afdf186c566c4 From 23a16c8387870a86aef8dea7f8dbad506dd29d94 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 3 Sep 2024 14:00:36 +0000 Subject: [PATCH 03/36] In testrunner.tcl, compile the Windows Default configuration using SQLITE_ENABLE_STMT_SCANSTATUS. FossilOrigin-Name: 49e8b1635f29d9fd0dc2ef6e312fd4129f2283f68e9423d95ff9b55077688ad7 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/testrunner_data.tcl | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 9d2a9f8cea..404dfb05fd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarnings\sin\sthe\spercentile\sextension. -D 2024-09-03T12:41:21.618 +C In\stestrunner.tcl,\scompile\sthe\sWindows\sDefault\sconfiguration\susing\nSQLITE_ENABLE_STMT_SCANSTATUS. +D 2024-09-03T14:00:36.487 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1716,7 +1716,7 @@ F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc F test/tester.tcl 2c203a2dd664298f239f0ec3ce22fbc65b5f021c1e09edbae8452af8a694e052 F test/testrunner.tcl 92e3c63072362cd56d7dec2548284425104b5b3a5c0af3371ee2911712d89bf6 -F test/testrunner_data.tcl d64e69aba227492fc459eb0f6a88c3e1b252bd9323847b7e8bcf5e1faa358714 +F test/testrunner_data.tcl dbc0bb1c5b912dfd1e32b25d544318e412edd6085bd5fc9e6619cb93a739b786 F test/thread001.test a0985c117eab62c0c65526e9fa5d1360dd1cac5b03bde223902763274ce21899 F test/thread002.test c24c83408e35ba5a952a3638b7ac03ccdf1ce4409289c54a050ac4c5f1de7502 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7 @@ -2212,8 +2212,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0ef65fd4ba17def4c13986365b3af300c4024725af4bc314845d1af8be568ab4 -R 2b10d334a3671a793c7868d6c8e12ad1 +P c5557f281c6294b6db6682349d245feac7c6ce7f4f61356f486afdf186c566c4 +R af529c0653dd6eafafdc4a7dab6ec4ca U drh -Z 339ed9f3d932cc4c3af3add15bcda94b +Z 9034919d1245ea10e06eda748f2be5e1 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d62eb99bea..32840dde2a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c5557f281c6294b6db6682349d245feac7c6ce7f4f61356f486afdf186c566c4 +49e8b1635f29d9fd0dc2ef6e312fd4129f2283f68e9423d95ff9b55077688ad7 diff --git a/test/testrunner_data.tcl b/test/testrunner_data.tcl index f9a23063cb..78f28bbf8e 100644 --- a/test/testrunner_data.tcl +++ b/test/testrunner_data.tcl @@ -88,6 +88,7 @@ namespace eval trd { --disable-amalgamation --disable-shared --enable-session -DSQLITE_ENABLE_RBU + -DSQLITE_ENABLE_STMT_SCANSTATUS } # These two are used by [testrunner.tcl mdevtest] (All-O0) and From fd486258a0295689e04e12ada08f76a061a37deb Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 3 Sep 2024 16:04:34 +0000 Subject: [PATCH 04/36] Enhance the "errors" command in testrunner.tcl so that it accepts the "-s" or "--summary" argument to see a list of failed jobs, and so that an additional argument is a GLOB pattern that restricts the output to jobs whose names match that pattern. FossilOrigin-Name: dcbebe30f594a99e23b5ccd8d199b92118204a3e52e75c78d98c394601252e81 --- manifest | 13 ++++++------- manifest.uuid | 2 +- test/testrunner.tcl | 45 +++++++++++++++++++++++++++++++++++---------- 3 files changed, 42 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index 6f14866d00..3b79cd2f96 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sext/consio\sso\sthat\sit\sworks\scorrectly\swith\sSQLITE_USE_ONLY_WIN32. -D 2024-09-03T14:15:57.425 +C Enhance\sthe\s"errors"\scommand\sin\stestrunner.tcl\sso\sthat\sit\saccepts\sthe\s\n"-s"\sor\s"--summary"\sargument\sto\ssee\sa\slist\sof\sfailed\sjobs,\sand\sso\sthat\nan\sadditional\sargument\sis\sa\sGLOB\spattern\sthat\srestricts\sthe\soutput\sto\njobs\swhose\snames\smatch\sthat\spattern. +D 2024-09-03T16:04:34.164 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1715,7 +1715,7 @@ F test/temptable2.test 76821347810ecc88203e6ef0dd6897b6036ac788e9dd3e6b04fd4d163 F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc F test/tester.tcl 2c203a2dd664298f239f0ec3ce22fbc65b5f021c1e09edbae8452af8a694e052 -F test/testrunner.tcl 92e3c63072362cd56d7dec2548284425104b5b3a5c0af3371ee2911712d89bf6 +F test/testrunner.tcl d117492b6ec7293f7906dd4ff48135dd9801bdf42db0f48f88b3328867aa94ff F test/testrunner_data.tcl dbc0bb1c5b912dfd1e32b25d544318e412edd6085bd5fc9e6619cb93a739b786 F test/thread001.test a0985c117eab62c0c65526e9fa5d1360dd1cac5b03bde223902763274ce21899 F test/thread002.test c24c83408e35ba5a952a3638b7ac03ccdf1ce4409289c54a050ac4c5f1de7502 @@ -2212,9 +2212,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 49e8b1635f29d9fd0dc2ef6e312fd4129f2283f68e9423d95ff9b55077688ad7 2d783524d1671d988ebb1b21c83de99c9f335963b6c20cf4df612f58c13da913 -R 1f005fca5fe5df8e4ebc8001e171645f -T +closed 2d783524d1671d988ebb1b21c83de99c9f335963b6c20cf4df612f58c13da913 +P efc6f3d7e92a25f440fb8d392daf325af5ca7dca104a5bb4bd59f7df93af53b0 +R 1cc8a7ec499a68123307944e86d4ccaa U drh -Z 207a73144d24fc75006dde9e7a085311 +Z 54536ba1c60ddf4f4f932c617d6c5425 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index f51ba4769e..06dec1c0fe 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -efc6f3d7e92a25f440fb8d392daf325af5ca7dca104a5bb4bd59f7df93af53b0 +dcbebe30f594a99e23b5ccd8d199b92118204a3e52e75c78d98c394601252e81 diff --git a/test/testrunner.tcl b/test/testrunner.tcl index 547dc08985..d265233f00 100644 --- a/test/testrunner.tcl +++ b/test/testrunner.tcl @@ -54,7 +54,7 @@ proc usage {} { Usage: $a0 ?SWITCHES? ?PERMUTATION? ?PATTERNS? $a0 PERMUTATION FILE - $a0 errors ?-v|--verbose? + $a0 errors ?-v|--verbose? ?-s|--summary? ?PATTERN? $a0 help $a0 njob ?NJOB? $a0 script ?-msvc? CONFIG @@ -112,10 +112,12 @@ The "script" command outputs the script used to build a configuration. Add the "-msvc" option for a Windows-compatible script. For a list of available configurations enter "$a0 script help". -The "errors" commands shows the output of all tests that failed in the +The "errors" commands shows the output of tests that failed in the most recent run. Complete output is shown if the -v or --verbose options are used. Otherwise, an attempt is made to minimize the output to show -only the parts that contain the error messages. +only the parts that contain the error messages. The --summary option just +shows the jobs that failed. If PATTERN are provided, the error information +is only provided for jobs that match PATTERN. Full documentation here: https://sqlite.org/src/doc/trunk/doc/testrunner.md }]] @@ -606,15 +608,21 @@ proc aggregate_test_counts {db} { #-------------------------------------------------------------------------- # Check if this is the "errors" command: # -if {[llength $argv]>=1 && [llength $argv]<=2 +if {[llength $argv]>=1 && ([string compare -nocase errors [lindex $argv 0]]==0 || [string match err* [lindex $argv 0]]==1) } { set verbose 0 + set pattern {} + set summary 0 for {set ii 1} {$ii<[llength $argv]} {incr ii} { set a0 [lindex $argv $ii] if {$a0=="-v" || $a0=="--verbose" || $a0=="-verbose"} { set verbose 1 + } elseif {$a0=="-s" || $a0=="--summary" || $a0=="-summary"} { + set summary 1 + } elseif {$pattern==""} { + set pattern *[string trim $a0 *]* } else { puts "unknown option: \"$a0\"". Use --help for more info." exit 1 @@ -622,9 +630,22 @@ if {[llength $argv]>=1 && [llength $argv]<=2 } set cnt 0 sqlite3 mydb $TRG(dbname) - mydb timeout 2000 - mydb eval {SELECT displaytype, displayname, output - FROM jobs WHERE state='failed'} { + mydb timeout 5000 + if {$summary} { + set sql "SELECT displayname FROM jobs WHERE state='failed'" + } else { + set sql "SELECT displaytype, displayname, output FROM jobs \ + WHERE state='failed'" + } + if {$pattern!=""} { + regsub -all {[^a-zA-Z0-9*/ ?]} $pattern . pattern + append sql " AND displayname GLOB '$pattern'" + } + mydb eval $sql { + if {$summary} { + puts "FAILED: $displayname" + continue + } puts "**** $displayname ****" if {$verbose || $displaytype!="tcl"} { puts $output @@ -637,9 +658,13 @@ if {[llength $argv]>=1 && [llength $argv]<=2 } incr cnt } - set summary [aggregate_test_counts mydb] - mydb close - puts "Total [lindex $summary 0] errors out of [lindex $summary 1] tests" + if {$pattern==""} { + set summary [aggregate_test_counts mydb] + mydb close + puts "Total [lindex $summary 0] errors out of [lindex $summary 1] tests" + } else { + mydb close + } exit } From c81ab76cd9f2ec77cb9e1219b504d741845a0703 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 4 Sep 2024 11:22:47 +0000 Subject: [PATCH 05/36] Add the "joblist" command to testrunner.tcl FossilOrigin-Name: f64469f4806d4d5d7103c171a37a542c7aab9db09a226bccd8411e9ccd55353d --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/testrunner.tcl | 46 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 3b79cd2f96..ffd3047098 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sthe\s"errors"\scommand\sin\stestrunner.tcl\sso\sthat\sit\saccepts\sthe\s\n"-s"\sor\s"--summary"\sargument\sto\ssee\sa\slist\sof\sfailed\sjobs,\sand\sso\sthat\nan\sadditional\sargument\sis\sa\sGLOB\spattern\sthat\srestricts\sthe\soutput\sto\njobs\swhose\snames\smatch\sthat\spattern. -D 2024-09-03T16:04:34.164 +C Add\sthe\s"joblist"\scommand\sto\stestrunner.tcl +D 2024-09-04T11:22:47.766 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1715,7 +1715,7 @@ F test/temptable2.test 76821347810ecc88203e6ef0dd6897b6036ac788e9dd3e6b04fd4d163 F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc F test/tester.tcl 2c203a2dd664298f239f0ec3ce22fbc65b5f021c1e09edbae8452af8a694e052 -F test/testrunner.tcl d117492b6ec7293f7906dd4ff48135dd9801bdf42db0f48f88b3328867aa94ff +F test/testrunner.tcl 34652534e0085d9578a8b6fe86e93c387bebbcd321dde4aecadc4cf0c8b220ee F test/testrunner_data.tcl dbc0bb1c5b912dfd1e32b25d544318e412edd6085bd5fc9e6619cb93a739b786 F test/thread001.test a0985c117eab62c0c65526e9fa5d1360dd1cac5b03bde223902763274ce21899 F test/thread002.test c24c83408e35ba5a952a3638b7ac03ccdf1ce4409289c54a050ac4c5f1de7502 @@ -2212,8 +2212,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P efc6f3d7e92a25f440fb8d392daf325af5ca7dca104a5bb4bd59f7df93af53b0 -R 1cc8a7ec499a68123307944e86d4ccaa +P dcbebe30f594a99e23b5ccd8d199b92118204a3e52e75c78d98c394601252e81 +R 526b505ed9ff61539ab1544a16cc762d U drh -Z 54536ba1c60ddf4f4f932c617d6c5425 +Z ec42621d2cdae6fdf4751b132650e441 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 06dec1c0fe..199d71e02d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dcbebe30f594a99e23b5ccd8d199b92118204a3e52e75c78d98c394601252e81 +f64469f4806d4d5d7103c171a37a542c7aab9db09a226bccd8411e9ccd55353d diff --git a/test/testrunner.tcl b/test/testrunner.tcl index d265233f00..11c4bbcd1f 100644 --- a/test/testrunner.tcl +++ b/test/testrunner.tcl @@ -56,6 +56,7 @@ Usage: $a0 PERMUTATION FILE $a0 errors ?-v|--verbose? ?-s|--summary? ?PATTERN? $a0 help + $a0 joblist ?PATTERN? $a0 njob ?NJOB? $a0 script ?-msvc? CONFIG $a0 status ?-d SECS? ?--cls? @@ -579,6 +580,51 @@ if {[llength $argv]>=1 exit } +#-------------------------------------------------------------------------- +# Check if this is the "joblist" command: +# +if {[llength $argv]>=1 + && [string compare -nocase "joblist" [lindex $argv 0]]==0 +} { + set pattern {} + for {set ii 1} {$ii<[llength $argv]} {incr ii} { + set a0 [lindex $argv $ii] + if {$pattern==""} { + set pattern [string trim $a0 *] + } else { + puts "unknown option: \"$a0\"" + exit 1 + } + } + set SQL {SELECT displaytype, displayname, state FROM jobs} + if {$pattern!=""} { + regsub -all {[^a-zA-Z0-9*.-/]} $pattern ? pattern + append SQL " WHERE displayname GLOB '*$pattern*'" + } + append SQL " ORDER BY starttime" + + if {![file readable $TRG(dbname)]} { + puts "Database missing: $TRG(dbname)" + exit + } + sqlite3 mydb $TRG(dbname) + mydb timeout 2000 + + mydb eval $SQL { + set label UNKNOWN + switch -- $state { + ready {set label READY} + done {set label DONE} + failed {set label FAILED} + omit {set label OMIT} + running {set label RUNNING} + } + puts [format {%-7s %-5s %s} $label $displaytype $displayname] + } + mydb close + exit +} + # Scan the output of all jobs looking for the summary lines that # report the number of test cases and the number of errors. # Aggregate these numbers and return them. From 3c72072a6f5e1145573f08c788f5f57a5fd2544e Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 4 Sep 2024 13:09:58 +0000 Subject: [PATCH 06/36] Make extra efforts to delete the over-size directory names created by the win32longpath.test module. FossilOrigin-Name: 3c8035ed3f023fb31f65f298fb9ce19d4fa7a5511bd90877b27969f5f0892472 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/win32longpath.test | 11 +++++++++++ 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index ffd3047098..049ece78b5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s"joblist"\scommand\sto\stestrunner.tcl -D 2024-09-04T11:22:47.766 +C Make\sextra\sefforts\sto\sdelete\sthe\sover-size\sdirectory\snames\screated\sby\nthe\swin32longpath.test\smodule. +D 2024-09-04T13:09:58.914 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -2051,7 +2051,7 @@ F test/wherelimit3.test 22d73e046870cf8bbe15573eda6b432b07ebe64a88711f9f849c6b36 F test/widetab1.test c296a98e123762de79917350e45fa33fdf88577a2571eb3a64c8bf7e44ef74d1 F test/win32heap.test 10fd891266bd00af68671e702317726375e5407561d859be1aa04696f2aeee74 F test/win32lock.test e0924eb8daac02bf80e9da88930747bd44dd9b230b7759fed927b1655b467c9c -F test/win32longpath.test 42210789bcfc5c0ac202643d6d0237db08df2c9218f2070d9a69e8af1eccf7d7 +F test/win32longpath.test 304006024ca47104bf5a7415ef31ca83ecfc29351af202baf8588b880cffc116 F test/win32nolock.test ac4f08811a562e45a5755e661f45ca85892bdbbc F test/window1.test 79dc3b9a2226f622d7e104a1fc750d1c4c3c08d6147b59085bdbe05352947ffa F test/window2.tcl 492c125fa550cda1dd3555768a2303b3effbeceee215293adf8871efc25f1476 @@ -2212,8 +2212,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P dcbebe30f594a99e23b5ccd8d199b92118204a3e52e75c78d98c394601252e81 -R 526b505ed9ff61539ab1544a16cc762d +P f64469f4806d4d5d7103c171a37a542c7aab9db09a226bccd8411e9ccd55353d +R b8f539347cbc9843b6ed8aa52a0e1bab U drh -Z ec42621d2cdae6fdf4751b132650e441 +Z f80762a3e5420a06ffd6178d73ee51c7 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 199d71e02d..7f556a75fd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f64469f4806d4d5d7103c171a37a542c7aab9db09a226bccd8411e9ccd55353d +3c8035ed3f023fb31f65f298fb9ce19d4fa7a5511bd90877b27969f5f0892472 diff --git a/test/win32longpath.test b/test/win32longpath.test index c0eac8d9de..9ffea775e7 100644 --- a/test/win32longpath.test +++ b/test/win32longpath.test @@ -127,6 +127,17 @@ foreach tn {1a 1b 1c 1d 1e 1f} { db3 close } +# These over-length file and directory names are difficult to delete. +# The "file delete -force" might not work, depending on the TCL build +# being used. So first try to delete using the windows rmdir command. +# +set fd [open cleanup.bat w] +puts $fd "rmdir /q /s $longPath(1)" +close $fd +if {[catch {exec cleanup.bat} msg]} { + puts "Command \[cleanup.bat\] returns $msg" +} + file delete -force $fileName file delete -force $longPath(3) file delete -force $longPath(2) From 109025c0a4a97d0fb48faf6e2d698955ab7f8e06 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 4 Sep 2024 13:42:19 +0000 Subject: [PATCH 07/36] Put a proper shebang at the start of the testrunner.tcl script and make the script executable. FossilOrigin-Name: c0c6e9abebf76358625f30a179658319b260baba6eded2a4c5ad356143e36f97 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/testrunner.tcl | 3 +++ 3 files changed, 10 insertions(+), 7 deletions(-) mode change 100644 => 100755 test/testrunner.tcl diff --git a/manifest b/manifest index 049ece78b5..8ea6cf8851 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\sextra\sefforts\sto\sdelete\sthe\sover-size\sdirectory\snames\screated\sby\nthe\swin32longpath.test\smodule. -D 2024-09-04T13:09:58.914 +C Put\sa\sproper\sshebang\sat\sthe\sstart\sof\sthe\stestrunner.tcl\sscript\sand\smake\sthe\nscript\sexecutable. +D 2024-09-04T13:42:19.755 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1715,7 +1715,7 @@ F test/temptable2.test 76821347810ecc88203e6ef0dd6897b6036ac788e9dd3e6b04fd4d163 F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc F test/tester.tcl 2c203a2dd664298f239f0ec3ce22fbc65b5f021c1e09edbae8452af8a694e052 -F test/testrunner.tcl 34652534e0085d9578a8b6fe86e93c387bebbcd321dde4aecadc4cf0c8b220ee +F test/testrunner.tcl 982939f0f1835007298b92e52694c207d16ef79143993b35e5cbc9f0c585938b x F test/testrunner_data.tcl dbc0bb1c5b912dfd1e32b25d544318e412edd6085bd5fc9e6619cb93a739b786 F test/thread001.test a0985c117eab62c0c65526e9fa5d1360dd1cac5b03bde223902763274ce21899 F test/thread002.test c24c83408e35ba5a952a3638b7ac03ccdf1ce4409289c54a050ac4c5f1de7502 @@ -2212,8 +2212,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f64469f4806d4d5d7103c171a37a542c7aab9db09a226bccd8411e9ccd55353d -R b8f539347cbc9843b6ed8aa52a0e1bab +P 3c8035ed3f023fb31f65f298fb9ce19d4fa7a5511bd90877b27969f5f0892472 +R f05e13077e7a870b4aeeb1b6e8984529 U drh -Z f80762a3e5420a06ffd6178d73ee51c7 +Z 24d0fb17858b3b348074c639926c08cb # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 7f556a75fd..fd22bb9d40 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3c8035ed3f023fb31f65f298fb9ce19d4fa7a5511bd90877b27969f5f0892472 +c0c6e9abebf76358625f30a179658319b260baba6eded2a4c5ad356143e36f97 diff --git a/test/testrunner.tcl b/test/testrunner.tcl old mode 100644 new mode 100755 index 11c4bbcd1f..a63a22d2fc --- a/test/testrunner.tcl +++ b/test/testrunner.tcl @@ -1,3 +1,6 @@ +#!/bin/sh +# Script to runs tests for SQLite. Run with option "help" for more info. \ +exec tclsh "$0" "$@" set dir [pwd] set testdir [file normalize [file dirname $argv0]] From bb4d2edf109be3f978807dd6f3a9d80624da1029 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 4 Sep 2024 16:01:44 +0000 Subject: [PATCH 08/36] Fix a bug in the parsing of some corner-case JSON PATH strings that contain escaped double-quotes. FossilOrigin-Name: 60ac55c4b76355aaf7cbde38bf1f6082ff5612bf4ffc49ab69d00fd4e3d64e64 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/json.c | 4 +++- test/json502.test | 13 +++++++++++++ 4 files changed, 24 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 8ea6cf8851..eb0b2807fd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Put\sa\sproper\sshebang\sat\sthe\sstart\sof\sthe\stestrunner.tcl\sscript\sand\smake\sthe\nscript\sexecutable. -D 2024-09-04T13:42:19.755 +C Fix\sa\sbug\sin\sthe\sparsing\sof\ssome\scorner-case\sJSON\sPATH\sstrings\sthat\scontain\nescaped\sdouble-quotes. +D 2024-09-04T16:01:44.145 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -726,7 +726,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c f8d1a0f8ee258411009c6b7f2d93170e351bd19f5ad89d57e1180644297cbe70 -F src/json.c 5b6a1d6015997b9ee848a32948720bdb26a0ef2de5a2127ebf7355ce66dbdc0d +F src/json.c 68a98c020c22127f2d65f08855f7fc7460ff352a6ce0b543d8931dde83319c22 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c e7b53893f9fb3ad76baa8513f85c167b34d5c8e25ce64608db440f5637d0fe9e @@ -1381,7 +1381,7 @@ F test/json106.test 4aed3afd16549045d198a8d9cea00deea96e1f2ecf55864dce96cac558b8 F test/json107.test 59054e815c8f6b67d634d44ace421cf975828fb5651c4460aa66015c8e19d562 F test/json108.test 0a5f1e2d4b35a1bc33052563d2a5ede03052e2099e58cb424547656c898e0f49 F test/json501.test b95e2d14988b682a5cadf079dd6162f0f85fb74cd59c6b1f1624110104a974eb -F test/json502.test 84634d3dbb521d2814e43624025b760c6198456c8197bbec6c977c0236648f5b +F test/json502.test 4ef68e4f272dfb083d4cbceb4e9e51d67ec1186a185e0c13637c50a4dc2f9796 F test/jsonb01.test f4cdfb4cf5a0c940091b17675ed9583f45add0c938f07d65b0de0e19d3a9a101 F test/keyword1.test 37ef6bba5d2ed5b07ecdd6810571de2956599dff F test/kvtest.c 6e0228409ea7ca0497dad503fbd109badb5e59545d131014b6aaac68b56f484a @@ -2212,8 +2212,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3c8035ed3f023fb31f65f298fb9ce19d4fa7a5511bd90877b27969f5f0892472 -R f05e13077e7a870b4aeeb1b6e8984529 +P c0c6e9abebf76358625f30a179658319b260baba6eded2a4c5ad356143e36f97 +R c9ec90e0d0ba3db3bca78a08a6c286b8 U drh -Z 24d0fb17858b3b348074c639926c08cb +Z c8dd2f7c1df9f10ff01ca20270187b6f # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index fd22bb9d40..70820e2597 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c0c6e9abebf76358625f30a179658319b260baba6eded2a4c5ad356143e36f97 +60ac55c4b76355aaf7cbde38bf1f6082ff5612bf4ffc49ab69d00fd4e3d64e64 diff --git a/src/json.c b/src/json.c index 7361c46d89..a0a075e66c 100644 --- a/src/json.c +++ b/src/json.c @@ -2847,7 +2847,9 @@ static u32 jsonLookupStep( zPath++; if( zPath[0]=='"' ){ zKey = zPath + 1; - for(i=1; zPath[i] && zPath[i]!='"'; i++){} + for(i=1; zPath[i] && zPath[i]!='"'; i++){ + if( zPath[i]=='\\' && zPath[i+1]!=0 ) i++; + } nKey = i-1; if( zPath[i] ){ i++; diff --git a/test/json502.test b/test/json502.test index cc14b8cb71..1eba00dba5 100644 --- a/test/json502.test +++ b/test/json502.test @@ -64,4 +64,17 @@ ifcapable vtab { } {{\x17} 1 integer 1 1 null {$."\x17"} {$}} } +# JSON PATH parsing bug involving backslash escapes, reported via +# private email from Florent De'Neve on 2024-09-04. +# +do_execsql_test 5.1 { + SELECT json_extract('{"A\"Key":1}', '$.A"Key'); +} 1 +do_execsql_test 5.2 { + SELECT json_extract('{"A\"Key":1}', '$."A\"Key"'); +} 1 +do_execsql_test 5.3 { + SELECT JSON_SET('{}', '$."\"Key"', 1); +} {{{"\"Key":1}}} + finish_test From 7bd230cf4da87afa6adb44a53fb29845e3f7e5fc Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 4 Sep 2024 18:51:59 +0000 Subject: [PATCH 09/36] Make the default TCLDIR be C:\Tcl in the Makefile.msc. FossilOrigin-Name: c2c0a9176ae7291a0b72f3d048a2ef5716c325b9f693cdc5eada552e0c881a9d --- Makefile.msc | 2 +- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Makefile.msc b/Makefile.msc index fde1f31a9f..36a0ed3abe 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -938,7 +938,7 @@ TCLSUFFIX = !ENDIF !IFNDEF TCLDIR -TCLDIR = $(TOP)\compat\tcl +TCLDIR = C:\Tcl !ENDIF !IFNDEF TCLINCDIR diff --git a/manifest b/manifest index eb0b2807fd..8775a8ed5f 100644 --- a/manifest +++ b/manifest @@ -1,11 +1,11 @@ -C Fix\sa\sbug\sin\sthe\sparsing\sof\ssome\scorner-case\sJSON\sPATH\sstrings\sthat\scontain\nescaped\sdouble-quotes. -D 2024-09-04T16:01:44.145 +C Make\sthe\sdefault\sTCLDIR\sbe\sC:\\Tcl\sin\sthe\sMakefile.msc. +D 2024-09-04T18:51:59.227 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 F Makefile.in af5fbc3453b745daa68c7aa5dfdb945c09cb724971db3b783d6b5e1a62279e28 F Makefile.linux-gcc f3842a0b1efbfbb74ac0ef60e56b301836d05b4d867d014f714fa750048f1ab6 -F Makefile.msc 2c905f4c795a628d7fd892294e8fdec6d5f719fa5b252cb839fed147e64435a0 +F Makefile.msc e72a4653ea8f48efd14c7dac1e7de308a1f3cc91f905f15cbcacc680301c8f03 F README.md 5b678e264236788390d11991f2c0052bd73f19790173883fc56d638bcb849154 F VERSION 0db40f92c04378404eb45bff93e9e42c148c7e54fd3da99469ed21e22411f5a6 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -2212,8 +2212,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c0c6e9abebf76358625f30a179658319b260baba6eded2a4c5ad356143e36f97 -R c9ec90e0d0ba3db3bca78a08a6c286b8 +P 60ac55c4b76355aaf7cbde38bf1f6082ff5612bf4ffc49ab69d00fd4e3d64e64 +R 7c445598106af51bfd865380701cef37 U drh -Z c8dd2f7c1df9f10ff01ca20270187b6f +Z 345e63504eb5893ea2a5338064417dd5 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 70820e2597..2340dd1354 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -60ac55c4b76355aaf7cbde38bf1f6082ff5612bf4ffc49ab69d00fd4e3d64e64 +c2c0a9176ae7291a0b72f3d048a2ef5716c325b9f693cdc5eada552e0c881a9d From 6aedd6d9fce717e366af1c10c801eb5b6afec2df Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 5 Sep 2024 11:46:43 +0000 Subject: [PATCH 10/36] Faster implementation of the aggregate_test_counts procedure inside of testrunner.tcl. FossilOrigin-Name: a01d869520329fb9e786cdc65f359785a95d19d289e4c6b844c758d6e5385aaf --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/testrunner.tcl | 23 ++++------------------- 3 files changed, 11 insertions(+), 26 deletions(-) diff --git a/manifest b/manifest index 8775a8ed5f..106a4d98ac 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\sthe\sdefault\sTCLDIR\sbe\sC:\\Tcl\sin\sthe\sMakefile.msc. -D 2024-09-04T18:51:59.227 +C Faster\simplementation\sof\sthe\saggregate_test_counts\sprocedure\sinside\sof\ntestrunner.tcl. +D 2024-09-05T11:46:43.981 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1715,7 +1715,7 @@ F test/temptable2.test 76821347810ecc88203e6ef0dd6897b6036ac788e9dd3e6b04fd4d163 F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc F test/tester.tcl 2c203a2dd664298f239f0ec3ce22fbc65b5f021c1e09edbae8452af8a694e052 -F test/testrunner.tcl 982939f0f1835007298b92e52694c207d16ef79143993b35e5cbc9f0c585938b x +F test/testrunner.tcl 68b23b52e9ba45eb294b2597c27c383d12129afc06e33c8771313c7238bd1c44 x F test/testrunner_data.tcl dbc0bb1c5b912dfd1e32b25d544318e412edd6085bd5fc9e6619cb93a739b786 F test/thread001.test a0985c117eab62c0c65526e9fa5d1360dd1cac5b03bde223902763274ce21899 F test/thread002.test c24c83408e35ba5a952a3638b7ac03ccdf1ce4409289c54a050ac4c5f1de7502 @@ -2212,8 +2212,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 60ac55c4b76355aaf7cbde38bf1f6082ff5612bf4ffc49ab69d00fd4e3d64e64 -R 7c445598106af51bfd865380701cef37 +P c2c0a9176ae7291a0b72f3d048a2ef5716c325b9f693cdc5eada552e0c881a9d +R f43ab99ba85b48d85b41205bb3574221 U drh -Z 345e63504eb5893ea2a5338064417dd5 +Z ff8ebe5698dd3ab04755716aae5f59e1 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 2340dd1354..7f61e5a5d3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c2c0a9176ae7291a0b72f3d048a2ef5716c325b9f693cdc5eada552e0c881a9d +a01d869520329fb9e786cdc65f359785a95d19d289e4c6b844c758d6e5385aaf diff --git a/test/testrunner.tcl b/test/testrunner.tcl index a63a22d2fc..ccf888b572 100755 --- a/test/testrunner.tcl +++ b/test/testrunner.tcl @@ -633,25 +633,10 @@ if {[llength $argv]>=1 # Aggregate these numbers and return them. # proc aggregate_test_counts {db} { - set ncase 0 - set nerr 0 - $db eval {SELECT output FROM jobs WHERE displaytype IN ('tcl','fuzz')} { - set n 0 - set m 0 - if {[regexp {(\d+) errors out of (\d+) tests} $output all n m] - && [string is integer -strict $n] - && [string is integer -strict $m]} { - incr ncase $m - incr nerr $n - } elseif {[regexp {sessionfuzz.*: *(\d+) cases, (\d+) crash} $output \ - all m n] - && [string is integer -strict $m] - && [string is integer -strict $n]} { - incr ncase $m - incr nerr $n - } - } - return [list $nerr $ncase] + set ne 0 + set nt 0 + $db eval {SELECT sum(nerr) AS ne, sum(ntest) as nt FROM jobs} break + return [list $ne $nt] } #-------------------------------------------------------------------------- From 21e7816e651eb0e4fc67283b7acbc540c2a96cae Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 5 Sep 2024 12:06:45 +0000 Subject: [PATCH 11/36] Enhance testrunner.tcl to keep track of the platform and SQLite version as reported by individual test cases and to report that information in the summary at the end of each test run. FossilOrigin-Name: 80ebb7c7e686cd936ac834f2258f585a7004762593e0bc859ecd75d6fb0badfd --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/testrunner.tcl | 19 +++++++++++++++++-- 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 106a4d98ac..0d01cba7f0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Faster\simplementation\sof\sthe\saggregate_test_counts\sprocedure\sinside\sof\ntestrunner.tcl. -D 2024-09-05T11:46:43.981 +C Enhance\stestrunner.tcl\sto\skeep\strack\sof\sthe\splatform\sand\sSQLite\sversion\nas\sreported\sby\sindividual\stest\scases\sand\sto\sreport\sthat\sinformation\sin\sthe\nsummary\sat\sthe\send\sof\seach\stest\srun. +D 2024-09-05T12:06:45.482 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1715,7 +1715,7 @@ F test/temptable2.test 76821347810ecc88203e6ef0dd6897b6036ac788e9dd3e6b04fd4d163 F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc F test/tester.tcl 2c203a2dd664298f239f0ec3ce22fbc65b5f021c1e09edbae8452af8a694e052 -F test/testrunner.tcl 68b23b52e9ba45eb294b2597c27c383d12129afc06e33c8771313c7238bd1c44 x +F test/testrunner.tcl 1ebde7e8b57c464b9eb6ce047977c1edb46688b2ab94cc67e3cd5e4a3645ff00 x F test/testrunner_data.tcl dbc0bb1c5b912dfd1e32b25d544318e412edd6085bd5fc9e6619cb93a739b786 F test/thread001.test a0985c117eab62c0c65526e9fa5d1360dd1cac5b03bde223902763274ce21899 F test/thread002.test c24c83408e35ba5a952a3638b7ac03ccdf1ce4409289c54a050ac4c5f1de7502 @@ -2212,8 +2212,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c2c0a9176ae7291a0b72f3d048a2ef5716c325b9f693cdc5eada552e0c881a9d -R f43ab99ba85b48d85b41205bb3574221 +P a01d869520329fb9e786cdc65f359785a95d19d289e4c6b844c758d6e5385aaf +R a6deed6e9bd78f423240221d74d8d7b3 U drh -Z ff8ebe5698dd3ab04755716aae5f59e1 +Z 8cf0ea19b5a0efdaca6c57466a00e96c # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 7f61e5a5d3..9f0908d687 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a01d869520329fb9e786cdc65f359785a95d19d289e4c6b844c758d6e5385aaf +80ebb7c7e686cd936ac834f2258f585a7004762593e0bc859ecd75d6fb0badfd diff --git a/test/testrunner.tcl b/test/testrunner.tcl index ccf888b572..cd9d0daff5 100755 --- a/test/testrunner.tcl +++ b/test/testrunner.tcl @@ -290,6 +290,8 @@ set TRG(schema) { state TEXT CHECK( state IN ('','ready','running','done','failed','omit') ), ntest INT, -- Number of test cases run nerr INT, -- Number of errors reported + svers TEXT, -- Reported SQLite version + pltfm TEXT, -- Host platform reported output TEXT -- test output ); @@ -1255,6 +1257,10 @@ proc mark_job_as_finished {jobid output state endtm} { set nerr $a set ntest $b } + regexp {\y\d+ errors out of \d+ tests (on [^\n]+-bit \S+-endian)} \ + $output all pltfm + regexp {\ySQLite \d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d [0-9a-fA-F]+} \ + $output svers } r_write_db { if {$state=="failed"} { @@ -1266,7 +1272,7 @@ proc mark_job_as_finished {jobid output state endtm} { trdb eval { UPDATE jobs SET output=$output, state=$state, endtime=$endtm, - ntest=$ntest, nerr=$nerr + ntest=$ntest, nerr=$nerr, svers=$svers, pltfm=$pltfm WHERE jobid=$jobid; UPDATE jobs SET state=$childstate WHERE depid=$jobid; } @@ -1519,7 +1525,16 @@ proc run_testset {} { FROM jobs WHERE endtime>0 } break; set et [elapsetime $totaltime] - puts "$totalerr errors out of $totaltest tests in about $et" + set pltfm {} + trdb eval { + SELECT pltfm, count(*) FROM jobs WHERE pltfm IS NOT NULL + ORDER BY 2 DESC LIMIT 1 + } {puts $pltfm} + puts "$totalerr errors out of $totaltest tests in about $et $pltfm" + trdb eval { + SELECT DISTINCT svers FROM jobs WHERE svers IS NOT NULL + } {puts $svers} + } # Handle the --buildonly option, if it was specified. From 4987c5c3c24755bd2b19c4bd0fa11dc3b04d260b Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 5 Sep 2024 15:28:15 +0000 Subject: [PATCH 12/36] Testrunner.tcl enhancements: (1) Attempt to build the SQLite tcl extension if it is not already available. (2) testrunner target "devtest" is added as an alias for "mdevtest". (3) Try to keep summary information at the end of a test below 80-characters per line. (4) Update the Makefile.in so that the "clean" target removes the tcl extension built by item 1 above. FossilOrigin-Name: aa5f10f21dbfb24ee54ca96bfb7b013ae29e26fec05b80681f19cc63d9face49 --- Makefile.in | 2 +- manifest | 16 ++++++++-------- manifest.uuid | 2 +- test/tester.tcl | 9 +++++++-- test/testrunner.tcl | 37 +++++++++++++++++++++++++++++++++---- 5 files changed, 50 insertions(+), 16 deletions(-) diff --git a/Makefile.in b/Makefile.in index 2377146417..cc75000033 100644 --- a/Makefile.in +++ b/Makefile.in @@ -1626,7 +1626,7 @@ tidy: rm -f LogEst$(TEXE) fts3view$(TEXE) rollback-test$(TEXE) showdb$(TEXE) rm -f showjournal$(TEXE) showstat4$(TEXE) showwal$(TEXE) speedtest1$(TEXE) rm -f wordcount$(TEXE) changeset$(TEXE) version-info$(TEXE) - rm -f *.dll *.lib *.exp *.def *.pc *.vsix + rm -f *.dll *.lib *.exp *.def *.pc *.vsix *.so *.dylib pkgIndex.tcl rm -f sqlite3_analyzer$(TEXE) rm -f mptester$(TEXE) rbu$(TEXE) srcck1$(TEXE) rm -f fuzzershell$(TEXE) fuzzcheck$(TEXE) sqldiff$(TEXE) dbhash$(TEXE) diff --git a/manifest b/manifest index 0d01cba7f0..2cb670f56a 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Enhance\stestrunner.tcl\sto\skeep\strack\sof\sthe\splatform\sand\sSQLite\sversion\nas\sreported\sby\sindividual\stest\scases\sand\sto\sreport\sthat\sinformation\sin\sthe\nsummary\sat\sthe\send\sof\seach\stest\srun. -D 2024-09-05T12:06:45.482 +C Testrunner.tcl\senhancements:\s\s(1)\sAttempt\sto\sbuild\sthe\sSQLite\stcl\sextension\nif\sit\sis\snot\salready\savailable.\s\s(2)\stestrunner\starget\s"devtest"\sis\sadded\sas\nan\salias\sfor\s"mdevtest".\s\s(3)\sTry\sto\skeep\ssummary\sinformation\sat\sthe\send\sof\na\stest\sbelow\s80-characters\sper\sline.\s\s(4)\sUpdate\sthe\sMakefile.in\sso\sthat\nthe\s"clean"\starget\sremoves\sthe\stcl\sextension\sbuilt\sby\sitem\s1\sabove. +D 2024-09-05T15:28:15.141 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 -F Makefile.in af5fbc3453b745daa68c7aa5dfdb945c09cb724971db3b783d6b5e1a62279e28 +F Makefile.in 098d9814c66d81a8bbfb3550876b7d9695a0b050c0d4680afb4931a7aa8fe7f1 F Makefile.linux-gcc f3842a0b1efbfbb74ac0ef60e56b301836d05b4d867d014f714fa750048f1ab6 F Makefile.msc e72a4653ea8f48efd14c7dac1e7de308a1f3cc91f905f15cbcacc680301c8f03 F README.md 5b678e264236788390d11991f2c0052bd73f19790173883fc56d638bcb849154 @@ -1714,8 +1714,8 @@ F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 F test/temptable2.test 76821347810ecc88203e6ef0dd6897b6036ac788e9dd3e6b04fd4d1631311a16 F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc -F test/tester.tcl 2c203a2dd664298f239f0ec3ce22fbc65b5f021c1e09edbae8452af8a694e052 -F test/testrunner.tcl 1ebde7e8b57c464b9eb6ce047977c1edb46688b2ab94cc67e3cd5e4a3645ff00 x +F test/tester.tcl 7b44f1a9b9a2de8112695b908afc21dd9a68cd2d44e84b73f1b27b53492c0d59 +F test/testrunner.tcl 8b8052ef2abf673729eeeeb222961c99b3e08231635c196533aff315e3f12f2d x F test/testrunner_data.tcl dbc0bb1c5b912dfd1e32b25d544318e412edd6085bd5fc9e6619cb93a739b786 F test/thread001.test a0985c117eab62c0c65526e9fa5d1360dd1cac5b03bde223902763274ce21899 F test/thread002.test c24c83408e35ba5a952a3638b7ac03ccdf1ce4409289c54a050ac4c5f1de7502 @@ -2212,8 +2212,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a01d869520329fb9e786cdc65f359785a95d19d289e4c6b844c758d6e5385aaf -R a6deed6e9bd78f423240221d74d8d7b3 +P 80ebb7c7e686cd936ac834f2258f585a7004762593e0bc859ecd75d6fb0badfd +R 83bee4b7a0d6a0c18b04c62c3e6012df U drh -Z 8cf0ea19b5a0efdaca6c57466a00e96c +Z 30dd6ec18ce8874d43e8198deb845f86 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 9f0908d687..5af6554025 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -80ebb7c7e686cd936ac834f2258f585a7004762593e0bc859ecd75d6fb0badfd +aa5f10f21dbfb24ee54ca96bfb7b013ae29e26fec05b80681f19cc63d9face49 diff --git a/test/tester.tcl b/test/tester.tcl index 164ee47f42..b5f49ebde9 100644 --- a/test/tester.tcl +++ b/test/tester.tcl @@ -1279,10 +1279,15 @@ proc finalize_testing {} { out of $nTest tests" } else { set cpuinfo {} - if {[catch {exec hostname} hname]==0} {set cpuinfo [string trim $hname]} + if {[catch {exec hostname} hname]==0} { + regsub {\.local$} $hname {} hname + set cpuinfo [string trim $hname] + } append cpuinfo " $::tcl_platform(os)" append cpuinfo " [expr {$::tcl_platform(pointerSize)*8}]-bit" - append cpuinfo " [string map {E -e} $::tcl_platform(byteOrder)]" + if {[string match big* $::tcl_platform(byteOrder)]} { + append cpuinfo " [string map {E -e} $::tcl_platform(byteOrder)]" + } output2 "SQLite [sqlite3 -sourceid]" output2 "$nErr errors out of $nTest tests on $cpuinfo" } diff --git a/test/testrunner.tcl b/test/testrunner.tcl index cd9d0daff5..09704c03d3 100755 --- a/test/testrunner.tcl +++ b/test/testrunner.tcl @@ -17,8 +17,14 @@ cd $dir # recommend that the user build one. # proc find_interpreter {} { + global dir set interpreter [file tail [info nameofexec]] set rc [catch { package require sqlite3 }] + if {$rc} { + if {[file readable pkgIndex.tcl] && [catch {source pkgIndex.tcl}]==0} { + set rc [catch { package require sqlite3 }] + } + } if {$rc} { if { [string match -nocase testfixture* $interpreter]==0 && [file executable ./testfixture] @@ -31,8 +37,30 @@ proc find_interpreter {} { } } if {$rc} { - puts stderr "Failed to find tcl package sqlite3" - puts stderr "Run \"make testfixture\" and then try again..." + puts "Cannot find tcl package sqlite3: Trying to build it now..." + if {$::tcl_platform(platform)=="windows"} { + set bat [open make-tcl-extension.bat w] + puts $bat "nmake /f Makefile.msc tclextension" + close $bat + catch {exec -ignorestderr -- make-tcl-extension.bat} + } else { + catch {exec make tclextension} + } + if {[file readable pkgIndex.tcl] && [catch {source pkgIndex.tcl}]==0} { + set rc [catch { package require sqlite3 }] + } + if {$rc==0} { + puts "The SQLite tcl extension was successfully built and loaded." + puts "Run \"make tclextension-install\" to avoid having to rebuild\ + it in the future." + } else { + puts "Unable to build the SQLite tcl extension" + } + } + if {$rc} { + puts stderr "Cannot find a working instance of the SQLite tcl extension." + puts stderr "Run \"make tclextension\" or \"make testfixture\" and\ + try again..." exit 1 } } @@ -1166,6 +1194,7 @@ proc add_jobs_from_cmdline {patternlist} { } } + devtest - mdevtest { set config_set { All-O0 @@ -1530,9 +1559,9 @@ proc run_testset {} { SELECT pltfm, count(*) FROM jobs WHERE pltfm IS NOT NULL ORDER BY 2 DESC LIMIT 1 } {puts $pltfm} - puts "$totalerr errors out of $totaltest tests in about $et $pltfm" + puts "$totalerr errors out of $totaltest tests in $et $pltfm" trdb eval { - SELECT DISTINCT svers FROM jobs WHERE svers IS NOT NULL + SELECT DISTINCT substr(svers,1,80) FROM jobs WHERE svers IS NOT NULL } {puts $svers} } From 6ffcdb1856fea916f9c98379e9cb57afc6db9104 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 5 Sep 2024 15:55:04 +0000 Subject: [PATCH 13/36] Update notes on build procedures in the README.md file. FossilOrigin-Name: 1d12744fe7cc294994e53e1ffe72799d0321dd5c1ef031213f083e200918fee5 --- README.md | 43 +++++++++++++++++++++++++++---------------- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 34 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index c208f11ea3..ad11c453ac 100644 --- a/README.md +++ b/README.md @@ -103,17 +103,19 @@ script found at the root of the source tree. Then run "make". For example: - tar xzf sqlite.tar.gz ;# Unpack the source tree into "sqlite" - mkdir bld ;# Build will occur in a sibling directory - cd bld ;# Change to the build directory - ../sqlite/configure ;# Run the configure script - make sqlite3 ;# Builds the "sqlite3" command-line tool - make sqlite3.c ;# Build the "amalgamation" source file - make devtest ;# Run development tests (requires tcl-dev) - make releasetest ;# Run full release tests (requires tcl-dev) - make sqldiff ;# Builds the "sqldiff" command-line tool - make sqlite3_analyzer ;# Builds the "sqlite3_analyzer" tool (requires tcl-dev) - make tclextension-install ;# Build and install the SQLite TCL extension + apt install gcc make tcl-dev ;# Make sure you have all the necessary build tools + tar xzf sqlite.tar.gz ;# Unpack the source tree into "sqlite" + mkdir bld ;# Build will occur in a sibling directory + cd bld ;# Change to the build directory + ../sqlite/configure ;# Run the configure script + make sqlite3 ;# Builds the "sqlite3" command-line tool + make sqlite3.c ;# Build the "amalgamation" source file + make sqldiff ;# Builds the "sqldiff" command-line tool + # Makefile targets below this point require tcl-dev + make tclextension-install ;# Build and install the SQLite TCL extension + make devtest ;# Run development tests + make releasetest ;# Run full release tests + make sqlite3_analyzer ;# Builds the "sqlite3_analyzer" tool See the makefile for additional targets. For debugging builds, the core developers typically run "configure" with options like this: @@ -124,9 +126,12 @@ For release builds, the core developers usually do: ../sqlite/configure --enable-all -Almost all makefile targets require a "tclsh" TCL interpreter -version 8.6 or later. The targets marked with "(requires tcl-dev)" also require -the TCL development libraries. +Almost all makefile targets require a "tclsh" TCL interpreter version 8.6 or +later. The "tclextension-install" target and the test targets that follow +all require TCL development libraries too. ("apt install tcl-dev"). It is +helpful, but is not required, to install the SQLite TCL extension (the +"tclextension-install" target) prior to running tests. The "releasetest" +target has additional requiremenst, such as "valgrind". On "make" command-lines, one can add "OPTIONS=..." to specify additional compile-time options over and above those set by ./configure. For example, @@ -157,15 +162,21 @@ TCL library, using a command like this: SQLite uses "tclsh.exe" as part of the build process, and so that program will need to be somewhere on your %PATH%. SQLite itself does not contain any TCL code, but it does use TCL to help with the -build process and to run tests. +build process and to run tests. You may need to install TCL development +libraries in order to successfully complete some makefile targets. +It is helpful, but is not required, to install the SQLite TCL extension +(the "tclextension-install" target) prior to running tests. Build using Makefile.msc. Example: nmake /f Makefile.msc sqlite3.exe nmake /f Makefile.msc sqlite3.c + nmake /f Makefile.msc sqldiff.exe + # Makefile targets below this point require TCL development libraries + nmake /f Makefile.msc tclextension-install nmake /f Makefile.msc devtest nmake /f Makefile.msc releasetest - nmake /f Makefile.msc tclextension-install + nmake /f Makefile.msc sqlite3_analyzer.exe There are many other makefile targets. See comments in Makefile.msc for details. diff --git a/manifest b/manifest index 2cb670f56a..b5a5da924d 100644 --- a/manifest +++ b/manifest @@ -1,12 +1,12 @@ -C Testrunner.tcl\senhancements:\s\s(1)\sAttempt\sto\sbuild\sthe\sSQLite\stcl\sextension\nif\sit\sis\snot\salready\savailable.\s\s(2)\stestrunner\starget\s"devtest"\sis\sadded\sas\nan\salias\sfor\s"mdevtest".\s\s(3)\sTry\sto\skeep\ssummary\sinformation\sat\sthe\send\sof\na\stest\sbelow\s80-characters\sper\sline.\s\s(4)\sUpdate\sthe\sMakefile.in\sso\sthat\nthe\s"clean"\starget\sremoves\sthe\stcl\sextension\sbuilt\sby\sitem\s1\sabove. -D 2024-09-05T15:28:15.141 +C Update\snotes\son\sbuild\sprocedures\sin\sthe\sREADME.md\sfile. +D 2024-09-05T15:55:04.340 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 F Makefile.in 098d9814c66d81a8bbfb3550876b7d9695a0b050c0d4680afb4931a7aa8fe7f1 F Makefile.linux-gcc f3842a0b1efbfbb74ac0ef60e56b301836d05b4d867d014f714fa750048f1ab6 F Makefile.msc e72a4653ea8f48efd14c7dac1e7de308a1f3cc91f905f15cbcacc680301c8f03 -F README.md 5b678e264236788390d11991f2c0052bd73f19790173883fc56d638bcb849154 +F README.md 282d999d14bfd1dcb36652abbfba9f8dffbda083ddcdb91361459e43164a1d23 F VERSION 0db40f92c04378404eb45bff93e9e42c148c7e54fd3da99469ed21e22411f5a6 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 F art/icon-243x273.gif 9750b734f82fdb3dc43127753d5e6fbf3b62c9f4e136c2fbf573b2f57ea87af5 @@ -2212,8 +2212,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 80ebb7c7e686cd936ac834f2258f585a7004762593e0bc859ecd75d6fb0badfd -R 83bee4b7a0d6a0c18b04c62c3e6012df +P aa5f10f21dbfb24ee54ca96bfb7b013ae29e26fec05b80681f19cc63d9face49 +R 7cc85ab35bb58c1a46f86590da7723ea U drh -Z 30dd6ec18ce8874d43e8198deb845f86 +Z a2b02066e2e694d910978f885128ac52 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 5af6554025..210827e73b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -aa5f10f21dbfb24ee54ca96bfb7b013ae29e26fec05b80681f19cc63d9face49 +1d12744fe7cc294994e53e1ffe72799d0321dd5c1ef031213f083e200918fee5 From 934e796e439495ab0f2655e5711c5fe6d39285b9 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 5 Sep 2024 23:22:55 +0000 Subject: [PATCH 14/36] Ensure that the WhereInfo.revMask bitmap is adjusted when tables are removed from the FROM clause by the Omit-Noop-Join optimization of [0cd82ee9a8413cf1]. Fix for the issue described by [forum:/forum/8a1e467e905b8d27|format post 8a1e467e905b8d27]. FossilOrigin-Name: 22ca5a2ffb89ccb5f337993b5a95e27c449c39014284156eabc33da012a8759c --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 3 +++ test/join2.test | 21 +++++++++++++++++++++ 4 files changed, 32 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index b5a5da924d..0a8c14d268 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\snotes\son\sbuild\sprocedures\sin\sthe\sREADME.md\sfile. -D 2024-09-05T15:55:04.340 +C Ensure\sthat\sthe\sWhereInfo.revMask\sbitmap\sis\sadjusted\swhen\stables\sare\sremoved\nfrom\sthe\sFROM\sclause\sby\sthe\sOmit-Noop-Join\soptimization\sof\n[0cd82ee9a8413cf1].\s\sFix\sfor\sthe\sissue\sdescribed\sby\n[forum:/forum/8a1e467e905b8d27|format\spost\s8a1e467e905b8d27]. +D 2024-09-05T23:22:55.647 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -852,7 +852,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 887fc4ca3f020ebb2e376f222069570834ac63bf50111ef0cbf3ae417048ed89 F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452 F src/walker.c d5006d6b005e4ea7302ad390957a8d41ed83faa177e412f89bc5600a7462a014 -F src/where.c c046dd58c3410f7b7528e1e6317cb876398557bad346d568ed8562321a7d002d +F src/where.c 079f1cf6abc086315f5c9873626b7c0537f382c55184c516b8e3d29f7a537fdc F src/whereInt.h a5d079c346a658b7a6e9e47bb943d021e02fa1e6aed3b964ca112112a4892192 F src/wherecode.c 5172d647798134e7c92536ddffe7e530c393d79b5dedd648b88faf2646c65baf F src/whereexpr.c 44f41ae554c7572e1de1485b3169b233ee04d464b2ee5881687ede3bf07cacfa @@ -1346,7 +1346,7 @@ F test/ioerr5.test 5984da7bf74b6540aa356f2ab0c6ae68a6d12039a3d798a9ac6a100abc17d F test/ioerr6.test a395a6ab144b26a9e3e21059a1ab6a7149cca65b F test/istrue.test e7f285bb70282625c258e866ce6337d4c762922f5a300e1b50f958aef6e7d9c9 F test/join.test f7abfef3faeaf2800308872e33a57e5b6e4a2b44fb8c6b90c6068412e71a6cf4 -F test/join2.test 8561fe82ce434ac96de91544072e578dc2cadddf2d9bc9cd802f866a9b92502e +F test/join2.test f59d63264fb24784ae9c3bc9d867eb569cd6d442da5660f8852effe5c1938c27 F test/join3.test 6f0c774ff1ba0489e6c88a3e77b9d3528fb4fda0 F test/join4.test 1a352e4e267114444c29266ce79e941af5885916 F test/join5.test 380d12a9350f99f0cc681a4f1fea999886f18b3fe0d71a9b3065bcaead1e007f @@ -2212,8 +2212,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P aa5f10f21dbfb24ee54ca96bfb7b013ae29e26fec05b80681f19cc63d9face49 -R 7cc85ab35bb58c1a46f86590da7723ea +P 1d12744fe7cc294994e53e1ffe72799d0321dd5c1ef031213f083e200918fee5 +R 45d88f5375170588d0b1bd314eace7a3 U drh -Z a2b02066e2e694d910978f885128ac52 +Z 4179122ae9403ba759b59f6aa6b2df32 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 210827e73b..dc2d1cd140 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1d12744fe7cc294994e53e1ffe72799d0321dd5c1ef031213f083e200918fee5 +22ca5a2ffb89ccb5f337993b5a95e27c449c39014284156eabc33da012a8759c diff --git a/src/where.c b/src/where.c index 2b835009db..7ca4f533f0 100644 --- a/src/where.c +++ b/src/where.c @@ -6199,6 +6199,7 @@ static SQLITE_NOINLINE Bitmask whereOmitNoopJoin( WhereTerm *pTerm, *pEnd; SrcItem *pItem; WhereLoop *pLoop; + Bitmask m1; pLoop = pWInfo->a[i].pWLoop; pItem = &pWInfo->pTabList->a[pLoop->iTab]; if( (pItem->fg.jointype & (JT_LEFT|JT_RIGHT))!=JT_LEFT ) continue; @@ -6226,6 +6227,8 @@ static SQLITE_NOINLINE Bitmask whereOmitNoopJoin( } if( pTerm drop loop %c not used\n", pLoop->cId)); + m1 = MASKBIT(i)-1; + pWInfo->revMask = (m1 & pWInfo->revMask) | ((pWInfo->revMask>>1) & ~m1); notReady &= ~pLoop->maskSelf; for(pTerm=pWInfo->sWC.a; pTermprereqAll & pLoop->maskSelf)!=0 ){ diff --git a/test/join2.test b/test/join2.test index 15e76f965d..6f2fe1d770 100644 --- a/test/join2.test +++ b/test/join2.test @@ -428,4 +428,25 @@ do_eqp_test 12.3 { `--SEARCH t1 USING INTEGER PRIMARY KEY (rowid=?) LEFT-JOIN } +# 2024-09-05 https://sqlite.org/forum/forumpost/8a1e467e905b8d27 +# When performing the Omit-Noop-Join optimization, if FROM clause terms +# to the right of the omitted join have the reverse-order bit set in the +# WhereInfo.revMask bitmask, those bits need to be shifted to account +# for the omitted join. +# +reset_db +do_execsql_test 13.0 { + CREATE TABLE t1(a1 INTEGER PRIMARY KEY, b1 INT); + CREATE TABLE t2(c2 INT, d2 INTEGER PRIMARY KEY); + CREATE TABLE t3(e3 INTEGER PRIMARY KEY); + INSERT INTO t1 VALUES(33,0); + INSERT INTO t2 VALUES(33,1),(33,2); +} +do_execsql_test 13.1 { + SELECT t1.a1, t2.d2 + FROM (t1 LEFT JOIN t3 ON t3.e3=t1.b1) JOIN t2 ON t2.c2=t1.a1 + WHERE t1.a1=33 + ORDER BY t2.d2 DESC; +} {33 2 33 1} + finish_test From dc181d41d56141fa151257ffd2892b7fdc3ba2b0 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 5 Sep 2024 23:40:13 +0000 Subject: [PATCH 15/36] Fix a testrunner status reporting problem introduced by [aa5f10f21dbfb24e]. FossilOrigin-Name: d20c65c3b4256a662ebf7ed024b7b7adaceca90358f58111dc645da322000794 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/testrunner.tcl | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 0a8c14d268..5ffa57feba 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Ensure\sthat\sthe\sWhereInfo.revMask\sbitmap\sis\sadjusted\swhen\stables\sare\sremoved\nfrom\sthe\sFROM\sclause\sby\sthe\sOmit-Noop-Join\soptimization\sof\n[0cd82ee9a8413cf1].\s\sFix\sfor\sthe\sissue\sdescribed\sby\n[forum:/forum/8a1e467e905b8d27|format\spost\s8a1e467e905b8d27]. -D 2024-09-05T23:22:55.647 +C Fix\sa\stestrunner\sstatus\sreporting\sproblem\sintroduced\sby\s[aa5f10f21dbfb24e]. +D 2024-09-05T23:40:13.439 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1715,7 +1715,7 @@ F test/temptable2.test 76821347810ecc88203e6ef0dd6897b6036ac788e9dd3e6b04fd4d163 F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc F test/tester.tcl 7b44f1a9b9a2de8112695b908afc21dd9a68cd2d44e84b73f1b27b53492c0d59 -F test/testrunner.tcl 8b8052ef2abf673729eeeeb222961c99b3e08231635c196533aff315e3f12f2d x +F test/testrunner.tcl a59e911e3732ba4d6b66e310f72d9599eb0392d3de659a525c519ba755cbf3e1 x F test/testrunner_data.tcl dbc0bb1c5b912dfd1e32b25d544318e412edd6085bd5fc9e6619cb93a739b786 F test/thread001.test a0985c117eab62c0c65526e9fa5d1360dd1cac5b03bde223902763274ce21899 F test/thread002.test c24c83408e35ba5a952a3638b7ac03ccdf1ce4409289c54a050ac4c5f1de7502 @@ -2212,8 +2212,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1d12744fe7cc294994e53e1ffe72799d0321dd5c1ef031213f083e200918fee5 -R 45d88f5375170588d0b1bd314eace7a3 +P 22ca5a2ffb89ccb5f337993b5a95e27c449c39014284156eabc33da012a8759c +R bc86dd135008dded1df1caf42383464b U drh -Z 4179122ae9403ba759b59f6aa6b2df32 +Z 5f358c18a2ed0f553bb9c3eac9d42cd6 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index dc2d1cd140..c0a2eed54c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -22ca5a2ffb89ccb5f337993b5a95e27c449c39014284156eabc33da012a8759c +d20c65c3b4256a662ebf7ed024b7b7adaceca90358f58111dc645da322000794 diff --git a/test/testrunner.tcl b/test/testrunner.tcl index 09704c03d3..d28bdd4fba 100755 --- a/test/testrunner.tcl +++ b/test/testrunner.tcl @@ -1561,8 +1561,8 @@ proc run_testset {} { } {puts $pltfm} puts "$totalerr errors out of $totaltest tests in $et $pltfm" trdb eval { - SELECT DISTINCT substr(svers,1,80) FROM jobs WHERE svers IS NOT NULL - } {puts $svers} + SELECT DISTINCT substr(svers,1,80) as v1 FROM jobs WHERE svers IS NOT NULL + } {puts $v1} } From 56a048054078e316ddb057057c323116b82d8b46 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 6 Sep 2024 09:49:13 +0000 Subject: [PATCH 16/36] Clarification and simplification to the README.md file and to the instructions for building on Windows. FossilOrigin-Name: f69ef1a37b2778bdf73ee7e3b3edd74f7344ab8e5eedbedc22203c782e521f5b --- README.md | 27 +++++++++++++++------------ doc/compile-for-windows.md | 13 ++++++++++++- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 35 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index ad11c453ac..689b52dabd 100644 --- a/README.md +++ b/README.md @@ -73,23 +73,26 @@ archives or [SQLite archives](https://sqlite.org/cli.html#sqlar) as follows: then click on the "Tarball" or "ZIP Archive" links on the information page. -To access sources directly using Fossil, first install Fossil version 2.0 or later. -Source tarballs and precompiled binaries available -[here](https://www.fossil-scm.org/home/uv/download.html). Fossil is +To access sources directly using [Fossil](https://fossil-scm.org/home), +first install Fossil version 2.0 or later. +Source tarballs and precompiled binaries available at +. Fossil is a stand-alone program. To install, simply download or build the single executable file and put that file someplace on your $PATH. Then run commands like this: - mkdir -p ~/sqlite ~/Fossils + mkdir -p ~/sqlite cd ~/sqlite - fossil clone https://www.sqlite.org/src ~/Fossils/sqlite.fossil - fossil open ~/Fossils/sqlite.fossil + fossil open https://sqlite.org/src -After setting up a repository using the steps above, you can do -bandwidth-efficient updates to the latest version using: +The "fossil open" command will take two or three minutes. Afterwards, +you can do fast, bandwidth-efficient updates to the whatever versions +of SQLite you like. Some examples: - fossil update trunk ;# latest trunk check-in - fossil update release ;# latest official release + fossil update trunk ;# latest trunk check-in + fossil update release ;# latest official release + fossil update trunk:2024-01-01 ;# First trunk check-in after 2024-01-01 + fossil update version-3.39.0 ;# Version 3.39.0 Or type "fossil ui" to get a web-based user interface. @@ -148,7 +151,7 @@ show what changes are needed. ## Compiling for Windows Using MSVC -On Windows, all applicable build products can be compiled with MSVC. +On Windows, everything can be compiled with MSVC. You will also need a working installation of TCL. See the [compile-for-windows.md](doc/compile-for-windows.md) document for additional information about how to install MSVC and TCL and configure your @@ -383,7 +386,7 @@ implementation. It will not be the easiest library in the world to hack. * **VERSION**, **manifest**, and **manifest.uuid** - These files define the current SQLite version number. The "VERSION" file is human generated, but the "manifest" and "manifest.uuid" files are automatically generated - by the [Fossil version control system](https://fossil-scm/). + by the [Fossil version control system](https://fossil-scm.org/). There are many other source files. Each has a succinct header comment that describes its purpose and role within the larger system. diff --git a/doc/compile-for-windows.md b/doc/compile-for-windows.md index 5e6a2d1a25..1627188acd 100644 --- a/doc/compile-for-windows.md +++ b/doc/compile-for-windows.md @@ -57,11 +57,22 @@ canonical source on a new Windows 11 PC, as of 2023-11-01:
  • `nmake /f makefile.msc`
  • `nmake /f makefile.msc sqlite3.c` +
  • `nmake /f makefile.msc sqlite3.exe` +
  • `nmake /f makefile.msc sqldiff.exe` +
  • `nmake /f makefile.msc tclextension-install`
  • `nmake /f makefile.msc devtest`
  • `nmake /f makefile.msc releasetest` -
  • `nmake /f makefile.msc sqlite3.exe` +
  • `nmake /f makefile.msc sqlite3_analyzer.exe`
+ It is not required that you run the "tclextension-install" target prior to + running tests. However, the tests will run more smoothly if you do. + The version of SQLite used for the TCL extension does *not* need to + correspond to the version of SQLite under test. So you can install the + SQLite TCL extension once, and then use it to test many different versions + of SQLite. + + 7. For a debugging build of the CLI, where the ".treetrace" and ".wheretrace" commands work, add the DEBUG=3 argument to nmake. Like this:
    diff --git a/manifest b/manifest index 5ffa57feba..f6017ee1ad 100644 --- a/manifest +++ b/manifest @@ -1,12 +1,12 @@ -C Fix\sa\stestrunner\sstatus\sreporting\sproblem\sintroduced\sby\s[aa5f10f21dbfb24e]. -D 2024-09-05T23:40:13.439 +C Clarification\sand\ssimplification\sto\sthe\sREADME.md\sfile\sand\sto\sthe\sinstructions\nfor\sbuilding\son\sWindows. +D 2024-09-06T09:49:13.510 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 F Makefile.in 098d9814c66d81a8bbfb3550876b7d9695a0b050c0d4680afb4931a7aa8fe7f1 F Makefile.linux-gcc f3842a0b1efbfbb74ac0ef60e56b301836d05b4d867d014f714fa750048f1ab6 F Makefile.msc e72a4653ea8f48efd14c7dac1e7de308a1f3cc91f905f15cbcacc680301c8f03 -F README.md 282d999d14bfd1dcb36652abbfba9f8dffbda083ddcdb91361459e43164a1d23 +F README.md c3c0f19532ce28f6297a71870f3c7b424729f0e6d9ab889616d3587dd2332159 F VERSION 0db40f92c04378404eb45bff93e9e42c148c7e54fd3da99469ed21e22411f5a6 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 F art/icon-243x273.gif 9750b734f82fdb3dc43127753d5e6fbf3b62c9f4e136c2fbf573b2f57ea87af5 @@ -39,7 +39,7 @@ F configure 49523f0a070b583cea040d26eff53a65fb0893eca4663b1343a4d5a9a964da53 x F configure.ac a100ebf7a07f5dedd319ef547dd467d1676ed059b85a7877aa9c44ac309f7000 F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad F doc/F2FS.txt c1d4a0ae9711cfe0e1d8b019d154f1c29e0d3abfe820787ba1e9ed7691160fcd -F doc/compile-for-windows.md e8635eea9153dcd6a51fd2740666ebc4492b3813cb1ac31cd8e99150df91762d +F doc/compile-for-windows.md 4d4bfafda42a7a33f166d23aed4db1bb4ea1e5751595a5cced2bad349fd14652 F doc/json-enhancements.md e356fc834781f1f1aa22ee300027a270b2c960122468499bf347bb123ce1ea4f F doc/jsonb.md 5fab4b8613aa9153fbeb6259297bd4697988af8b3d23900deba588fa7841456b F doc/lemon.html 8b266ff711d2ec7f867c3dca37634963f48a630329908cc282beebfa8c708706 @@ -2212,8 +2212,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 22ca5a2ffb89ccb5f337993b5a95e27c449c39014284156eabc33da012a8759c -R bc86dd135008dded1df1caf42383464b +P d20c65c3b4256a662ebf7ed024b7b7adaceca90358f58111dc645da322000794 +R e0122c673835ce2d6d2df201e5b1e575 U drh -Z 5f358c18a2ed0f553bb9c3eac9d42cd6 +Z 474793855763ef2064fdff2fee5b0897 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index c0a2eed54c..f130b71200 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d20c65c3b4256a662ebf7ed024b7b7adaceca90358f58111dc645da322000794 +f69ef1a37b2778bdf73ee7e3b3edd74f7344ab8e5eedbedc22203c782e521f5b From c4e84b72958d46aad7a08e77b6e8744e8e3e88ab Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 6 Sep 2024 10:35:36 +0000 Subject: [PATCH 17/36] A testcase() macro added to help ensure that the fix to the omit-noop-join optimization from yesterday does not regress. FossilOrigin-Name: 224628b1039b996499e0d806fc0215f175da6f1059eb1b9ac491ac11126971ab --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 3 ++- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index f6017ee1ad..d0c4ee56d5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Clarification\sand\ssimplification\sto\sthe\sREADME.md\sfile\sand\sto\sthe\sinstructions\nfor\sbuilding\son\sWindows. -D 2024-09-06T09:49:13.510 +C A\stestcase()\smacro\sadded\sto\shelp\sensure\sthat\sthe\sfix\sto\sthe\somit-noop-join\noptimization\sfrom\syesterday\sdoes\snot\sregress. +D 2024-09-06T10:35:36.196 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -852,7 +852,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 887fc4ca3f020ebb2e376f222069570834ac63bf50111ef0cbf3ae417048ed89 F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452 F src/walker.c d5006d6b005e4ea7302ad390957a8d41ed83faa177e412f89bc5600a7462a014 -F src/where.c 079f1cf6abc086315f5c9873626b7c0537f382c55184c516b8e3d29f7a537fdc +F src/where.c 7fb55836eb7fd07f0a0d8400c50619fc02cda1f46a617cfb003c2990f040193d F src/whereInt.h a5d079c346a658b7a6e9e47bb943d021e02fa1e6aed3b964ca112112a4892192 F src/wherecode.c 5172d647798134e7c92536ddffe7e530c393d79b5dedd648b88faf2646c65baf F src/whereexpr.c 44f41ae554c7572e1de1485b3169b233ee04d464b2ee5881687ede3bf07cacfa @@ -2212,8 +2212,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d20c65c3b4256a662ebf7ed024b7b7adaceca90358f58111dc645da322000794 -R e0122c673835ce2d6d2df201e5b1e575 +P f69ef1a37b2778bdf73ee7e3b3edd74f7344ab8e5eedbedc22203c782e521f5b +R a9f753c3499efd243abe359303301f78 U drh -Z 474793855763ef2064fdff2fee5b0897 +Z a70e5af6a81ae35c981b5946f602f67c # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index f130b71200..b195d89eaa 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f69ef1a37b2778bdf73ee7e3b3edd74f7344ab8e5eedbedc22203c782e521f5b +224628b1039b996499e0d806fc0215f175da6f1059eb1b9ac491ac11126971ab diff --git a/src/where.c b/src/where.c index 7ca4f533f0..35da4caa6e 100644 --- a/src/where.c +++ b/src/where.c @@ -6226,8 +6226,9 @@ static SQLITE_NOINLINE Bitmask whereOmitNoopJoin( } } if( pTerm drop loop %c not used\n", pLoop->cId)); + WHERETRACE(0xffffffff,("-> omit unused FROM-clause term %c\n",pLoop->cId)); m1 = MASKBIT(i)-1; + testcase( ((pWInfo->revMask>>1) & ~m1)!=0 ); pWInfo->revMask = (m1 & pWInfo->revMask) | ((pWInfo->revMask>>1) & ~m1); notReady &= ~pLoop->maskSelf; for(pTerm=pWInfo->sWC.a; pTerm Date: Fri, 6 Sep 2024 11:21:53 +0000 Subject: [PATCH 18/36] Fix harmless compiler warnings. FossilOrigin-Name: 60fb8ee153ec293b6b3a4170dafa305e4c16af575aced72daef46116d8dc2bb6 --- ext/rbu/sqlite3rbu.c | 41 ++++++++++++++++++++++++++++++++------ ext/session/test_session.c | 8 ++++---- manifest | 20 +++++++++---------- manifest.uuid | 2 +- src/test1.c | 2 +- test/fuzzcheck.c | 7 ++++--- test/fuzzinvariants.c | 7 ++++--- 7 files changed, 59 insertions(+), 28 deletions(-) diff --git a/ext/rbu/sqlite3rbu.c b/ext/rbu/sqlite3rbu.c index feb7695d66..8e29ab820f 100644 --- a/ext/rbu/sqlite3rbu.c +++ b/ext/rbu/sqlite3rbu.c @@ -336,6 +336,27 @@ struct RbuFrame { u32 iWalFrame; }; +#ifndef UNUSED_PARAMETER +/* +** The following macros are used to suppress compiler warnings and to +** make it clear to human readers when a function parameter is deliberately +** left unused within the body of a function. This usually happens when +** a function is called via a function pointer. For example the +** implementation of an SQL aggregate step callback may not use the +** parameter indicating the number of arguments passed to the aggregate, +** if it knows that this is enforced elsewhere. +** +** When a function parameter is not used at all within the body of a function, +** it is generally named "NotUsed" or "NotUsed2" to make things even clearer. +** However, these macros may also be used to suppress warnings related to +** parameters that may or may not be used depending on compilation options. +** For example those parameters only used in assert() statements. In these +** cases the parameters are named as per the usual conventions. +*/ +#define UNUSED_PARAMETER(x) (void)(x) +#define UNUSED_PARAMETER2(x,y) UNUSED_PARAMETER(x),UNUSED_PARAMETER(y) +#endif + /* ** RBU handle. ** @@ -387,7 +408,7 @@ struct sqlite3rbu { int rc; /* Value returned by last rbu_step() call */ char *zErrmsg; /* Error message if rc!=SQLITE_OK */ int nStep; /* Rows processed for current object */ - int nProgress; /* Rows processed for all objects */ + sqlite3_int64 nProgress; /* Rows processed for all objects */ RbuObjIter objiter; /* Iterator for skipping through tbl/idx */ const char *zVfsName; /* Name of automatically created rbu vfs */ rbu_file *pTargetFd; /* File handle open on target db */ @@ -504,7 +525,7 @@ static unsigned int rbuDeltaGetInt(const char **pz, int *pLen){ v = (v<<6) + c; } z--; - *pLen -= z - zStart; + *pLen -= (int)(z - zStart); *pz = (char*)z; return v; } @@ -689,6 +710,7 @@ static void rbuFossilDeltaFunc( char *aOut; assert( argc==2 ); + UNUSED_PARAMETER(argc); nOrig = sqlite3_value_bytes(argv[0]); aOrig = (const char*)sqlite3_value_blob(argv[0]); @@ -2268,13 +2290,13 @@ static char *rbuObjIterGetIndexWhere(sqlite3rbu *p, RbuObjIter *pIter){ else if( c==')' ){ nParen--; if( nParen==0 ){ - int nSpan = &zSql[i] - pIter->aIdxCol[iIdxCol].zSpan; + int nSpan = (int)(&zSql[i] - pIter->aIdxCol[iIdxCol].zSpan); pIter->aIdxCol[iIdxCol++].nSpan = nSpan; i++; break; } }else if( c==',' && nParen==1 ){ - int nSpan = &zSql[i] - pIter->aIdxCol[iIdxCol].zSpan; + int nSpan = (int)(&zSql[i] - pIter->aIdxCol[iIdxCol].zSpan); pIter->aIdxCol[iIdxCol++].nSpan = nSpan; pIter->aIdxCol[iIdxCol].zSpan = &zSql[i+1]; }else if( c=='"' || c=='\'' || c=='`' ){ @@ -2964,6 +2986,8 @@ static void rbuFileSuffix3(const char *zBase, char *z){ for(i=sz-1; i>0 && z[i]!='/' && z[i]!='.'; i--){} if( z[i]=='.' && sz>i+4 ) memmove(&z[i+1], &z[sz-3], 4); } +#else + UNUSED_PARAMETER2(zBase,z); #endif } @@ -3548,7 +3572,7 @@ static void rbuSaveState(sqlite3rbu *p, int eStage){ "(%d, %Q), " "(%d, %Q), " "(%d, %d), " - "(%d, %d), " + "(%d, %lld), " "(%d, %lld), " "(%d, %lld), " "(%d, %lld), " @@ -3906,6 +3930,7 @@ static void rbuIndexCntFunc( sqlite3 *db = (rbuIsVacuum(p) ? p->dbRbu : p->dbMain); assert( nVal==1 ); + UNUSED_PARAMETER(nVal); rc = prepareFreeAndCollectError(db, &pStmt, &zErrmsg, sqlite3_mprintf("SELECT count(*) FROM sqlite_schema " @@ -4181,7 +4206,7 @@ sqlite3rbu *sqlite3rbu_vacuum( ){ if( zTarget==0 ){ return rbuMisuseError(); } if( zState ){ - int n = strlen(zState); + size_t n = strlen(zState); if( n>=7 && 0==memcmp("-vactmp", &zState[n-7], 7) ){ return rbuMisuseError(); } @@ -4398,6 +4423,7 @@ int sqlite3rbu_savestate(sqlite3rbu *p){ */ static int xDefaultRename(void *pArg, const char *zOld, const char *zNew){ int rc = SQLITE_OK; + UNUSED_PARAMETER(pArg); #if defined(_WIN32_WCE) { LPWSTR zWideOld; @@ -5302,6 +5328,9 @@ static int rbuVfsCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){ ** No-op. */ static int rbuVfsGetLastError(sqlite3_vfs *pVfs, int a, char *b){ + UNUSED_PARAMETER(pVfs); + UNUSED_PARAMETER(a ); + UNUSED_PARAMETER(b); return 0; } diff --git a/ext/session/test_session.c b/ext/session/test_session.c index 79ad000e18..29aeadf537 100644 --- a/ext/session/test_session.c +++ b/ext/session/test_session.c @@ -383,10 +383,10 @@ static int SQLITE_TCLAPI test_session_cmd( { "rowid", SQLITE_SESSION_OBJCONFIG_ROWID }, { 0, 0 } }; - size_t sz = sizeof(aOpt[0]); + int sz = (int)sizeof(aOpt[0]); int iArg; - int iOpt; + Tcl_Size iOpt; if( Tcl_GetIndexFromObjStruct(interp,objv[2],aOpt,sz,"option",0,&iOpt) ){ return TCL_ERROR; } @@ -803,7 +803,7 @@ static int SQLITE_TCLAPI testSqlite3changesetApply( if( bV2 ){ while( objc>1 ){ const char *z1 = Tcl_GetString(objv[1]); - int n = strlen(z1); + int n = (int)strlen(z1); if( n>3 && n<=12 && 0==sqlite3_strnicmp("-nosavepoint", z1, n) ){ flags |= SQLITE_CHANGESETAPPLY_NOSAVEPOINT; } @@ -1119,7 +1119,7 @@ static int SQLITE_TCLAPI test_sqlite3session_foreach( while( objc>1 ){ char *zOpt = Tcl_GetString(objv[1]); - int nOpt = strlen(zOpt); + int nOpt = (int)strlen(zOpt); if( zOpt[0]!='-' ) break; if( nOpt<=7 && 0==sqlite3_strnicmp(zOpt, "-invert", nOpt) ){ isInvert = 1; diff --git a/manifest b/manifest index d0c4ee56d5..238ef2d4dc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C A\stestcase()\smacro\sadded\sto\shelp\sensure\sthat\sthe\sfix\sto\sthe\somit-noop-join\noptimization\sfrom\syesterday\sdoes\snot\sregress. -D 2024-09-06T10:35:36.196 +C Fix\sharmless\scompiler\swarnings. +D 2024-09-06T11:21:53.173 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -484,7 +484,7 @@ F ext/rbu/rbuvacuum.test 542561741ff2b262e3694bc6012b44694ee62c545845319a06f3237 F ext/rbu/rbuvacuum2.test ae097d04feb041446a74fac94b24bffeb3fdd60e32b848c5611e507ab702b81b F ext/rbu/rbuvacuum3.test 3ce42695fdf21aaa3499e857d7d4253bc499ad759bcd6c9362042c13cd37d8de F ext/rbu/rbuvacuum4.test ffccd22f67e2d0b380d2889685742159dfe0d19a3880ca3d2d1d69eefaebb205 -F ext/rbu/sqlite3rbu.c 4a3376c0fb9a844a799ac529fb81260523f6b13c9f629bc270c632dbae5fc1f8 +F ext/rbu/sqlite3rbu.c d90c79c2e9998e0f54da86d28c8433db7997864279fcfce02592e0838c8ce71d F ext/rbu/sqlite3rbu.h 9d923eb135c5d04aa6afd7c39ca47b0d1d0707c100e02f19fdde6a494e414304 F ext/rbu/test_rbu.c b9727c3394307d058e806c1da0f8bb7b24daf3c6bb94cb10cca88ea4d5c806c0 F ext/recover/dbdata.c 5295f4f922b60d7035b6b9fd5846b13071b9d97ed7fad8496837bb7640d24771 @@ -595,7 +595,7 @@ F ext/session/sessionstat1.test 5e718d5888c0c49bbb33a7a4f816366db85f59f6a4f97544 F ext/session/sessionwor.test 6fd9a2256442cebde5b2284936ae9e0d54bde692d0f5fd009ecef8511f4cf3fc F ext/session/sqlite3session.c c7473aafbd88f796391a8c25aa90975a8f3729ab7f4f8cf74ab9d3b014e10abe F ext/session/sqlite3session.h 683ccbf16e2c2521661fc4c1cf918ce57002039efbcabcd8097fa4bca569104b -F ext/session/test_session.c 6acbe67db80ab0806147eb62a12f9e3a44930f4a740b68b0a4340dddda2c10d7 +F ext/session/test_session.c aa29abdcc9011ac02f4fa38e8ede226106eaeee7c3ea7d8b2b999a124e0c368c F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt ca7e9ee82ca4e1c1744295f8184dd70edfae1992865d26c64303f539eb6c084c F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb @@ -777,7 +777,7 @@ F src/status.c cb11f8589a6912af2da3bb1ec509a94dd8ef27df4d4c1a97e0bcf2309ece972b F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 F src/tclsqlite.c c6888598f08dee3d9112a38ef42c8f5c89ca7f3190f4694744d0b84250f4bf8c F src/tclsqlite.h c6af51f31a2b2172d674608763a4b98fdf5cd587e4025053e546fb8077757262 -F src/test1.c 3f18399557d954bc85f4564aec8ea1777d2161a81d98a3ff6c9e9046bf3554c1 +F src/test1.c 8d7cd219c004cd2ced60659ebf045025cc5c16ce19d12459589dacd4310f7f07 F src/test2.c 7ebc518e6735939d8979273a6f7b1d9b5702babf059f6ad62499f7f60a9eb9a3 F src/test3.c e7573aa0f78ee4e070a4bc8c3493941c1aa64d5c66d4825c74c0f055451f432b F src/test4.c 13e57ae7ec7a959ee180970aef09deed141252fe9bb07c61054f0dfa4f1dfd5d @@ -1267,7 +1267,7 @@ F test/fuzz3.test 70ba57260364b83e964707b9d4b5625284239768ab907dd387c740c0370ce3 F test/fuzz4.test c229bcdb45518a89e1d208a21343e061503460ac69fae1539320a89f572eb634 F test/fuzz_common.tcl b7197de6ed1ee8250a4f82d67876f4561b42ee8cbbfc6160dcb66331bad3f830 F test/fuzz_malloc.test f348276e732e814802e39f042b1f6da6362a610af73a528d8f76898fde6b22f2 -F test/fuzzcheck.c 20be6c96bd0da2e91d25f089a037c1b3f8142211c97104f20629bf15610019e6 +F test/fuzzcheck.c 3b8b39e3c0c88422c51ef0a93481d3d528fb370668344bf0ae4c87629c18b021 F test/fuzzdata1.db 3e86d9cf5aea68ddb8e27c02d7dfdaa226347426c7eb814918e4d95475bf8517 F test/fuzzdata2.db 128b3feeb78918d075c9b14b48610145a0dd4c8d6f1ca7c2870c7e425f5bf31f F test/fuzzdata3.db c6586d3e3cef0fbc18108f9bb649aa77bfc38aba @@ -1279,7 +1279,7 @@ F test/fuzzdata8.db 4a53b6d077c6a5c23b609d8d3ac66996fa55ba3f8d02f9b6efdd0214a767 F test/fuzzer1.test 3d4c4b7e547aba5e5511a2991e3e3d07166cfbb8 F test/fuzzer2.test a85ef814ce071293bce1ad8dffa217cbbaad4c14 F test/fuzzerfault.test f64c4aef4c9e9edf1d6dc0d3f1e65dcc81e67c996403c88d14f09b74807a42bc -F test/fuzzinvariants.c 81167c9a7e82c0539a1d704aeb3384046d01f4108cda160a2447cb2a149d6362 +F test/fuzzinvariants.c 3de49c7b33f5641b67edc2496328a49af029738e92c8499fafbf8618ad42f68d F test/gcfault.test 4ea410ac161e685f17b19e1f606f58514a2850e806c65b846d05f60d436c5b0d F test/gencol1.test e169bdfa11c7ed5e9f322a98a7db3afe9e66235750b68c923efee8e1876b46ec F test/genesis.tcl 1e2e2e8e5cc4058549a154ff1892fe5c9de19f98 @@ -2212,8 +2212,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f69ef1a37b2778bdf73ee7e3b3edd74f7344ab8e5eedbedc22203c782e521f5b -R a9f753c3499efd243abe359303301f78 +P 224628b1039b996499e0d806fc0215f175da6f1059eb1b9ac491ac11126971ab +R 4ffcc50319bb317830ac49a5c5529b0f U drh -Z a70e5af6a81ae35c981b5946f602f67c +Z 1e521328c9a1b05b589836828e6bb245 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b195d89eaa..7d403270b8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -224628b1039b996499e0d806fc0215f175da6f1059eb1b9ac491ac11126971ab +60fb8ee153ec293b6b3a4170dafa305e4c16af575aced72daef46116d8dc2bb6 diff --git a/src/test1.c b/src/test1.c index 177091a0e1..38ea6229ee 100644 --- a/src/test1.c +++ b/src/test1.c @@ -2343,7 +2343,7 @@ static int SQLITE_TCLAPI test_stmt_scanstatus( } for(ii=0; ii<(int)nFlag; ii++){ int iVal = 0; - int res = Tcl_GetIndexFromObjStruct( + res = Tcl_GetIndexFromObjStruct( interp, aFlag[ii], aTbl, sizeof(aTbl[0]), "flag", 0, &iVal ); if( res ) return TCL_ERROR; diff --git a/test/fuzzcheck.c b/test/fuzzcheck.c index ea0d949b5b..f81e0982d9 100644 --- a/test/fuzzcheck.c +++ b/test/fuzzcheck.c @@ -1045,10 +1045,11 @@ static void bindDebugParameters(sqlite3_stmt *pStmt){ sqlite3_bind_int(pStmt, i+1, atoi(&zVar[5])); }else if( strncmp(zVar, "$text_", 6)==0 ){ - char *zBuf = sqlite3_malloc64( strlen(zVar)-5 ); + size_t szVar = strlen(zVar); + char *zBuf = sqlite3_malloc64( szVar-5 ); if( zBuf ){ - memcpy(zBuf, &zVar[6], strlen(zVar)-5); - sqlite3_bind_text64(pStmt, i+1, zBuf, -1, sqlite3_free, SQLITE_UTF8); + memcpy(zBuf, &zVar[6], szVar-5); + sqlite3_bind_text64(pStmt, i, zBuf, szVar-6, sqlite3_free, SQLITE_UTF8); } } } diff --git a/test/fuzzinvariants.c b/test/fuzzinvariants.c index 36121963f5..2f09c43ab6 100644 --- a/test/fuzzinvariants.c +++ b/test/fuzzinvariants.c @@ -54,10 +54,11 @@ static void bindDebugParameters(sqlite3_stmt *pStmt){ sqlite3_bind_int(pStmt, i+1, atoi(&zVar[5])); }else if( strncmp(zVar, "$text_", 6)==0 ){ - char *zBuf = sqlite3_malloc64( strlen(zVar)-5 ); + size_t szVar = strlen(zVar); + char *zBuf = sqlite3_malloc64( szVar-5 ); if( zBuf ){ - memcpy(zBuf, &zVar[6], strlen(zVar)-5); - sqlite3_bind_text64(pStmt, i+1, zBuf, -1, sqlite3_free, SQLITE_UTF8); + memcpy(zBuf, &zVar[6], szVar-5); + sqlite3_bind_text64(pStmt, i, zBuf, szVar-6, sqlite3_free, SQLITE_UTF8); } } } From c43d16d69ab2ee9eb4375d8e2e6892b5e58c5c4b Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 6 Sep 2024 13:13:25 +0000 Subject: [PATCH 19/36] In testrunner.tcl, correctly capture and display information about the system under test: hostname, OS, pointer size, byte-order. FossilOrigin-Name: ec75cfc5d4b69e4aed64d17748cac15cd62a759a1cbe7feaa4580ad8346b0b95 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/testrunner.tcl | 10 +++++----- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 238ef2d4dc..63f65d36d5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarnings. -D 2024-09-06T11:21:53.173 +C In\stestrunner.tcl,\scorrectly\scapture\sand\sdisplay\sinformation\sabout\sthe\nsystem\sunder\stest:\s\shostname,\sOS,\spointer\ssize,\sbyte-order. +D 2024-09-06T13:13:25.545 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1715,7 +1715,7 @@ F test/temptable2.test 76821347810ecc88203e6ef0dd6897b6036ac788e9dd3e6b04fd4d163 F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc F test/tester.tcl 7b44f1a9b9a2de8112695b908afc21dd9a68cd2d44e84b73f1b27b53492c0d59 -F test/testrunner.tcl a59e911e3732ba4d6b66e310f72d9599eb0392d3de659a525c519ba755cbf3e1 x +F test/testrunner.tcl 3dd75b45593d2afa2e3bca76121297a9f163bebb98474b13536f326829a71db1 x F test/testrunner_data.tcl dbc0bb1c5b912dfd1e32b25d544318e412edd6085bd5fc9e6619cb93a739b786 F test/thread001.test a0985c117eab62c0c65526e9fa5d1360dd1cac5b03bde223902763274ce21899 F test/thread002.test c24c83408e35ba5a952a3638b7ac03ccdf1ce4409289c54a050ac4c5f1de7502 @@ -2212,8 +2212,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 224628b1039b996499e0d806fc0215f175da6f1059eb1b9ac491ac11126971ab -R 4ffcc50319bb317830ac49a5c5529b0f +P 60fb8ee153ec293b6b3a4170dafa305e4c16af575aced72daef46116d8dc2bb6 +R 10b53f0d3edd4deb6d00308ae86e15b9 U drh -Z 1e521328c9a1b05b589836828e6bb245 +Z 1062f5bb698948cf0174718b6a1b925f # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 7d403270b8..e8fed7784f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -60fb8ee153ec293b6b3a4170dafa305e4c16af575aced72daef46116d8dc2bb6 +ec75cfc5d4b69e4aed64d17748cac15cd62a759a1cbe7feaa4580ad8346b0b95 diff --git a/test/testrunner.tcl b/test/testrunner.tcl index d28bdd4fba..a4a71a85e0 100755 --- a/test/testrunner.tcl +++ b/test/testrunner.tcl @@ -1282,12 +1282,11 @@ proc mark_job_as_finished {jobid output state endtm} { set ntest 1 set nerr 0 if {$endtm>0} { - if {[regexp {\y(\d+) errors out of (\d+) tests} $output all a b]} { + set re {\y(\d+) errors out of (\d+) tests( on [^\n]+\n)?} + if {[regexp $re $output all a b pltfm]} { set nerr $a set ntest $b } - regexp {\y\d+ errors out of \d+ tests (on [^\n]+-bit \S+-endian)} \ - $output all pltfm regexp {\ySQLite \d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d [0-9a-fA-F]+} \ $output svers } @@ -1298,6 +1297,7 @@ proc mark_job_as_finished {jobid output state endtm} { } else { set childstate ready } + if {[info exists pltfm]} {set pltfm [string trim $pltfm]} trdb eval { UPDATE jobs SET output=$output, state=$state, endtime=$endtm, @@ -1558,10 +1558,10 @@ proc run_testset {} { trdb eval { SELECT pltfm, count(*) FROM jobs WHERE pltfm IS NOT NULL ORDER BY 2 DESC LIMIT 1 - } {puts $pltfm} + } break puts "$totalerr errors out of $totaltest tests in $et $pltfm" trdb eval { - SELECT DISTINCT substr(svers,1,80) as v1 FROM jobs WHERE svers IS NOT NULL + SELECT DISTINCT substr(svers,1,79) as v1 FROM jobs WHERE svers IS NOT NULL } {puts $v1} } From bc976303137dd905dc610910d7154908d89c4159 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 6 Sep 2024 14:01:48 +0000 Subject: [PATCH 20/36] Fix a harmless stray carriage-return character. FossilOrigin-Name: 84a6c5f975de36ba93b888e01cc5188717b5644ab8384beb8fd8906bdfc8e227 --- ext/rbu/sqlite3rbu.c | 2 +- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ext/rbu/sqlite3rbu.c b/ext/rbu/sqlite3rbu.c index 8e29ab820f..36688925d6 100644 --- a/ext/rbu/sqlite3rbu.c +++ b/ext/rbu/sqlite3rbu.c @@ -5329,7 +5329,7 @@ static int rbuVfsCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){ */ static int rbuVfsGetLastError(sqlite3_vfs *pVfs, int a, char *b){ UNUSED_PARAMETER(pVfs); - UNUSED_PARAMETER(a ); + UNUSED_PARAMETER(a); UNUSED_PARAMETER(b); return 0; } diff --git a/manifest b/manifest index 63f65d36d5..b721afee72 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\stestrunner.tcl,\scorrectly\scapture\sand\sdisplay\sinformation\sabout\sthe\nsystem\sunder\stest:\s\shostname,\sOS,\spointer\ssize,\sbyte-order. -D 2024-09-06T13:13:25.545 +C Fix\sa\sharmless\sstray\scarriage-return\scharacter. +D 2024-09-06T14:01:48.949 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -484,7 +484,7 @@ F ext/rbu/rbuvacuum.test 542561741ff2b262e3694bc6012b44694ee62c545845319a06f3237 F ext/rbu/rbuvacuum2.test ae097d04feb041446a74fac94b24bffeb3fdd60e32b848c5611e507ab702b81b F ext/rbu/rbuvacuum3.test 3ce42695fdf21aaa3499e857d7d4253bc499ad759bcd6c9362042c13cd37d8de F ext/rbu/rbuvacuum4.test ffccd22f67e2d0b380d2889685742159dfe0d19a3880ca3d2d1d69eefaebb205 -F ext/rbu/sqlite3rbu.c d90c79c2e9998e0f54da86d28c8433db7997864279fcfce02592e0838c8ce71d +F ext/rbu/sqlite3rbu.c c07817e89477b8fc286ab6ed87da5bc82fc3490bbbe9e9b22eb2d900e81ee5dc F ext/rbu/sqlite3rbu.h 9d923eb135c5d04aa6afd7c39ca47b0d1d0707c100e02f19fdde6a494e414304 F ext/rbu/test_rbu.c b9727c3394307d058e806c1da0f8bb7b24daf3c6bb94cb10cca88ea4d5c806c0 F ext/recover/dbdata.c 5295f4f922b60d7035b6b9fd5846b13071b9d97ed7fad8496837bb7640d24771 @@ -2212,8 +2212,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 60fb8ee153ec293b6b3a4170dafa305e4c16af575aced72daef46116d8dc2bb6 -R 10b53f0d3edd4deb6d00308ae86e15b9 +P ec75cfc5d4b69e4aed64d17748cac15cd62a759a1cbe7feaa4580ad8346b0b95 +R fff388cd7307c5b0da6a2a2532ba0d3f U drh -Z 1062f5bb698948cf0174718b6a1b925f +Z 8d9185ba5e8e9cd8d975e704cf12a0ed # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index e8fed7784f..104d10f3e6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ec75cfc5d4b69e4aed64d17748cac15cd62a759a1cbe7feaa4580ad8346b0b95 +84a6c5f975de36ba93b888e01cc5188717b5644ab8384beb8fd8906bdfc8e227 From 6946a7fa508f4be45f2d32b5ebc2a9a31a30a92c Mon Sep 17 00:00:00 2001 From: stephan Date: Fri, 6 Sep 2024 15:01:00 +0000 Subject: [PATCH 21/36] Resolve the "No rule to make target '0'" error when building with --disable-tcl. FossilOrigin-Name: d94541ae76b5d8b69f5524f10dcccc0814283f438a03f553848ed631a1983633 --- Makefile.in | 14 ++++++++++++-- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/Makefile.in b/Makefile.in index cc75000033..9036babcb2 100644 --- a/Makefile.in +++ b/Makefile.in @@ -661,11 +661,16 @@ SQLITE3_SHELL_TARGET_ = sqlite3$(TEXE) SQLITE3_SHELL_TARGET_1 = SQLITE3_SHELL_TARGET = $(SQLITE3_SHELL_TARGET_@HAVE_WASI_SDK@) +# Use $(libtclsqlite3.la_$(HAVE_TCL)) to resolve to either +# libtclsqlite3.la or an empty value. +libtclsqlite3.la_0 = +libtclsqlite3.la_1 = libtclsqlite3.la + # This is the default Makefile target. The objects listed here # are what get build when you type just "make" with no arguments. # all: sqlite3.h libsqlite3.la $(SQLITE3_SHELL_TARGET) \ - $(HAVE_TCL:1=libtclsqlite3.la) + $(libtclsqlite3.la_$(HAVE_TCL)) Makefile: $(TOP)/Makefile.in ./config.status @@ -1566,7 +1571,12 @@ lib_install: libsqlite3.la $(INSTALL) -d $(DESTDIR)$(libdir) $(LTINSTALL) libsqlite3.la $(DESTDIR)$(libdir) -install: sqlite3$(TEXE) lib_install sqlite3.h sqlite3.pc ${HAVE_TCL:1=tcl_install} +# Use $(tcl_install_$(HAVE_TCL)) to resolve to either tcl_install or +# an empty value. +tcl_install_0 = +tcl_install_1 = tcl_install + +install: sqlite3$(TEXE) lib_install sqlite3.h sqlite3.pc $(tcl_install_$(HAVE_TCL)) $(INSTALL) -d $(DESTDIR)$(bindir) $(LTINSTALL) sqlite3$(TEXE) $(DESTDIR)$(bindir) $(INSTALL) -d $(DESTDIR)$(includedir) diff --git a/manifest b/manifest index b721afee72..9e6ae9fd87 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Fix\sa\sharmless\sstray\scarriage-return\scharacter. -D 2024-09-06T14:01:48.949 +C Resolve\sthe\s"No\srule\sto\smake\starget\s'0'"\serror\swhen\sbuilding\swith\s--disable-tcl. +D 2024-09-06T15:01:00.761 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 -F Makefile.in 098d9814c66d81a8bbfb3550876b7d9695a0b050c0d4680afb4931a7aa8fe7f1 +F Makefile.in c41539ec7e8e69091e74c7d9e011115ae5e81d1f53829e640d299f3a754f94a4 F Makefile.linux-gcc f3842a0b1efbfbb74ac0ef60e56b301836d05b4d867d014f714fa750048f1ab6 F Makefile.msc e72a4653ea8f48efd14c7dac1e7de308a1f3cc91f905f15cbcacc680301c8f03 F README.md c3c0f19532ce28f6297a71870f3c7b424729f0e6d9ab889616d3587dd2332159 @@ -2212,8 +2212,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ec75cfc5d4b69e4aed64d17748cac15cd62a759a1cbe7feaa4580ad8346b0b95 -R fff388cd7307c5b0da6a2a2532ba0d3f -U drh -Z 8d9185ba5e8e9cd8d975e704cf12a0ed +P 84a6c5f975de36ba93b888e01cc5188717b5644ab8384beb8fd8906bdfc8e227 +R b7c82ea9b1442aabb45d735bf528bcb3 +U stephan +Z 5b4290c809ba1bac71c5f3f2e0536ccb # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 104d10f3e6..e5b5999a3e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -84a6c5f975de36ba93b888e01cc5188717b5644ab8384beb8fd8906bdfc8e227 +d94541ae76b5d8b69f5524f10dcccc0814283f438a03f553848ed631a1983633 From f1c750e4ca012e6cecf808363717910685724ff7 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 6 Sep 2024 20:12:59 +0000 Subject: [PATCH 22/36] Add tests for an fts5 NEAR() expression with a single argument phrase. FossilOrigin-Name: e319d43bfd5ee4ed92b93531b239af4d1be0a8215b2a06c3532122ff2c7b6a7c --- ext/fts5/test/fts5phrase.test | 16 +++++++++++----- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/ext/fts5/test/fts5phrase.test b/ext/fts5/test/fts5phrase.test index ea425a4dd6..708cdfd83e 100644 --- a/ext/fts5/test/fts5phrase.test +++ b/ext/fts5/test/fts5phrase.test @@ -93,15 +93,21 @@ foreach {tn cols tokens} { 10 {b} "i e" 11 {a} "i e" } { - set fts "{$cols}:[join $tokens +]" set where [list] foreach c $cols { lappend where "pmatch($c, '$tokens')" } set where [join $where " OR "] - set res [db eval "SELECT rowid FROM t3 WHERE $where"] - do_execsql_test "1.$tn.$fts->([llength $res] rows)" { - SELECT rowid FROM t3($fts) - } $res + foreach fts [list \ + "{$cols}:[join $tokens +]" \ + "{$cols}:NEAR([join $tokens +])" \ + "{$cols}:NEAR([join $tokens +],1)" \ + "{$cols}:NEAR([join $tokens +],111)" \ + ] { + set res [db eval "SELECT rowid FROM t3 WHERE $where"] + do_execsql_test "1.$tn.$fts->([llength $res] rows)" { + SELECT rowid FROM t3($fts) + } $res + } } do_execsql_test 2.0 { diff --git a/manifest b/manifest index 9e6ae9fd87..9672a951bf 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Resolve\sthe\s"No\srule\sto\smake\starget\s'0'"\serror\swhen\sbuilding\swith\s--disable-tcl. -D 2024-09-06T15:01:00.761 +C Add\stests\sfor\san\sfts5\sNEAR()\sexpression\swith\sa\ssingle\sargument\sphrase. +D 2024-09-06T20:12:59.105 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -206,7 +206,7 @@ F ext/fts5/test/fts5origintext2.test f4505ff79bf7369f2b8b10b9cef7476049d844e20b3 F ext/fts5/test/fts5origintext3.test 45c33cf0c91a9ca0e36d298462db3edc7c8fe45fd185649a9dbfd66bb670058b F ext/fts5/test/fts5origintext4.test 0d3ef0a8038f471dbc83001c34fe5f7ae39b571bfc209670771eb28bc0fc50e8 F ext/fts5/test/fts5origintext5.test ee12b440ec335e5b422d1668aca0051b52ff28b6ee67073e8bbc29f509fd562b -F ext/fts5/test/fts5phrase.test 6260b8be40d51ec287992cd983a5e58a9be92e5dc6e01d48cbce1ad0e95a99d8 +F ext/fts5/test/fts5phrase.test bb2554bb61d15f859678c96dc89a7de415cd5fc3b7b54c29b82a0d0ad138091c F ext/fts5/test/fts5plan.test f8b0d752a818059a934cdc96c0f77de058a67a0a57bb3a8181d28307ab5b1626 F ext/fts5/test/fts5porter.test 15b514fac8690b58e99c330efe5bf5615bc43f2fae4a3cca3f923dbaff55a0c0 F ext/fts5/test/fts5porter2.test 94f0e4351e2c99b4e74f1fae05a4ddf1cb5b926620a8c14554160d075ddc7a59 @@ -2212,8 +2212,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 84a6c5f975de36ba93b888e01cc5188717b5644ab8384beb8fd8906bdfc8e227 -R b7c82ea9b1442aabb45d735bf528bcb3 -U stephan -Z 5b4290c809ba1bac71c5f3f2e0536ccb +P d94541ae76b5d8b69f5524f10dcccc0814283f438a03f553848ed631a1983633 +R 0ff6a7b291ffdc8dd8ae89040689ac04 +U dan +Z 0535d08aac8a0a882873604dcc93ce4a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index e5b5999a3e..245b18e39e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d94541ae76b5d8b69f5524f10dcccc0814283f438a03f553848ed631a1983633 +e319d43bfd5ee4ed92b93531b239af4d1be0a8215b2a06c3532122ff2c7b6a7c From 8755e695c53dec94d7cbb9c96d2db3246a3786f7 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 7 Sep 2024 16:04:04 +0000 Subject: [PATCH 23/36] Fix an off-by-one error in the routines that bind the special $test_TTT and $int_NNN parameters for fuzz testing. Fix to testing logic only - no changes to the SQLite core. FossilOrigin-Name: 6206b90a4ec3f05e3bbb4844e71569bbde7df237550569e6419ff7c3146505dc --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- test/fuzzcheck.c | 6 +++--- test/fuzzinvariants.c | 6 +++--- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 9672a951bf..6d4099c51d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stests\sfor\san\sfts5\sNEAR()\sexpression\swith\sa\ssingle\sargument\sphrase. -D 2024-09-06T20:12:59.105 +C Fix\san\soff-by-one\serror\sin\sthe\sroutines\sthat\sbind\sthe\sspecial\s$test_TTT\sand\n$int_NNN\sparameters\sfor\sfuzz\stesting.\s\sFix\sto\stesting\slogic\sonly\s-\sno\schanges\nto\sthe\sSQLite\score. +D 2024-09-07T16:04:04.674 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1267,7 +1267,7 @@ F test/fuzz3.test 70ba57260364b83e964707b9d4b5625284239768ab907dd387c740c0370ce3 F test/fuzz4.test c229bcdb45518a89e1d208a21343e061503460ac69fae1539320a89f572eb634 F test/fuzz_common.tcl b7197de6ed1ee8250a4f82d67876f4561b42ee8cbbfc6160dcb66331bad3f830 F test/fuzz_malloc.test f348276e732e814802e39f042b1f6da6362a610af73a528d8f76898fde6b22f2 -F test/fuzzcheck.c 3b8b39e3c0c88422c51ef0a93481d3d528fb370668344bf0ae4c87629c18b021 +F test/fuzzcheck.c 89b71d92b150a532e945e489d6e0721a4b15353c9255e079c198ed2a1958018b F test/fuzzdata1.db 3e86d9cf5aea68ddb8e27c02d7dfdaa226347426c7eb814918e4d95475bf8517 F test/fuzzdata2.db 128b3feeb78918d075c9b14b48610145a0dd4c8d6f1ca7c2870c7e425f5bf31f F test/fuzzdata3.db c6586d3e3cef0fbc18108f9bb649aa77bfc38aba @@ -1279,7 +1279,7 @@ F test/fuzzdata8.db 4a53b6d077c6a5c23b609d8d3ac66996fa55ba3f8d02f9b6efdd0214a767 F test/fuzzer1.test 3d4c4b7e547aba5e5511a2991e3e3d07166cfbb8 F test/fuzzer2.test a85ef814ce071293bce1ad8dffa217cbbaad4c14 F test/fuzzerfault.test f64c4aef4c9e9edf1d6dc0d3f1e65dcc81e67c996403c88d14f09b74807a42bc -F test/fuzzinvariants.c 3de49c7b33f5641b67edc2496328a49af029738e92c8499fafbf8618ad42f68d +F test/fuzzinvariants.c 057e910241d85aa4aaf75cef1a7adc45c632b173288d07d9dbbef4e6bda83d5a F test/gcfault.test 4ea410ac161e685f17b19e1f606f58514a2850e806c65b846d05f60d436c5b0d F test/gencol1.test e169bdfa11c7ed5e9f322a98a7db3afe9e66235750b68c923efee8e1876b46ec F test/genesis.tcl 1e2e2e8e5cc4058549a154ff1892fe5c9de19f98 @@ -2212,8 +2212,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d94541ae76b5d8b69f5524f10dcccc0814283f438a03f553848ed631a1983633 -R 0ff6a7b291ffdc8dd8ae89040689ac04 -U dan -Z 0535d08aac8a0a882873604dcc93ce4a +P e319d43bfd5ee4ed92b93531b239af4d1be0a8215b2a06c3532122ff2c7b6a7c +R 2c4135598e1fecddc6fdbdd8727336d2 +U drh +Z 73b413ffeaf70ba5cf262762754b392b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 245b18e39e..78ce5b184c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e319d43bfd5ee4ed92b93531b239af4d1be0a8215b2a06c3532122ff2c7b6a7c +6206b90a4ec3f05e3bbb4844e71569bbde7df237550569e6419ff7c3146505dc diff --git a/test/fuzzcheck.c b/test/fuzzcheck.c index f81e0982d9..9f339096bc 100644 --- a/test/fuzzcheck.c +++ b/test/fuzzcheck.c @@ -1038,11 +1038,11 @@ static int recoverDatabase(sqlite3 *db){ static void bindDebugParameters(sqlite3_stmt *pStmt){ int nVar = sqlite3_bind_parameter_count(pStmt); int i; - for(i=0; i Date: Sat, 7 Sep 2024 16:22:22 +0000 Subject: [PATCH 24/36] Fix a problem with fts5 locale=1 tables and UPDATE statements that may affect more than one row. FossilOrigin-Name: 70e42f941c0778a04b82655409c7caf4c1039589f7e43a8ec1e736ea8f931b26 --- ext/fts5/fts5Int.h | 2 + ext/fts5/fts5_main.c | 172 ++++++++++++++++++++-------------- ext/fts5/fts5_storage.c | 21 +++-- ext/fts5/test/fts5locale.test | 30 +++++- manifest | 22 ++--- manifest.uuid | 2 +- test/fts3corrupt4.test | 2 +- 7 files changed, 154 insertions(+), 97 deletions(-) diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index 7e41119572..0b8851d227 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -647,6 +647,8 @@ int sqlite3Fts5ExtractText( void sqlite3Fts5ClearLocale(Fts5Config *pConfig); +int sqlite3Fts5IsLocaleValue(Fts5Config *pConfig, sqlite3_value *pVal); + /* ** End of interface to code in fts5.c. **************************************************************************/ diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index 03c1bb83fa..cb68c1444f 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -83,8 +83,17 @@ struct Fts5Global { Fts5TokenizerModule *pTok; /* First in list of all tokenizer modules */ Fts5TokenizerModule *pDfltTok; /* Default tokenizer module */ Fts5Cursor *pCsr; /* First in list of all open cursors */ + u32 aLocaleHdr[4]; }; +/* +** Size of header on fts5_locale() values. And macro to access a buffer +** containing a copy of the header from an Fts5Config pointer. +*/ +#define FTS5_LOCALE_HDR_SIZE sizeof( ((Fts5Global*)0)->aLocaleHdr ) +#define FTS5_LOCALE_HDR(pConfig) ((const u8*)(pConfig->pGlobal->aLocaleHdr)) + + /* ** Each auxiliary function registered with the FTS5 module is represented ** by an object of the following type. All such objects are stored as part @@ -247,12 +256,6 @@ struct Fts5Cursor { #define BitFlagAllTest(x,y) (((x) & (y))==(y)) #define BitFlagTest(x,y) (((x) & (y))!=0) -/* -** The subtype value and header bytes used by fts5_locale(). -*/ -#define FTS5_LOCALE_SUBTYPE ((unsigned int)'L') -#define FTS5_LOCALE_HEADER "\x00\xE0\xB2\xEB" - /* ** Macros to Set(), Clear() and Test() cursor flags. @@ -1274,6 +1277,22 @@ void sqlite3Fts5ClearLocale(Fts5Config *pConfig){ fts5SetLocale(pConfig, 0, 0); } +/* +** Return true if the value passed as the only argument is an +** fts5_locale() value. +*/ +int sqlite3Fts5IsLocaleValue(Fts5Config *pConfig, sqlite3_value *pVal){ + int ret = 0; + if( sqlite3_value_type(pVal)==SQLITE_BLOB ){ + if( sqlite3_value_bytes(pVal)>FTS5_LOCALE_HDR_SIZE + && 0==memcmp(sqlite3_value_blob(pVal), FTS5_LOCALE_HDR(pConfig), 4) + ){ + ret = 1; + } + } + return ret; +} + /* ** This function is used to extract utf-8 text from an sqlite3_value. This ** is usually done in order to tokenize it. For example, when: @@ -1292,17 +1311,15 @@ void sqlite3Fts5ClearLocale(Fts5Config *pConfig){ ** 2) Combination text/locale blobs created by fts5_locale(). There ** are several cases for these: ** -** * Blobs tagged with FTS5_LOCALE_SUBTYPE. -** * Blobs read from the content table of a locale=1 external-content -** table, and +** * Blobs that have the 16-byte header, and ** * Blobs read from the content table of a locale=1 regular ** content table. ** -** The first two cases above should have the 4 byte FTS5_LOCALE_HEADER -** header. It is an error if a blob with the subtype or a blob read -** from the content table of an external content table does not have -** the required header. A blob read from the content table of a regular -** locale=1 table does not have the header. This is to save space. +** The first case above has the 16 byte FTS5_LOCALE_HDR(pConfig) +** header. It is an error if a blob read from the content table of +** an external content table does not have the required header. A blob +** read from the content table of a regular locale=1 table does not +** have the header. This is to save space. ** ** If successful, SQLITE_OK is returned and output parameters (*ppText) ** and (*pnText) are set to point to a buffer containing the extracted utf-8 @@ -1330,53 +1347,54 @@ int sqlite3Fts5ExtractText( const char *pText = 0; int nText = 0; int rc = SQLITE_OK; - int bDecodeBlob = 0; + + /* 0: Do not decode blob + ** 1: Decode blob, expect fts5_locale() header + ** 2: Decode blob, expect no fts5_locale() header + */ + int eDecodeBlob = 0; assert( pbResetTokenizer==0 || *pbResetTokenizer==0 ); assert( bContent==0 || pConfig->eContent!=FTS5_CONTENT_NONE ); - assert( bContent==0 || sqlite3_value_subtype(pVal)==0 ); if( sqlite3_value_type(pVal)==SQLITE_BLOB ){ - if( sqlite3_value_subtype(pVal)==FTS5_LOCALE_SUBTYPE - || (bContent && pConfig->bLocale) + if( bContent + && pConfig->bLocale + && pConfig->eContent==FTS5_CONTENT_NORMAL ){ - bDecodeBlob = 1; + eDecodeBlob = 2; + }else if( sqlite3Fts5IsLocaleValue(pConfig, pVal) ){ + eDecodeBlob = 1; + }else if( bContent && pConfig->bLocale ){ + return SQLITE_ERROR; } } - if( bDecodeBlob ){ - const int SZHDR = sizeof(FTS5_LOCALE_HEADER)-1; + if( eDecodeBlob ){ const u8 *pBlob = sqlite3_value_blob(pVal); int nBlob = sqlite3_value_bytes(pVal); + int nLocale = 0; /* Unless this blob was read from the %_content table of an ** FTS5_CONTENT_NORMAL table, it should have the 4 byte fts5_locale() ** header. Check for this. If it is not found, return an error. */ - if( (!bContent || pConfig->eContent!=FTS5_CONTENT_NORMAL) ){ - if( nBlobnCol; ii++){ - if( sqlite3_value_type(apVal[ii+2])==SQLITE_BLOB ){ - int bSub = (sqlite3_value_subtype(apVal[ii+2])==FTS5_LOCALE_SUBTYPE); - if( (pConfig->bLocale && !bSub && pConfig->abUnindexed[ii]==0) - || (pConfig->bLocale==0 && bSub) - ){ - if( pConfig->bLocale==0 ){ - fts5SetVtabError(pTab, "fts5_locale() requires locale=1"); + sqlite3_value *pVal = apVal[ii+2]; + if( sqlite3_value_type(pVal)==SQLITE_BLOB ){ + int isLocale = sqlite3Fts5IsLocaleValue(pConfig, pVal); + if( pConfig->bLocale ){ + if( isLocale==0 && pConfig->abUnindexed[ii]==0 ){ + rc = SQLITE_MISMATCH; + goto update_out; + } + }else{ + if( isLocale ){ + fts5SetVtabError(pTab, "fts5_locale() requires locale=1"); + rc = SQLITE_MISMATCH; + goto update_out; } - rc = SQLITE_MISMATCH; - goto update_out; } } } @@ -2716,21 +2735,21 @@ static int fts5ApiColumnLocale( /* Load the value into pVal. pVal is a locale/text pair iff: ** ** 1) It is an SQLITE_BLOB, and - ** 2) Either the subtype is FTS5_LOCALE_SUBTYPE, or else the - ** value was loaded from an FTS5_CONTENT_NORMAL table, and - ** 3) It does not begin with an 0x00 byte. + ** 2) Either the FTS5_LOCALE_HDR header is present, or else the + ** value was loaded from an FTS5_CONTENT_NORMAL table. + ** + ** If condition (1) is met but condition (2) is not, it is an error. */ sqlite3_value *pVal = sqlite3_column_value(pCsr->pStmt, iCol+1); if( sqlite3_value_type(pVal)==SQLITE_BLOB ){ const u8 *pBlob = (const u8*)sqlite3_value_blob(pVal); int nBlob = sqlite3_value_bytes(pVal); if( pConfig->eContent==FTS5_CONTENT_EXTERNAL ){ - const int SZHDR = sizeof(FTS5_LOCALE_HEADER)-1; - if( nBlobabUnindexed[iCol]==0 ){ - const int SZHDR = sizeof(FTS5_LOCALE_HEADER)-1; const u8 *pBlob = sqlite3_value_blob(pVal); int nBlob = sqlite3_value_bytes(pVal); int ii; if( pConfig->eContent==FTS5_CONTENT_EXTERNAL ){ - if( nBlobaLocaleHdr, FTS5_LOCALE_HDR_SIZE); + pCsr += FTS5_LOCALE_HDR_SIZE; memcpy(pCsr, zLocale, nLocale); pCsr += nLocale; (*pCsr++) = 0x00; @@ -3657,7 +3676,6 @@ static void fts5LocaleFunc( assert( &pCsr[nText]==&pBlob[nBlob] ); sqlite3_result_blob(pCtx, pBlob, nBlob, sqlite3_free); - sqlite3_result_subtype(pCtx, FTS5_LOCALE_SUBTYPE); } } @@ -3759,6 +3777,16 @@ static int fts5Init(sqlite3 *db){ pGlobal->api.xFindTokenizer = fts5FindTokenizer; pGlobal->api.xCreateTokenizer_v2 = fts5CreateTokenizer_v2; pGlobal->api.xFindTokenizer_v2 = fts5FindTokenizer_v2; + + /* Initialize pGlobal->aLocaleHdr[] to a 128-bit pseudo-random vector. + ** The constants below were generated randomly. */ + sqlite3_randomness(sizeof(pGlobal->aLocaleHdr), pGlobal->aLocaleHdr); + pGlobal->aLocaleHdr[0] ^= 0xF924976D; + pGlobal->aLocaleHdr[1] ^= 0x16596E13; + pGlobal->aLocaleHdr[2] ^= 0x7C80BEAA; + pGlobal->aLocaleHdr[3] ^= 0x9B03A67F; + assert( sizeof(pGlobal->aLocaleHdr)==16 ); + rc = sqlite3_create_module_v2(db, "fts5", &fts5Mod, p, fts5ModuleDestroy); if( rc==SQLITE_OK ) rc = sqlite3Fts5IndexInit(db); if( rc==SQLITE_OK ) rc = sqlite3Fts5ExprInit(pGlobal, db); diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c index cf25eb361e..3bf20b3390 100644 --- a/ext/fts5/fts5_storage.c +++ b/ext/fts5/fts5_storage.c @@ -891,23 +891,26 @@ int sqlite3Fts5StorageContentInsert( /* This is an UPDATE statement, and column (i-2) was not modified. ** Retrieve the value from Fts5Storage.pSavedRow instead. */ pVal = sqlite3_column_value(p->pSavedRow, i-1); - }else if( sqlite3_value_subtype(pVal)==FTS5_LOCALE_SUBTYPE ){ + }else if( sqlite3_value_type(pVal)==SQLITE_BLOB && pConfig->bLocale ){ assert( pConfig->bLocale ); assert( i>1 ); if( pConfig->abUnindexed[i-2] ){ - /* At attempt to insert an fts5_locale() value into an UNINDEXED - ** column. Strip the locale away and just bind the text. */ - const char *pText = 0; - int nText = 0; - rc = sqlite3Fts5ExtractText(pConfig, pVal, 0, 0, &pText, &nText); - sqlite3_bind_text(pInsert, i, pText, nText, SQLITE_TRANSIENT); + if( sqlite3Fts5IsLocaleValue(pConfig, pVal) ){ + /* At attempt to insert an fts5_locale() value into an UNINDEXED + ** column. Strip the locale away and just bind the text. */ + const char *pText = 0; + int nText = 0; + rc = sqlite3Fts5ExtractText(pConfig, pVal, 0, 0, &pText, &nText); + sqlite3_bind_text(pInsert, i, pText, nText, SQLITE_TRANSIENT); + continue; + } }else{ const u8 *pBlob = (const u8*)sqlite3_value_blob(pVal); int nBlob = sqlite3_value_bytes(pVal); assert( nBlob>4 ); - sqlite3_bind_blob(pInsert, i, pBlob+4, nBlob-4, SQLITE_TRANSIENT); + sqlite3_bind_blob(pInsert, i, pBlob+16, nBlob-16, SQLITE_TRANSIENT); + continue; } - continue; } rc = sqlite3_bind_value(pInsert, i, pVal); diff --git a/ext/fts5/test/fts5locale.test b/ext/fts5/test/fts5locale.test index 684dcecd80..f0df4969dc 100644 --- a/ext/fts5/test/fts5locale.test +++ b/ext/fts5/test/fts5locale.test @@ -488,7 +488,7 @@ foreach_detail_mode $::testprefix { do_catchsql_test 10.2.$tn.4 " SELECT * FROM ft( test_setsubtype($v, 76) ); - " {1 {SQL logic error}} + " {1 {fts5: syntax error near ""}} do_execsql_test 10.2.$tn.5 { INSERT INTO ft(rowid, x) VALUES(1, 'hello world'); @@ -523,11 +523,11 @@ foreach_detail_mode $::testprefix { do_catchsql_test 10.2.$tn.11 " INSERT INTO ft(ft, rowid, x) VALUES('delete', 1, test_setsubtype($v,76) ) - " {1 {SQL logic error}} + " {0 {}} do_catchsql_test 10.2.$tn.12 " INSERT INTO ft(rowid, x) VALUES(2, test_setsubtype($v,76) ) - " {1 {SQL logic error}} + " {1 {datatype mismatch}} do_execsql_test 10.2.$tn.13 { INSERT INTO ft2(rowid, x) VALUES(1, 'hello world'); @@ -663,5 +663,29 @@ do_catchsql_test 13.2.7 { FROM ft('one AND three') ORDER BY rowid } {1 {non-integer argument passed to function fts5_get_locale()}} +#------------------------------------------------------------------------- +# Check that UPDATE statements that may affect more than one row work. +# +reset_db +do_execsql_test 14.1 { + CREATE VIRTUAL TABLE ft USING fts5(a, b, locale=1); +} + +do_execsql_test 14.2 { + INSERT INTO ft VALUES('hello', 'world'); +} + +do_execsql_test 14.3 { + UPDATE ft SET b = fts5_locale('en_AU', 'world'); +} + +do_catchsql_test 14.4 { + INSERT INTO ft VALUES(X'abcd', X'1234'); +} {1 {datatype mismatch}} + +do_execsql_test 14.4 { + SELECT * FROM ft +} {hello world} + finish_test diff --git a/manifest b/manifest index 6d4099c51d..037c1b19a0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\soff-by-one\serror\sin\sthe\sroutines\sthat\sbind\sthe\sspecial\s$test_TTT\sand\n$int_NNN\sparameters\sfor\sfuzz\stesting.\s\sFix\sto\stesting\slogic\sonly\s-\sno\schanges\nto\sthe\sSQLite\score. -D 2024-09-07T16:04:04.674 +C Fix\sa\sproblem\swith\sfts5\slocale=1\stables\sand\sUPDATE\sstatements\sthat\smay\saffect\smore\sthan\sone\srow. +D 2024-09-07T16:22:22.943 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -93,15 +93,15 @@ F ext/fts3/unicode/mkunicode.tcl 63db9624ccf70d4887836c320eda93ab552f21008f3be7e F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb F ext/fts5/extract_api_docs.tcl 009cf59c77afa86d137b0cca3e3b1a5efbe2264faa2df233f9a7aa8563926d15 F ext/fts5/fts5.h efaaac0df3d3bc740383044c144b582f47921aafa21d7b10eb98f42c24c740b0 -F ext/fts5/fts5Int.h 26a71a09cefa4ef6b4516b204ed48da3e1380970a19b3482eea7c5d805655360 +F ext/fts5/fts5Int.h 7ab1d838adc4f22fdad5e1ba19182d6899ebded1d3ecadbe995322b0f0de7b9f F ext/fts5/fts5_aux.c 65a0468dd177d6093aa9ae1622e6d86b0136b8d267c62c0ad6493ad1e9a3d759 F ext/fts5/fts5_buffer.c 0eec58bff585f1a44ea9147eae5da2447292080ea435957f7488c70673cb6f09 F ext/fts5/fts5_config.c 353d2a0d12678cae6ab5b9ce54aed8dac0825667b69248b5a4ed81cbefc109ea F ext/fts5/fts5_expr.c 9a56f53700d1860f0ee2f373c2b9074eaf2a7aa0637d0e27a6476de26a3fee33 F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 F ext/fts5/fts5_index.c 571483823193f09439356741669aa8c81da838ae6f5e1bfa7517f7ee2fb3addd -F ext/fts5/fts5_main.c 1fddb53f495425d9314c74b30c5848a9dd254be0e5f445bfe38292d5ab21c288 -F ext/fts5/fts5_storage.c 9a9b880be12901f1962ae2a5a7e1b74348b3099a1e728764e419f75d98e3e612 +F ext/fts5/fts5_main.c c9c5fcce73ad05ef6abc4f69b9ade54093b2a6cb8ceb3ef647bd2e0d5f93b628 +F ext/fts5/fts5_storage.c 42cde97eb7d8506a8d2c7ea80b292fc3017b1f5469e1acb0035a69c345e6cf71 F ext/fts5/fts5_tcl.c 4db9258a7882c5eac0da4433042132aaf15b87dd1e1636c7a6ca203abd2c8bfe F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee F ext/fts5/fts5_test_tok.c 3cb0a9b508b30d17ef025ccddd26ae3dc8ddffbe76c057616e59a9aa85d36f3b @@ -189,7 +189,7 @@ F ext/fts5/test/fts5interrupt.test 20d04204d3e341b104c0c24a41596b6393a3a81eba104 F ext/fts5/test/fts5lastrowid.test f36298a1fb9f988bde060a274a7ce638faa9c38a31400f8d2d27ea9373e0c4a1 F ext/fts5/test/fts5leftjoin.test c0b4cafb9661379e576dc4405c0891d8fcc2782680740513c4d1fc114b43d4ad F ext/fts5/test/fts5limits.test 8ab67cf5d311c124b6ceb0062d0297767176df4572d955fce79fa43004dff01c -F ext/fts5/test/fts5locale.test 797cf6f5e017462ab11313ce884b9f1df8ff063811e74ef42190cd19ed6b600b +F ext/fts5/test/fts5locale.test 58ce0515c4f49cbb9905e3711168050d58fc184daf885c9ef7483e20aab63e5a F ext/fts5/test/fts5matchinfo.test 877520582feb86bbfd95ab780099bcba4526f18ac75ee34979144cf86ba3a5a3 F ext/fts5/test/fts5merge.test 2654df0bcdb2d117c2d38b6aeb0168061be01c643f9e9194b36c43a2970e8082 F ext/fts5/test/fts5merge2.test 3ebad1a59d6ad3fb66eff6523a09e95dc6367cbefb3cd73196801dea0425c8e2 @@ -1184,7 +1184,7 @@ F test/fts3conf.test c9cd45433b6787d48a43e84949aa2eb8b3b3d242bac7276731c1476290d F test/fts3corrupt.test 6732477c5ace050c5758a40a8b5706c8c0cccd416b9c558e0e15224805a40e57 F test/fts3corrupt2.test e318f0676e5e78d5a4b702637e2bb25265954c08a1b1e4aaf93c7880bb0c67d0 F test/fts3corrupt3.test 0d5b69a0998b4adf868cc301fc78f3d0707745f1d984ce044c205cdb764b491f -F test/fts3corrupt4.test 48bd57baed9654e511709a02dbef2d22ee54c012ad466e8648f0f825233faa08 +F test/fts3corrupt4.test 294684add5f235ea8a77a350b66eb74a80ac8ecee6ac38c07885348f5fb2e233 F test/fts3corrupt5.test 0549f85ec4bd22e992f645f13c59b99d652f2f5e643dac75568bfd23a6db7ed5 F test/fts3corrupt6.test f417c910254f32c0bc9ead7affa991a1d5aec35b3b32a183ffb05eea78289525 F test/fts3cov.test 7eacdbefd756cfa4dc2241974e3db2834e9b372ca215880e00032222f32194cf @@ -2212,8 +2212,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e319d43bfd5ee4ed92b93531b239af4d1be0a8215b2a06c3532122ff2c7b6a7c -R 2c4135598e1fecddc6fdbdd8727336d2 -U drh -Z 73b413ffeaf70ba5cf262762754b392b +P 6206b90a4ec3f05e3bbb4844e71569bbde7df237550569e6419ff7c3146505dc +R a6acb083cae329c41bdac6bb67941a5b +U dan +Z 2566687c7d077d4a754f89b771fd38fa # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 78ce5b184c..73e0954b37 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6206b90a4ec3f05e3bbb4844e71569bbde7df237550569e6419ff7c3146505dc +70e42f941c0778a04b82655409c7caf4c1039589f7e43a8ec1e736ea8f931b26 diff --git a/test/fts3corrupt4.test b/test/fts3corrupt4.test index 433a486359..7a9ab3fbd0 100644 --- a/test/fts3corrupt4.test +++ b/test/fts3corrupt4.test @@ -4404,7 +4404,7 @@ do_catchsql_test 25.5 { do_catchsql_test 25.6 { INSERT INTO t1(t1) SELECT x FROM t2; INSERT INTO t1(t1) SELECT x FROM t2; -} {1 {database disk image is malformed}} +} {0 {}} #------------------------------------------------------------------------- reset_db From 5600adf1b15a66b3f5029d2a5cbaa5faed79075c Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 9 Sep 2024 14:50:23 +0000 Subject: [PATCH 25/36] Move the vfstrace extension out of src/ over into ext/misc/ where it belongs. Make it part of the standard build for the CLI. Bring some of the vfstrace output up-to-date. FossilOrigin-Name: 055b97de8d2397987d72dbab1cde78ece2d1c066e95042b4ed6b7b36b2cf9006 --- src/test_vfstrace.c => ext/misc/vfstrace.c | 186 +++++++++++++++------ manifest | 25 +-- manifest.uuid | 2 +- src/shell.c.in | 21 +-- src/wal.c | 2 +- tool/build-shell.sh | 2 +- tool/pagesig.c | 2 +- 7 files changed, 160 insertions(+), 80 deletions(-) rename src/test_vfstrace.c => ext/misc/vfstrace.c (82%) diff --git a/src/test_vfstrace.c b/ext/misc/vfstrace.c similarity index 82% rename from src/test_vfstrace.c rename to ext/misc/vfstrace.c index 8b7d2fec79..ecc756795f 100644 --- a/src/test_vfstrace.c +++ b/ext/misc/vfstrace.c @@ -207,36 +207,34 @@ static void vfstrace_printf( } /* -** Convert value rc into a string and print it using zFormat. zFormat -** should have exactly one %s +** Try to convert an error code into a symbolic name for that error code. */ -static void vfstrace_print_errcode( - vfstrace_info *pInfo, - const char *zFormat, - int rc -){ - char zBuf[50]; - char *zVal; +static const char *vfstrace_errcode_name(int rc ){ + const char *zVal = 0; switch( rc ){ - case SQLITE_OK: zVal = "SQLITE_OK"; break; - case SQLITE_ERROR: zVal = "SQLITE_ERROR"; break; - case SQLITE_PERM: zVal = "SQLITE_PERM"; break; - case SQLITE_ABORT: zVal = "SQLITE_ABORT"; break; - case SQLITE_BUSY: zVal = "SQLITE_BUSY"; break; - case SQLITE_NOMEM: zVal = "SQLITE_NOMEM"; break; - case SQLITE_READONLY: zVal = "SQLITE_READONLY"; break; - case SQLITE_INTERRUPT: zVal = "SQLITE_INTERRUPT"; break; - case SQLITE_IOERR: zVal = "SQLITE_IOERR"; break; - case SQLITE_CORRUPT: zVal = "SQLITE_CORRUPT"; break; - case SQLITE_FULL: zVal = "SQLITE_FULL"; break; - case SQLITE_CANTOPEN: zVal = "SQLITE_CANTOPEN"; break; - case SQLITE_PROTOCOL: zVal = "SQLITE_PROTOCOL"; break; - case SQLITE_EMPTY: zVal = "SQLITE_EMPTY"; break; - case SQLITE_SCHEMA: zVal = "SQLITE_SCHEMA"; break; - case SQLITE_CONSTRAINT: zVal = "SQLITE_CONSTRAINT"; break; - case SQLITE_MISMATCH: zVal = "SQLITE_MISMATCH"; break; - case SQLITE_MISUSE: zVal = "SQLITE_MISUSE"; break; - case SQLITE_NOLFS: zVal = "SQLITE_NOLFS"; break; + case SQLITE_OK: zVal = "SQLITE_OK"; break; + case SQLITE_INTERNAL: zVal = "SQLITE_INTERNAL"; break; + case SQLITE_ERROR: zVal = "SQLITE_ERROR"; break; + case SQLITE_PERM: zVal = "SQLITE_PERM"; break; + case SQLITE_ABORT: zVal = "SQLITE_ABORT"; break; + case SQLITE_BUSY: zVal = "SQLITE_BUSY"; break; + case SQLITE_LOCKED: zVal = "SQLITE_LOCKED"; break; + case SQLITE_NOMEM: zVal = "SQLITE_NOMEM"; break; + case SQLITE_READONLY: zVal = "SQLITE_READONLY"; break; + case SQLITE_INTERRUPT: zVal = "SQLITE_INTERRUPT"; break; + case SQLITE_IOERR: zVal = "SQLITE_IOERR"; break; + case SQLITE_CORRUPT: zVal = "SQLITE_CORRUPT"; break; + case SQLITE_NOTFOUND: zVal = "SQLITE_NOTFOUND"; break; + case SQLITE_FULL: zVal = "SQLITE_FULL"; break; + case SQLITE_CANTOPEN: zVal = "SQLITE_CANTOPEN"; break; + case SQLITE_PROTOCOL: zVal = "SQLITE_PROTOCOL"; break; + case SQLITE_EMPTY: zVal = "SQLITE_EMPTY"; break; + case SQLITE_SCHEMA: zVal = "SQLITE_SCHEMA"; break; + case SQLITE_TOOBIG: zVal = "SQLITE_TOOBIG"; break; + case SQLITE_CONSTRAINT: zVal = "SQLITE_CONSTRAINT"; break; + case SQLITE_MISMATCH: zVal = "SQLITE_MISMATCH"; break; + case SQLITE_MISUSE: zVal = "SQLITE_MISUSE"; break; + case SQLITE_NOLFS: zVal = "SQLITE_NOLFS"; break; case SQLITE_IOERR_READ: zVal = "SQLITE_IOERR_READ"; break; case SQLITE_IOERR_SHORT_READ: zVal = "SQLITE_IOERR_SHORT_READ"; break; case SQLITE_IOERR_WRITE: zVal = "SQLITE_IOERR_WRITE"; break; @@ -266,11 +264,30 @@ static void vfstrace_print_errcode( case SQLITE_LOCKED_SHAREDCACHE: zVal = "SQLITE_LOCKED_SHAREDCACHE"; break; case SQLITE_BUSY_RECOVERY: zVal = "SQLITE_BUSY_RECOVERY"; break; case SQLITE_CANTOPEN_NOTEMPDIR: zVal = "SQLITE_CANTOPEN_NOTEMPDIR"; break; - default: { - sqlite3_snprintf(sizeof(zBuf), zBuf, "%d", rc); - zVal = zBuf; - break; + } + return zVal; +} + +/* +** Convert value rc into a string and print it using zFormat. zFormat +** should have exactly one %s +*/ +static void vfstrace_print_errcode( + vfstrace_info *pInfo, + const char *zFormat, + int rc +){ + const char *zVal; + char zBuf[50]; + zVal = vfstrace_errcode_name(rc); + if( zVal==0 ){ + zVal = vfstrace_errcode_name(rc&0xff); + if( zVal ){ + sqlite3_snprintf(sizeof(zBuf), zBuf, "%s | 0x%x", zVal, rc&0xffff00); + }else{ + sqlite3_snprintf(sizeof(zBuf), zBuf, "%d (0x%x)", rc, rc); } + zVal = zBuf; } vfstrace_printf(pInfo, zFormat, zVal); } @@ -457,12 +474,14 @@ static int vfstraceFileControl(sqlite3_file *pFile, int op, void *pArg){ vfstrace_info *pInfo = p->pInfo; int rc; char zBuf[100]; + char zBuf2[100]; char *zOp; + char *zRVal = 0; switch( op ){ - case SQLITE_FCNTL_LOCKSTATE: zOp = "LOCKSTATE"; break; - case SQLITE_GET_LOCKPROXYFILE: zOp = "GET_LOCKPROXYFILE"; break; - case SQLITE_SET_LOCKPROXYFILE: zOp = "SET_LOCKPROXYFILE"; break; - case SQLITE_LAST_ERRNO: zOp = "LAST_ERRNO"; break; + case SQLITE_FCNTL_LOCKSTATE: zOp = "LOCKSTATE"; break; + case SQLITE_GET_LOCKPROXYFILE: zOp = "GET_LOCKPROXYFILE"; break; + case SQLITE_SET_LOCKPROXYFILE: zOp = "SET_LOCKPROXYFILE"; break; + case SQLITE_LAST_ERRNO: zOp = "LAST_ERRNO"; break; case SQLITE_FCNTL_SIZE_HINT: { sqlite3_snprintf(sizeof(zBuf), zBuf, "SIZE_HINT,%lld", *(sqlite3_int64*)pArg); @@ -474,20 +493,57 @@ static int vfstraceFileControl(sqlite3_file *pFile, int op, void *pArg){ zOp = zBuf; break; } - case SQLITE_FCNTL_FILE_POINTER: zOp = "FILE_POINTER"; break; - case SQLITE_FCNTL_SYNC_OMITTED: zOp = "SYNC_OMITTED"; break; - case SQLITE_FCNTL_WIN32_AV_RETRY: zOp = "WIN32_AV_RETRY"; break; - case SQLITE_FCNTL_PERSIST_WAL: zOp = "PERSIST_WAL"; break; - case SQLITE_FCNTL_OVERWRITE: zOp = "OVERWRITE"; break; - case SQLITE_FCNTL_VFSNAME: zOp = "VFSNAME"; break; - case SQLITE_FCNTL_TEMPFILENAME: zOp = "TEMPFILENAME"; break; - case 0xca093fa0: zOp = "DB_UNCHANGED"; break; + case SQLITE_FCNTL_FILE_POINTER: zOp = "FILE_POINTER"; break; + case SQLITE_FCNTL_WIN32_AV_RETRY: zOp = "WIN32_AV_RETRY"; break; + case SQLITE_FCNTL_PERSIST_WAL: { + sqlite3_snprintf(sizeof(zBuf), zBuf, "PERSIST_WAL,%d", *(int*)pArg); + zOp = zBuf; + break; + } + case SQLITE_FCNTL_OVERWRITE: zOp = "OVERWRITE"; break; + case SQLITE_FCNTL_VFSNAME: zOp = "VFSNAME"; break; + case SQLITE_FCNTL_POWERSAFE_OVERWRITE: zOp = "POWERSAFE_OVERWRITE"; break; case SQLITE_FCNTL_PRAGMA: { const char *const* a = (const char*const*)pArg; sqlite3_snprintf(sizeof(zBuf), zBuf, "PRAGMA,[%s,%s]",a[1],a[2]); zOp = zBuf; break; } + case SQLITE_FCNTL_BUSYHANDLER: zOp = "BUSYHANDLER"; break; + case SQLITE_FCNTL_TEMPFILENAME: zOp = "TEMPFILENAME"; break; + case SQLITE_FCNTL_MMAP_SIZE: zOp = "MMAP_SIZE"; break; + case SQLITE_FCNTL_TRACE: zOp = "TRACE"; break; + case SQLITE_FCNTL_HAS_MOVED: zOp = "HAS_MOVED"; break; + case SQLITE_FCNTL_SYNC: zOp = "SYNC"; break; + case SQLITE_FCNTL_COMMIT_PHASETWO: zOp = "COMMIT_PHASETWO"; break; + case SQLITE_FCNTL_WIN32_SET_HANDLE: zOp = "WIN32_SET_HANDLE"; break; + case SQLITE_FCNTL_WAL_BLOCK: zOp = "WAL_BLOCK"; break; + case SQLITE_FCNTL_ZIPVFS: zOp = "ZIPVFS"; break; + case SQLITE_FCNTL_RBU: zOp = "RBU"; break; + case SQLITE_FCNTL_VFS_POINTER: zOp = "VFS_POINTER"; break; + case SQLITE_FCNTL_JOURNAL_POINTER: zOp = "JOURNAL_POINTER"; break; + case SQLITE_FCNTL_WIN32_GET_HANDLE: zOp = "WIN32_GET_HANDLE"; break; + case SQLITE_FCNTL_PDB: zOp = "PDB"; break; + case SQLITE_FCNTL_BEGIN_ATOMIC_WRITE: zOp = "BEGIN_ATOMIC_WRITE"; break; + case SQLITE_FCNTL_COMMIT_ATOMIC_WRITE: zOp = "COMMIT_ATOMIC_WRITE"; break; + case SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE: { + zOp = "ROLLBACK_ATOMIC_WRITE"; + break; + } + case SQLITE_FCNTL_LOCK_TIMEOUT: { + sqlite3_snprintf(sizeof(zBuf), zBuf, "LOCK_TIMEOUT,%d", *(int*)pArg); + zOp = zBuf; + break; + } + case SQLITE_FCNTL_DATA_VERSION: zOp = "DATA_VERSION"; break; + case SQLITE_FCNTL_SIZE_LIMIT: zOp = "SIZE_LIMIT"; break; + case SQLITE_FCNTL_CKPT_DONE: zOp = "CKPT_DONE"; break; + case SQLITE_FCNTL_RESERVE_BYTES: zOp = "RESERVED_BYTES"; break; + case SQLITE_FCNTL_CKPT_START: zOp = "CKPT_START"; break; + case SQLITE_FCNTL_EXTERNAL_READER: zOp = "EXTERNAL_READER"; break; + case SQLITE_FCNTL_CKSM_FILE: zOp = "CKSM_FILE"; break; + case SQLITE_FCNTL_RESET_CACHE: zOp = "RESET_CACHE"; break; + case 0xca093fa0: zOp = "DB_UNCHANGED"; break; default: { sqlite3_snprintf(sizeof zBuf, zBuf, "%d", op); zOp = zBuf; @@ -497,15 +553,31 @@ static int vfstraceFileControl(sqlite3_file *pFile, int op, void *pArg){ vfstrace_printf(pInfo, "%s.xFileControl(%s,%s)", pInfo->zVfsName, p->zFName, zOp); rc = p->pReal->pMethods->xFileControl(p->pReal, op, pArg); - vfstrace_print_errcode(pInfo, " -> %s\n", rc); - if( op==SQLITE_FCNTL_VFSNAME && rc==SQLITE_OK ){ - *(char**)pArg = sqlite3_mprintf("vfstrace.%s/%z", + if( rc==SQLITE_OK ){ + switch( op ){ + case SQLITE_FCNTL_VFSNAME: { + *(char**)pArg = sqlite3_mprintf("vfstrace.%s/%z", pInfo->zVfsName, *(char**)pArg); + zRVal = *(char**)pArg; + break; + } + case SQLITE_FCNTL_PERSIST_WAL: { + sqlite3_snprintf(sizeof(zBuf2), zBuf2, "%d", *(int*)pArg); + zRVal = zBuf2; + break; + } + case SQLITE_FCNTL_PRAGMA: + case SQLITE_FCNTL_TEMPFILENAME: { + zRVal = *(char**)pArg; + break; + } + } } - if( (op==SQLITE_FCNTL_PRAGMA || op==SQLITE_FCNTL_TEMPFILENAME) - && rc==SQLITE_OK && *(char**)pArg ){ - vfstrace_printf(pInfo, "%s.xFileControl(%s,%s) returns %s", - pInfo->zVfsName, p->zFName, zOp, *(char**)pArg); + if( zRVal ){ + vfstrace_print_errcode(pInfo, " -> %s", rc); + vfstrace_printf(pInfo, ", %s\n", zRVal); + }else{ + vfstrace_print_errcode(pInfo, " -> %s\n", rc); } return rc; } @@ -890,3 +962,15 @@ int vfstrace_register( pInfo->zVfsName, pRoot->zName); return sqlite3_vfs_register(pNew, makeDefault); } + +/* +** Look for the named VFS. If it is a TRACEVFS, then unregister it +** and delete it. +*/ +void vfstrace_unregister(const char *zTraceName){ + sqlite3_vfs *pVfs = sqlite3_vfs_find(zTraceName); + if( pVfs==0 ) return; + if( pVfs->xOpen!=vfstraceOpen ) return; + sqlite3_vfs_unregister(pVfs); + sqlite3_free(pVfs); +} diff --git a/manifest b/manifest index 037c1b19a0..4c907f4559 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\swith\sfts5\slocale=1\stables\sand\sUPDATE\sstatements\sthat\smay\saffect\smore\sthan\sone\srow. -D 2024-09-07T16:22:22.943 +C Move\sthe\svfstrace\sextension\sout\sof\ssrc/\sover\sinto\sext/misc/\swhere\sit\sbelongs.\nMake\sit\spart\sof\sthe\sstandard\sbuild\sfor\sthe\sCLI.\s\sBring\ssome\sof\sthe\svfstrace\noutput\sup-to-date. +D 2024-09-09T14:50:23.152 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -435,6 +435,7 @@ F ext/misc/urifuncs.c f71360d14fa9e7626b563f1f781c6148109462741c5235ac63ae0f8917 F ext/misc/uuid.c 5bb2264c1b64d163efa46509544fd7500cb8769cb7c16dd52052da8d961505cf F ext/misc/vfslog.c 3932ab932eeb2601dbc4447cb14d445aaa9fbe43b863ef5f014401c3420afd20 F ext/misc/vfsstat.c a85df08654743922a19410d7b1e3111de41bb7cd07d20dd16eda4e2b808d269d +F ext/misc/vfstrace.c 54b91e641b8e76a7e9049628c104f4b33bde015969dacf0d95ef11140fb5f6bc w src/test_vfstrace.c F ext/misc/vtablog.c 1100250ce8782db37c833e3a9a5c9a3ecf1af5e15b8325572b82e6e0a138ffb5 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd F ext/misc/wholenumber.c 0fa0c082676b7868bf2fa918e911133f2b349bcdceabd1198bba5f65b4fc0668 @@ -767,7 +768,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 2c127880c0634962837f16f2f48a295e514357af959330cc038de73015d5b5e8 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 F src/select.c 4b14337a2742f0c0beeba490e9a05507e9b4b12184b9cd12773501d08d48e3fe -F src/shell.c.in 40de636c1d90fb8a9ca7f49dc8f50d930f1b60736e73aca5eb37c4c7d0e47f9d +F src/shell.c.in 470db843788d74234cc1e6873ac51c0ae6529994a52146fefe2e77c0754cbf96 F src/sqlite.h.in 77f55bd1978a04a14db211732f0a609077cf60ba4ccf9baf39988f508945419c F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54 @@ -823,7 +824,6 @@ F src/test_tclvar.c ae873248a0188459b1c16ca7cc431265dacce524399e8b46725c2b3b7e04 F src/test_thread.c d7a8bcea7445f37cc2a1f7f81dd6059634f45e0c61bfe80182b02872fb0328bb F src/test_vdbecov.c 5c426d9cd2b351f5f9ceb30cabf8c64a63bfcad644c507e0bd9ce2f6ae1a3bf3 F src/test_vfs.c f298475e468c7e14945b20af885917181090c265aa3c4ade897849c9fbd396f2 -F src/test_vfstrace.c a2ea82df2ed8927e9eba49bdba1aa1aeb1dcb13dbf6558cff036da813031de9a F src/test_windirent.c a895e2c068a06644eef91a7f0a32182445a893b9a0f33d0cdb4283dca2486ac1 F src/test_windirent.h da2e5b73c32d09905fbdd00f27cd802212a32a58ead882736fe4f5eb775ebc50 F src/test_window.c 6d80e11fba89a1796525e6f0048ff0c7789aa2c6b0b11c80827dc1437bd8ea72 @@ -849,7 +849,7 @@ F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1945f1d2e6be5927ec12c06d89b03ef2a4def34bf8 F src/vdbevtab.c fc46b9cbd759dc013f0b3724549cc0d71379183c667df3a5988f7e2f1bd485f3 F src/vtab.c 5fb499d20494b7eecaadb7584634af9afcb374cb0524912b475fcb1712458a1b F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 887fc4ca3f020ebb2e376f222069570834ac63bf50111ef0cbf3ae417048ed89 +F src/wal.c ef68130ba330ee18c1cb22da36a881c82e3a3b109badbdc6a9b9acaf788a6688 F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452 F src/walker.c d5006d6b005e4ea7302ad390957a8d41ed83faa177e412f89bc5600a7462a014 F src/where.c 7fb55836eb7fd07f0a0d8400c50619fc02cda1f46a617cfb003c2990f040193d @@ -2102,7 +2102,7 @@ F tool/GetFile.cs 47852aa0d806fe47ed1ac5138bdce7f000fe87aaa7f28107d0cb1e26682aeb F tool/GetTclKit.bat d84033c6a93dfe735d247f48ba00292a1cc284dcf69963e5e672444e04534bbf F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91 F tool/build-all-msvc.bat c817b716e0edeecaf265a6775b63e5f45c34a6544f1d4114a222701ed5ac79ab x -F tool/build-shell.sh f193b5e3eb4afcb4abbf96bf1475be6cfb74763ee2e50c82bc7ca105e8a136c5 +F tool/build-shell.sh 369c4b171cc877ad974fef691e4da782b4c1e99fe8f4361316c735f64d49280f F tool/buildtclext.tcl b64d250517b148e644d26fcbc097851867a0df52cd4bafe9bcd94b8421e1428a F tool/cg_anno.tcl c1f875f5a4c9caca3d59937b16aff716f8b1883935f1b4c9ae23124705bc8099 x F tool/checkSpacing.c 810e51703529a204fc4e1eb060e9ab663e3c06d2 @@ -2152,7 +2152,7 @@ F tool/offsets.c 8ed2b344d33f06e71366a9b93ccedaa38c096cc1dbd4c3c26ad08c611528584 F tool/omittest-msvc.tcl d6b8f501ac1d7798c4126065030f89812379012cad98a1735d6d7221492abc08 F tool/omittest.tcl 5ca5e4e01716d5f35b48b00fd351d929f01fbb98169a5a3cd00baf3d2e2019a9 F tool/opcodesum.tcl 740ed206ba8c5040018988129abbf3089a0ccf4a -F tool/pagesig.c ff0ca355fd3c2398e933da5e22439bbff89b803b +F tool/pagesig.c f98909b4168d9cac11a2de7f031adea0e2f3131faa7515a72807c03ec58eafeb F tool/replace.tcl 511c61acfe563dfb58675efb4628bb158a13d48ff8322123ac447e9d25a82d9a F tool/restore_jrnl.tcl 1079ecba47cc82fa82115b81c1f68097ab1f956f357ee8da5fc4b2589af6bd98 F tool/rollback-test.c 9fc98427d1e23e84429d7e6d07d9094fbdec65a5 @@ -2212,8 +2212,11 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6206b90a4ec3f05e3bbb4844e71569bbde7df237550569e6419ff7c3146505dc -R a6acb083cae329c41bdac6bb67941a5b -U dan -Z 2566687c7d077d4a754f89b771fd38fa +P 70e42f941c0778a04b82655409c7caf4c1039589f7e43a8ec1e736ea8f931b26 +R 1378141f27acd6b5e1f32f92428038b9 +T *branch * vfstrace +T *sym-vfstrace * +T -sym-trunk * +U drh +Z 9efca4af73cc272f57a358ecf75e82b9 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 73e0954b37..3c68b6d655 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -70e42f941c0778a04b82655409c7caf4c1039589f7e43a8ec1e736ea8f931b26 +055b97de8d2397987d72dbab1cde78ece2d1c066e95042b4ed6b7b36b2cf9006 diff --git a/src/shell.c.in b/src/shell.c.in index b8f9327948..9bc6c2566f 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -1236,6 +1236,7 @@ INCLUDE ../ext/expert/sqlite3expert.c INCLUDE ../ext/intck/sqlite3intck.h INCLUDE ../ext/intck/sqlite3intck.c INCLUDE ../ext/misc/stmtrand.c +INCLUDE ../ext/misc/vfstrace.c #if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) #define SQLITE_SHELL_HAVE_RECOVER 1 @@ -12254,9 +12255,7 @@ static const char zOptions[] = " -unsafe-testing allow unsafe commands and modes for testing\n" " -version show SQLite version\n" " -vfs NAME use NAME as the default VFS\n" -#ifdef SQLITE_ENABLE_VFSTRACE " -vfstrace enable tracing of all VFS calls\n" -#endif #ifdef SQLITE_HAVE_ZLIB " -zip open the file as a ZIP Archive\n" #endif @@ -12382,6 +12381,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ int readStdin = 1; int nCmd = 0; int nOptsEnd = argc; + int bEnableVfstrace = 0; char **azCmd = 0; const char *zVfs = 0; /* Value of -vfs command-line option */ #if !SQLITE_SHELL_IS_UTF8 @@ -12576,17 +12576,9 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ case 2: sqlite3_config(SQLITE_CONFIG_MULTITHREAD); break; default: sqlite3_config(SQLITE_CONFIG_SERIALIZED); break; } -#ifdef SQLITE_ENABLE_VFSTRACE }else if( cli_strcmp(z,"-vfstrace")==0 ){ - extern int vfstrace_register( - const char *zTraceName, - const char *zOldVfsName, - int (*xOut)(const char*,void*), - void *pOutArg, - int makeDefault - ); vfstrace_register("trace",0,(int(*)(const char*,void*))fputs,stderr,1); -#endif + bEnableVfstrace = 1; #ifdef SQLITE_ENABLE_MULTIPLEX }else if( cli_strcmp(z,"-multiplex")==0 ){ extern int sqlite3_multiplex_initialize(const char*,int); @@ -12642,7 +12634,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ } } #ifndef SQLITE_SHELL_FIDDLE - verify_uninitialized(); + if( !bEnableVfstrace ) verify_uninitialized(); #endif @@ -12831,10 +12823,8 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ #endif }else if( cli_strcmp(z,"-vfs")==0 ){ i++; -#ifdef SQLITE_ENABLE_VFSTRACE }else if( cli_strcmp(z,"-vfstrace")==0 ){ i++; -#endif #ifdef SQLITE_ENABLE_MULTIPLEX }else if( cli_strcmp(z,"-multiplex")==0 ){ i++; @@ -13002,6 +12992,9 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ /* Clear the global data structure so that valgrind will detect memory ** leaks */ memset(&data, 0, sizeof(data)); + if( bEnableVfstrace ){ + vfstrace_unregister("trace"); + } #ifdef SQLITE_DEBUG if( sqlite3_memory_used()>mem_main_enter ){ eputf("Memory leaked: %u bytes\n", diff --git a/src/wal.c b/src/wal.c index fd2eabfd96..89106038b8 100644 --- a/src/wal.c +++ b/src/wal.c @@ -44,7 +44,7 @@ ** 28: Checksum-2 (second part of checksum for first 24 bytes of header). ** ** Immediately following the wal-header are zero or more frames. Each -** frame consists of a 24-byte frame-header followed by a bytes +** frame consists of a 24-byte frame-header followed by bytes ** of page data. The frame-header is six big-endian 32-bit unsigned ** integer values, as follows: ** diff --git a/tool/build-shell.sh b/tool/build-shell.sh index a57f753039..7899080ebe 100644 --- a/tool/build-shell.sh +++ b/tool/build-shell.sh @@ -17,5 +17,5 @@ gcc -o sqlite3 -g -Os -I. \ -DSQLITE_ENABLE_RTREE \ -DHAVE_READLINE \ ../sqlite/src/shell.c \ - ../sqlite/src/test_vfstrace.c \ + ../sqlite/ext/misc/vfstrace.c \ sqlite3.c -ldl -lreadline -lncurses diff --git a/tool/pagesig.c b/tool/pagesig.c index 540c9d7226..37cc804a53 100644 --- a/tool/pagesig.c +++ b/tool/pagesig.c @@ -26,7 +26,7 @@ ** the entire block. ** ** For blocks of more than 16 bytes, the signature is a hex dump of the -** first 8 bytes followed by a 64-bit has of the entire block. +** first 8 bytes followed by a 64-bit hash of the entire block. */ static void vlogSignature(unsigned char *p, int n, char *zCksum){ unsigned int s0 = 0, s1 = 0; From a8cf79e32234368ef936b3a1a7f890b70712f936 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 9 Sep 2024 15:12:21 +0000 Subject: [PATCH 26/36] Fix harmless compiler warnings in FTS5. FossilOrigin-Name: aa75e701de61fe63ec15c35d70e53e950ff73b0dcb0d871dd8721412f3af297a --- ext/fts5/fts5_main.c | 4 ++-- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index cb68c1444f..0cc6fd30a9 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -1284,7 +1284,7 @@ void sqlite3Fts5ClearLocale(Fts5Config *pConfig){ int sqlite3Fts5IsLocaleValue(Fts5Config *pConfig, sqlite3_value *pVal){ int ret = 0; if( sqlite3_value_type(pVal)==SQLITE_BLOB ){ - if( sqlite3_value_bytes(pVal)>FTS5_LOCALE_HDR_SIZE + if( sqlite3_value_bytes(pVal)>(int)FTS5_LOCALE_HDR_SIZE && 0==memcmp(sqlite3_value_blob(pVal), FTS5_LOCALE_HDR(pConfig), 4) ){ ret = 1; @@ -3011,7 +3011,7 @@ static void fts5ExtractValueFromColumn( int ii; if( pConfig->eContent==FTS5_CONTENT_EXTERNAL ){ - if( nBlob Date: Mon, 9 Sep 2024 15:19:26 +0000 Subject: [PATCH 27/36] Fix harmless compiler warnings in the vfstrace.c extension. FossilOrigin-Name: f23954e604bf4da45f07194b54a4fe1c83002ab65d6c6f0ac095e88baba18547 --- ext/misc/vfstrace.c | 8 ++++---- manifest | 15 ++++++--------- manifest.uuid | 2 +- 3 files changed, 11 insertions(+), 14 deletions(-) diff --git a/ext/misc/vfstrace.c b/ext/misc/vfstrace.c index ecc756795f..fd972411d8 100644 --- a/ext/misc/vfstrace.c +++ b/ext/misc/vfstrace.c @@ -182,7 +182,7 @@ static const char *vfstraceNextSystemCall(sqlite3_vfs*, const char *zName); ** xyzzy.txt -> xyzzy.txt */ static const char *fileTail(const char *z){ - int i; + size_t i; if( z==0 ) return 0; i = strlen(z)-1; while( i>0 && z[i-1]!='/' ){ i--; } @@ -912,14 +912,14 @@ int vfstrace_register( sqlite3_vfs *pNew; sqlite3_vfs *pRoot; vfstrace_info *pInfo; - int nName; - int nByte; + size_t nName; + size_t nByte; pRoot = sqlite3_vfs_find(zOldVfsName); if( pRoot==0 ) return SQLITE_NOTFOUND; nName = strlen(zTraceName); nByte = sizeof(*pNew) + sizeof(*pInfo) + nName + 1; - pNew = sqlite3_malloc( nByte ); + pNew = sqlite3_malloc64( nByte ); if( pNew==0 ) return SQLITE_NOMEM; memset(pNew, 0, nByte); pInfo = (vfstrace_info*)&pNew[1]; diff --git a/manifest b/manifest index 4c907f4559..3b8a584824 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Move\sthe\svfstrace\sextension\sout\sof\ssrc/\sover\sinto\sext/misc/\swhere\sit\sbelongs.\nMake\sit\spart\sof\sthe\sstandard\sbuild\sfor\sthe\sCLI.\s\sBring\ssome\sof\sthe\svfstrace\noutput\sup-to-date. -D 2024-09-09T14:50:23.152 +C Fix\sharmless\scompiler\swarnings\sin\sthe\svfstrace.c\sextension. +D 2024-09-09T15:19:26.342 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -435,7 +435,7 @@ F ext/misc/urifuncs.c f71360d14fa9e7626b563f1f781c6148109462741c5235ac63ae0f8917 F ext/misc/uuid.c 5bb2264c1b64d163efa46509544fd7500cb8769cb7c16dd52052da8d961505cf F ext/misc/vfslog.c 3932ab932eeb2601dbc4447cb14d445aaa9fbe43b863ef5f014401c3420afd20 F ext/misc/vfsstat.c a85df08654743922a19410d7b1e3111de41bb7cd07d20dd16eda4e2b808d269d -F ext/misc/vfstrace.c 54b91e641b8e76a7e9049628c104f4b33bde015969dacf0d95ef11140fb5f6bc w src/test_vfstrace.c +F ext/misc/vfstrace.c 9ff6f4352cdd3ed611e93239ad641c6e70dd48a50cae2d8c563dd1a4c549d5f9 F ext/misc/vtablog.c 1100250ce8782db37c833e3a9a5c9a3ecf1af5e15b8325572b82e6e0a138ffb5 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd F ext/misc/wholenumber.c 0fa0c082676b7868bf2fa918e911133f2b349bcdceabd1198bba5f65b4fc0668 @@ -2212,11 +2212,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 70e42f941c0778a04b82655409c7caf4c1039589f7e43a8ec1e736ea8f931b26 -R 1378141f27acd6b5e1f32f92428038b9 -T *branch * vfstrace -T *sym-vfstrace * -T -sym-trunk * +P 055b97de8d2397987d72dbab1cde78ece2d1c066e95042b4ed6b7b36b2cf9006 +R 121ef6e86bed835ee78e7e9a5da1af0d U drh -Z 9efca4af73cc272f57a358ecf75e82b9 +Z 339b357d35c1fb68817d90123ff83e8c # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3c68b6d655..d3ff79492e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -055b97de8d2397987d72dbab1cde78ece2d1c066e95042b4ed6b7b36b2cf9006 +f23954e604bf4da45f07194b54a4fe1c83002ab65d6c6f0ac095e88baba18547 From dbab768d078549db8496744d2a0e1bee7f9dfc26 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 9 Sep 2024 15:33:37 +0000 Subject: [PATCH 28/36] Fix dependencies in makefiles to include ext/misc/vfstrace.c as necessary. Improved xFileControl() output from vfstrace.c. FossilOrigin-Name: e8f2d6313075c92fdeebcdfd8b50f43e9d45225890b2ef6b77148a766a42e940 --- Makefile.in | 1 + Makefile.msc | 1 + ext/misc/vfstrace.c | 13 ++++++++++++- main.mk | 1 + manifest | 18 +++++++++--------- manifest.uuid | 2 +- 6 files changed, 25 insertions(+), 11 deletions(-) diff --git a/Makefile.in b/Makefile.in index 9036babcb2..e359975d87 100644 --- a/Makefile.in +++ b/Makefile.in @@ -1194,6 +1194,7 @@ SHELL_DEP = \ $(TOP)/ext/misc/shathree.c \ $(TOP)/ext/misc/sqlar.c \ $(TOP)/ext/misc/uint.c \ + $(TOP)/ext/misc/vfstrace.c \ $(TOP)/ext/misc/zipfile.c \ $(TOP)/ext/recover/dbdata.c \ $(TOP)/ext/recover/sqlite3recover.c \ diff --git a/Makefile.msc b/Makefile.msc index 36a0ed3abe..41e5c1081a 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -2320,6 +2320,7 @@ SHELL_DEP = \ $(TOP)\ext\misc\shathree.c \ $(TOP)\ext\misc\sqlar.c \ $(TOP)\ext\misc\uint.c \ + $(TOP)\ext\misc\vfstrace.c \ $(TOP)\ext\misc\zipfile.c \ $(TOP)\ext\recover\dbdata.c \ $(TOP)\ext\recover\sqlite3recover.c \ diff --git a/ext/misc/vfstrace.c b/ext/misc/vfstrace.c index fd972411d8..9d36dc022b 100644 --- a/ext/misc/vfstrace.c +++ b/ext/misc/vfstrace.c @@ -511,7 +511,12 @@ static int vfstraceFileControl(sqlite3_file *pFile, int op, void *pArg){ } case SQLITE_FCNTL_BUSYHANDLER: zOp = "BUSYHANDLER"; break; case SQLITE_FCNTL_TEMPFILENAME: zOp = "TEMPFILENAME"; break; - case SQLITE_FCNTL_MMAP_SIZE: zOp = "MMAP_SIZE"; break; + case SQLITE_FCNTL_MMAP_SIZE: { + sqlite3_int64 iMMap = *(sqlite3_int64*)pArg; + sqlite3_snprintf(sizeof(zBuf), zBuf, "MMAP_SIZE,%lld",iMMap); + zOp = zBuf; + break; + } case SQLITE_FCNTL_TRACE: zOp = "TRACE"; break; case SQLITE_FCNTL_HAS_MOVED: zOp = "HAS_MOVED"; break; case SQLITE_FCNTL_SYNC: zOp = "SYNC"; break; @@ -561,6 +566,12 @@ static int vfstraceFileControl(sqlite3_file *pFile, int op, void *pArg){ zRVal = *(char**)pArg; break; } + case SQLITE_FCNTL_MMAP_SIZE: { + sqlite3_snprintf(sizeof(zBuf2), zBuf2, "%lld", *(sqlite3_int64*)pArg); + zRVal = zBuf2; + break; + } + case SQLITE_FCNTL_HAS_MOVED: case SQLITE_FCNTL_PERSIST_WAL: { sqlite3_snprintf(sizeof(zBuf2), zBuf2, "%d", *(int*)pArg); zRVal = zBuf2; diff --git a/main.mk b/main.mk index b379d2f8f8..17284dd9f3 100644 --- a/main.mk +++ b/main.mk @@ -771,6 +771,7 @@ SHELL_DEP = \ $(TOP)/ext/misc/shathree.c \ $(TOP)/ext/misc/sqlar.c \ $(TOP)/ext/misc/uint.c \ + $(TOP)/ext/misc/vfstrace.c \ $(TOP)/ext/misc/zipfile.c \ $(TOP)/ext/recover/dbdata.c \ $(TOP)/ext/recover/sqlite3recover.c \ diff --git a/manifest b/manifest index 3b8a584824..ea9a99c59f 100644 --- a/manifest +++ b/manifest @@ -1,11 +1,11 @@ -C Fix\sharmless\scompiler\swarnings\sin\sthe\svfstrace.c\sextension. -D 2024-09-09T15:19:26.342 +C Fix\sdependencies\sin\smakefiles\sto\sinclude\sext/misc/vfstrace.c\sas\snecessary.\nImproved\sxFileControl()\soutput\sfrom\svfstrace.c. +D 2024-09-09T15:33:37.183 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 -F Makefile.in c41539ec7e8e69091e74c7d9e011115ae5e81d1f53829e640d299f3a754f94a4 +F Makefile.in 7753650b4204e3ccd55a4e6a0d73a5a01f737dcefb099d901ce1de5df9d0b82c F Makefile.linux-gcc f3842a0b1efbfbb74ac0ef60e56b301836d05b4d867d014f714fa750048f1ab6 -F Makefile.msc e72a4653ea8f48efd14c7dac1e7de308a1f3cc91f905f15cbcacc680301c8f03 +F Makefile.msc 6e8925dca6dc8c3e9cce042bbf347d20164653e63aeafcf6f6a28e27cf976d8b F README.md c3c0f19532ce28f6297a71870f3c7b424729f0e6d9ab889616d3587dd2332159 F VERSION 0db40f92c04378404eb45bff93e9e42c148c7e54fd3da99469ed21e22411f5a6 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -435,7 +435,7 @@ F ext/misc/urifuncs.c f71360d14fa9e7626b563f1f781c6148109462741c5235ac63ae0f8917 F ext/misc/uuid.c 5bb2264c1b64d163efa46509544fd7500cb8769cb7c16dd52052da8d961505cf F ext/misc/vfslog.c 3932ab932eeb2601dbc4447cb14d445aaa9fbe43b863ef5f014401c3420afd20 F ext/misc/vfsstat.c a85df08654743922a19410d7b1e3111de41bb7cd07d20dd16eda4e2b808d269d -F ext/misc/vfstrace.c 9ff6f4352cdd3ed611e93239ad641c6e70dd48a50cae2d8c563dd1a4c549d5f9 +F ext/misc/vfstrace.c 03f90dd465968e01f5d1d3e79c36cbc53a5bfe1bd55d239435ce94df19d5b0ac F ext/misc/vtablog.c 1100250ce8782db37c833e3a9a5c9a3ecf1af5e15b8325572b82e6e0a138ffb5 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd F ext/misc/wholenumber.c 0fa0c082676b7868bf2fa918e911133f2b349bcdceabd1198bba5f65b4fc0668 @@ -687,7 +687,7 @@ F ext/wasm/wasmfs.make 8a4955882aaa0783b3f60a9484a1f0f3d8b6f775c0fcd17c082f31966 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 5ade0bc977aa135e79e3faaea894d5671b26107cc91e70783aa7dc83f22f3ba0 -F main.mk 5a2e7d4a852c058373efc78407816de41595d06975148c766092b3cf0fea4298 +F main.mk 391342c3c0907f57bbb9ab60ce4b3cfe1ea61161996b449033984673d18980fd F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 F mptest/crash01.test 61e61469e257df0850df4293d7d4d6c2af301421 @@ -2212,8 +2212,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 055b97de8d2397987d72dbab1cde78ece2d1c066e95042b4ed6b7b36b2cf9006 -R 121ef6e86bed835ee78e7e9a5da1af0d +P f23954e604bf4da45f07194b54a4fe1c83002ab65d6c6f0ac095e88baba18547 +R 7061e15741ec86af06f2beddce48e46a U drh -Z 339b357d35c1fb68817d90123ff83e8c +Z 809fa2eaa1108af2a9ce0f3c9039775d # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d3ff79492e..811c1c885c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f23954e604bf4da45f07194b54a4fe1c83002ab65d6c6f0ac095e88baba18547 +e8f2d6313075c92fdeebcdfd8b50f43e9d45225890b2ef6b77148a766a42e940 From 882aba40906150d96ef2ba1ca2cd15aeada1dee2 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 9 Sep 2024 18:45:58 +0000 Subject: [PATCH 29/36] Generalize the sqlite3_dbpage virtual table so that it is able to write new pages onto the end of the database file using INSERT. FossilOrigin-Name: fe0d67e72d4228661c021f227bfc0d5ddb1b726db0f36c7221ead8dd8bd1dc73 --- manifest | 15 +++++++-------- manifest.uuid | 2 +- src/dbpage.c | 31 ++++++++++++++++++------------- 3 files changed, 26 insertions(+), 22 deletions(-) diff --git a/manifest b/manifest index 7692e3ad79..c5f3a5e79e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Move\sthe\svfstrace\sextension\sout\sof\ssrc/\sand\sinto\sext/misc/.\s\sMake\sit\sa\nstandard\spart\sof\sthe\sCLI.\s\sImprove\sits\soutput.\s\sAlso\sfix\ssome\sunrelated\ncomment\stypos. -D 2024-09-09T15:39:40.442 +C Generalize\sthe\ssqlite3_dbpage\svirtual\stable\sso\sthat\sit\sis\sable\sto\swrite\nnew\spages\sonto\sthe\send\sof\sthe\sdatabase\sfile\susing\sINSERT. +D 2024-09-09T18:45:58.205 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -435,7 +435,7 @@ F ext/misc/urifuncs.c f71360d14fa9e7626b563f1f781c6148109462741c5235ac63ae0f8917 F ext/misc/uuid.c 5bb2264c1b64d163efa46509544fd7500cb8769cb7c16dd52052da8d961505cf F ext/misc/vfslog.c 3932ab932eeb2601dbc4447cb14d445aaa9fbe43b863ef5f014401c3420afd20 F ext/misc/vfsstat.c a85df08654743922a19410d7b1e3111de41bb7cd07d20dd16eda4e2b808d269d -F ext/misc/vfstrace.c 03f90dd465968e01f5d1d3e79c36cbc53a5bfe1bd55d239435ce94df19d5b0ac w src/test_vfstrace.c +F ext/misc/vfstrace.c 03f90dd465968e01f5d1d3e79c36cbc53a5bfe1bd55d239435ce94df19d5b0ac F ext/misc/vtablog.c 1100250ce8782db37c833e3a9a5c9a3ecf1af5e15b8325572b82e6e0a138ffb5 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd F ext/misc/wholenumber.c 0fa0c082676b7868bf2fa918e911133f2b349bcdceabd1198bba5f65b4fc0668 @@ -714,7 +714,7 @@ F src/callback.c db3a45e376deff6a16c0058163fe0ae2b73a2945f3f408ca32cf74960b28d49 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c b224d3db0f28c4a5f1407c50107a0a8133bd244ff3c7f6f8cedeb896a8cf1b64 F src/date.c 89ce1ff20512a7fa5070ba6e7dd5c171148ca7d580955795bf97c79c2456144a -F src/dbpage.c 80e46e1df623ec40486da7a5086cb723b0275a6e2a7b01d9f9b5da0f04ba2782 +F src/dbpage.c f8c93e845d1093554247c1e757cb443fc48ffbcb112cecfdebeca4b6aa6e5c6e F src/dbstat.c 73362c0df0f40ad5523a6f5501224959d0976757b511299bf892313e79d14f5c F src/delete.c 444c4d1eaac40103461e3b6f0881846dd3aafc1cec1dd169d3482fa331667da7 F src/expr.c 6d5f2c38fe3ec06a7eac599dac822788b36064124e20112a844e9cd5156cb239 @@ -2212,9 +2212,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P aa75e701de61fe63ec15c35d70e53e950ff73b0dcb0d871dd8721412f3af297a e8f2d6313075c92fdeebcdfd8b50f43e9d45225890b2ef6b77148a766a42e940 -R d410fcc13042586e54655831c4a948f2 -T +closed e8f2d6313075c92fdeebcdfd8b50f43e9d45225890b2ef6b77148a766a42e940 +P 123cb1f579daec3ed092fe9dd1bc0d3250f2b56d4cda1efa92af139029e112e2 +R 3c119894ad399726f0c98d7eab61a4dc U drh -Z 1f0ae8b76c17f7c9bad8f3f1287c0801 +Z 54e5dc5728a062a0fc19b7e3f1b82dff # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 9b5f9e95bc..49f91d3e2e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -123cb1f579daec3ed092fe9dd1bc0d3250f2b56d4cda1efa92af139029e112e2 +fe0d67e72d4228661c021f227bfc0d5ddb1b726db0f36c7221ead8dd8bd1dc73 diff --git a/src/dbpage.c b/src/dbpage.c index 73c31f0dab..9740b418a3 100644 --- a/src/dbpage.c +++ b/src/dbpage.c @@ -321,7 +321,6 @@ static int dbpageUpdate( DbPage *pDbPage = 0; int rc = SQLITE_OK; char *zErr = 0; - const char *zSchema; int iDb; Btree *pBt; Pager *pPager; @@ -336,21 +335,27 @@ static int dbpageUpdate( zErr = "cannot delete"; goto update_fail; } - pgno = sqlite3_value_int(argv[0]); - if( sqlite3_value_type(argv[0])==SQLITE_NULL - || (Pgno)sqlite3_value_int(argv[1])!=pgno - ){ - zErr = "cannot insert"; - goto update_fail; + if( sqlite3_value_type(argv[0])==SQLITE_NULL ){ + pgno = (Pgno)sqlite3_value_int(argv[2]); + }else{ + pgno = sqlite3_value_int(argv[0]); + if( (Pgno)sqlite3_value_int(argv[1])!=pgno ){ + zErr = "cannot insert"; + goto update_fail; + } } - zSchema = (const char*)sqlite3_value_text(argv[4]); - iDb = ALWAYS(zSchema) ? sqlite3FindDbName(pTab->db, zSchema) : -1; - if( NEVER(iDb<0) ){ - zErr = "no such schema"; - goto update_fail; + if( sqlite3_value_type(argv[4])==SQLITE_NULL ){ + iDb = 0; + }else{ + const char *zSchema = (const char*)sqlite3_value_text(argv[4]); + iDb = zSchema ? sqlite3FindDbName(pTab->db, zSchema) : -1; + if( iDb<0 ){ + zErr = "no such schema"; + goto update_fail; + } } pBt = pTab->db->aDb[iDb].pBt; - if( NEVER(pgno<1) || NEVER(pBt==0) || NEVER(pgno>sqlite3BtreeLastPage(pBt)) ){ + if( pgno<1 || NEVER(pBt==0) ){ zErr = "bad page number"; goto update_fail; } From cd889c7a88b2bd23ac71a897c54c43c84eee972d Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 9 Sep 2024 19:12:57 +0000 Subject: [PATCH 30/36] Fix an OOM-handling problem affecting locale=1 fts5 tables. FossilOrigin-Name: d8103684f660ff9b3186d0f89afb113ca580bd16f0bf413ed8a9434236b54426 --- ext/fts5/fts5_main.c | 16 ++++++++++++---- ext/fts5/test/fts5faultI.test | 13 ++++++++++++- manifest | 16 ++++++++-------- manifest.uuid | 2 +- 4 files changed, 33 insertions(+), 14 deletions(-) diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index 0cc6fd30a9..6ccca8a3ee 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -90,7 +90,7 @@ struct Fts5Global { ** Size of header on fts5_locale() values. And macro to access a buffer ** containing a copy of the header from an Fts5Config pointer. */ -#define FTS5_LOCALE_HDR_SIZE sizeof( ((Fts5Global*)0)->aLocaleHdr ) +#define FTS5_LOCALE_HDR_SIZE ((int)sizeof( ((Fts5Global*)0)->aLocaleHdr )) #define FTS5_LOCALE_HDR(pConfig) ((const u8*)(pConfig->pGlobal->aLocaleHdr)) @@ -1284,8 +1284,16 @@ void sqlite3Fts5ClearLocale(Fts5Config *pConfig){ int sqlite3Fts5IsLocaleValue(Fts5Config *pConfig, sqlite3_value *pVal){ int ret = 0; if( sqlite3_value_type(pVal)==SQLITE_BLOB ){ - if( sqlite3_value_bytes(pVal)>(int)FTS5_LOCALE_HDR_SIZE - && 0==memcmp(sqlite3_value_blob(pVal), FTS5_LOCALE_HDR(pConfig), 4) + /* Call sqlite3_value_bytes() after sqlite3_value_blob() in this case. + ** If the blob was created using zeroblob(), then sqlite3_value_blob() + ** may call malloc(). If this malloc() fails, then the values returned + ** by both value_blob() and value_bytes() will be 0. If value_bytes() were + ** called first, then the NULL pointer returned by value_blob() might + ** be dereferenced. */ + const u8 *pBlob = sqlite3_value_blob(pVal); + int nBlob = sqlite3_value_bytes(pVal); + if( nBlob>FTS5_LOCALE_HDR_SIZE + && 0==memcmp(pBlob, FTS5_LOCALE_HDR(pConfig), FTS5_LOCALE_HDR_SIZE) ){ ret = 1; } @@ -3011,7 +3019,7 @@ static void fts5ExtractValueFromColumn( int ii; if( pConfig->eContent==FTS5_CONTENT_EXTERNAL ){ - if( nBlob<(int)FTS5_LOCALE_HDR_SIZE + if( nBlob Date: Tue, 10 Sep 2024 16:19:31 +0000 Subject: [PATCH 31/36] Alternative implementation of fts5 locale=1 feature that allows blobs to be stored in indexed columns of fts5 locale=1 tables. FossilOrigin-Name: 55c5c119a0a77fac2c9f46d718ef78c0f33ed3520e10c240cf5bf1801e0586ee --- ext/fts5/fts5Int.h | 13 +- ext/fts5/fts5_config.c | 9 + ext/fts5/fts5_main.c | 349 +++++++++++----------------------- ext/fts5/fts5_storage.c | 243 ++++++++++++++++------- ext/fts5/test/fts5blob.test | 6 +- ext/fts5/test/fts5faultI.test | 5 +- ext/fts5/test/fts5locale.test | 115 ++++++++--- ext/fts5/test/fts5simple.test | 1 + manifest | 29 +-- manifest.uuid | 2 +- 10 files changed, 408 insertions(+), 364 deletions(-) diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index 0b8851d227..b15521f163 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -636,18 +636,13 @@ Fts5Table *sqlite3Fts5TableFromCsrid(Fts5Global*, i64); int sqlite3Fts5FlushToDisk(Fts5Table*); -int sqlite3Fts5ExtractText( - Fts5Config *pConfig, - sqlite3_value *pVal, /* Value to extract text from */ - int bContent, /* Loaded from content table */ - int *pbResetTokenizer, /* OUT: True if ClearLocale() required */ - const char **ppText, /* OUT: Pointer to text buffer */ - int *pnText /* OUT: Size of (*ppText) in bytes */ -); - void sqlite3Fts5ClearLocale(Fts5Config *pConfig); +void sqlite3Fts5SetLocale(Fts5Config *pConfig, const char *pLoc, int nLoc); int sqlite3Fts5IsLocaleValue(Fts5Config *pConfig, sqlite3_value *pVal); +int sqlite3Fts5DecodeLocaleValue(sqlite3_value *pVal, + const char **ppText, int *pnText, const char **ppLoc, int *pnLoc +); /* ** End of interface to code in fts5.c. diff --git a/ext/fts5/fts5_config.c b/ext/fts5/fts5_config.c index 3cb1bd3bea..1fade5f7a8 100644 --- a/ext/fts5/fts5_config.c +++ b/ext/fts5/fts5_config.c @@ -516,6 +516,15 @@ static int fts5ConfigMakeExprlist(Fts5Config *p){ } } } + if( p->eContent==FTS5_CONTENT_NORMAL && p->bLocale ){ + for(i=0; inCol; i++){ + if( p->abUnindexed[i]==0 ){ + sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", T.l%d", i); + }else{ + sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", NULL"); + } + } + } assert( p->zContentExprlist==0 ); p->zContentExprlist = (char*)buf.p; diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index 6ccca8a3ee..b4922a8b6f 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -1259,7 +1259,7 @@ static void fts5SetVtabError(Fts5FullTable *p, const char *zFormat, ...){ ** valid until after the final call to sqlite3Fts5Tokenize() that will use ** the locale. */ -static void fts5SetLocale( +static void sqlite3Fts5SetLocale( Fts5Config *pConfig, const char *zLocale, int nLocale @@ -1270,11 +1270,10 @@ static void fts5SetLocale( } /* -** Clear any locale configured by an earlier call to fts5SetLocale() or -** sqlite3Fts5ExtractText(). +** Clear any locale configured by an earlier call to sqlite3Fts5SetLocale(). */ void sqlite3Fts5ClearLocale(Fts5Config *pConfig){ - fts5SetLocale(pConfig, 0, 0); + sqlite3Fts5SetLocale(pConfig, 0, 0); } /* @@ -1302,118 +1301,33 @@ int sqlite3Fts5IsLocaleValue(Fts5Config *pConfig, sqlite3_value *pVal){ } /* -** This function is used to extract utf-8 text from an sqlite3_value. This -** is usually done in order to tokenize it. For example, when: -** -** * a value is written to an fts5 table, -** * a value is deleted from an FTS5_CONTENT_NORMAL table, -** * a value containing a query expression is passed to xFilter() -** -** and so on. -** -** This function handles 2 cases: -** -** 1) Ordinary values. The text can be extracted from these using -** sqlite3_value_text(). -** -** 2) Combination text/locale blobs created by fts5_locale(). There -** are several cases for these: -** -** * Blobs that have the 16-byte header, and -** * Blobs read from the content table of a locale=1 regular -** content table. -** -** The first case above has the 16 byte FTS5_LOCALE_HDR(pConfig) -** header. It is an error if a blob read from the content table of -** an external content table does not have the required header. A blob -** read from the content table of a regular locale=1 table does not -** have the header. This is to save space. -** -** If successful, SQLITE_OK is returned and output parameters (*ppText) -** and (*pnText) are set to point to a buffer containing the extracted utf-8 -** text and its length in bytes, respectively. The buffer is not -** nul-terminated. It has the same lifetime as the sqlite3_value object -** from which it is extracted. -** -** Parameter bContent must be true if the value was read from an indexed -** column (i.e. not UNINDEXED) of the on disk content. -** -** If pbResetTokenizer is not NULL and if case (2) is used, then -** fts5SetLocale() is called to ensure subsequent sqlite3Fts5Tokenize() calls -** use the locale. In this case (*pbResetTokenizer) is set to true before -** returning, to indicate that the caller must call sqlite3Fts5ClearLocale() -** to clear the locale after tokenizing the text. +** Value pVal is guaranteed to be an fts5_locale() value. */ -int sqlite3Fts5ExtractText( - Fts5Config *pConfig, - sqlite3_value *pVal, /* Value to extract text from */ - int bContent, /* True if indexed table content */ - int *pbResetTokenizer, /* OUT: True if xSetLocale(NULL) required */ - const char **ppText, /* OUT: Pointer to text buffer */ - int *pnText /* OUT: Size of (*ppText) in bytes */ +int sqlite3Fts5DecodeLocaleValue( + sqlite3_value *pVal, + const char **ppText, + int *pnText, + const char **ppLoc, + int *pnLoc ){ - const char *pText = 0; - int nText = 0; - int rc = SQLITE_OK; + const char *p = sqlite3_value_blob(pVal); + int n = sqlite3_value_bytes(pVal); + int nLoc = 0; - /* 0: Do not decode blob - ** 1: Decode blob, expect fts5_locale() header - ** 2: Decode blob, expect no fts5_locale() header - */ - int eDecodeBlob = 0; + assert( sqlite3_value_type(pVal)==SQLITE_BLOB ); + assert( n>FTS5_LOCALE_HDR_SIZE ); - assert( pbResetTokenizer==0 || *pbResetTokenizer==0 ); - assert( bContent==0 || pConfig->eContent!=FTS5_CONTENT_NONE ); - - if( sqlite3_value_type(pVal)==SQLITE_BLOB ){ - if( bContent - && pConfig->bLocale - && pConfig->eContent==FTS5_CONTENT_NORMAL - ){ - eDecodeBlob = 2; - }else if( sqlite3Fts5IsLocaleValue(pConfig, pVal) ){ - eDecodeBlob = 1; - }else if( bContent && pConfig->bLocale ){ - return SQLITE_ERROR; + for(nLoc=FTS5_LOCALE_HDR_SIZE; p[nLoc]; nLoc++){ + if( nLoc==(n-1) ){ + return SQLITE_MISMATCH; } } + *ppLoc = &p[FTS5_LOCALE_HDR_SIZE]; + *pnLoc = nLoc - FTS5_LOCALE_HDR_SIZE; - if( eDecodeBlob ){ - const u8 *pBlob = sqlite3_value_blob(pVal); - int nBlob = sqlite3_value_bytes(pVal); - int nLocale = 0; - - /* Unless this blob was read from the %_content table of an - ** FTS5_CONTENT_NORMAL table, it should have the 4 byte fts5_locale() - ** header. Check for this. If it is not found, return an error. */ - if( eDecodeBlob==1 ){ - pBlob += FTS5_LOCALE_HDR_SIZE; - nBlob -= FTS5_LOCALE_HDR_SIZE; - } - - for(nLocale=0; nLocalenCol; ii++){ - sqlite3_value *pVal = apVal[ii+2]; - if( sqlite3_value_type(pVal)==SQLITE_BLOB ){ - int isLocale = sqlite3Fts5IsLocaleValue(pConfig, pVal); - if( pConfig->bLocale ){ - if( isLocale==0 && pConfig->abUnindexed[ii]==0 ){ - rc = SQLITE_MISMATCH; - goto update_out; - } - }else{ - if( isLocale ){ - fts5SetVtabError(pTab, "fts5_locale() requires locale=1"); - rc = SQLITE_MISMATCH; - goto update_out; - } + /* It is an error to write an fts5_locale() value to a table without + ** the locale=1 option. */ + if( pConfig->bLocale==0 ){ + int ii; + for(ii=0; iinCol; ii++){ + sqlite3_value *pVal = apVal[ii+2]; + if( sqlite3Fts5IsLocaleValue(pConfig, pVal) ){ + fts5SetVtabError(pTab, "fts5_locale() requires locale=1"); + rc = SQLITE_MISMATCH; + goto update_out; } } } @@ -2166,11 +2072,11 @@ static int fts5ApiTokenize_v2( Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); int rc = SQLITE_OK; - fts5SetLocale(pTab->pConfig, pLoc, nLoc); + sqlite3Fts5SetLocale(pTab->pConfig, pLoc, nLoc); rc = sqlite3Fts5Tokenize(pTab->pConfig, FTS5_TOKENIZE_AUX, pText, nText, pUserData, xToken ); - fts5SetLocale(pTab->pConfig, 0, 0); + sqlite3Fts5SetLocale(pTab->pConfig, 0, 0); return rc; } @@ -2198,6 +2104,38 @@ static int fts5ApiPhraseSize(Fts5Context *pCtx, int iPhrase){ return sqlite3Fts5ExprPhraseSize(pCsr->pExpr, iPhrase); } +/* +** Argument pStmt is an SQL statement of the type used by Fts5Cursor. +*/ +static int fts5TextFromStmt( + Fts5Config *pConfig, + sqlite3_stmt *pStmt, + int iCol, + const char **ppText, + int *pnText +){ + sqlite3_value *pVal = sqlite3_column_value(pStmt, iCol+1); + const char *pLoc = 0; + int nLoc = 0; + int rc = SQLITE_OK; + + if( pConfig->bLocale + && pConfig->eContent==FTS5_CONTENT_EXTERNAL + && sqlite3Fts5IsLocaleValue(pConfig, pVal) + ){ + rc = sqlite3Fts5DecodeLocaleValue(pVal, ppText, pnText, &pLoc, &nLoc); + }else{ + *ppText = (const char*)sqlite3_value_text(pVal); + *pnText = sqlite3_value_bytes(pVal); + if( pConfig->bLocale && pConfig->eContent==FTS5_CONTENT_NORMAL ){ + pLoc = (const char*)sqlite3_column_text(pStmt, iCol+1+pConfig->nCol); + nLoc = sqlite3_column_bytes(pStmt, iCol+1+pConfig->nCol); + } + } + sqlite3Fts5SetLocale(pConfig, pLoc, nLoc); + return rc; +} + static int fts5ApiColumnText( Fts5Context *pCtx, int iCol, @@ -2217,10 +2155,8 @@ static int fts5ApiColumnText( }else{ rc = fts5SeekCursor(pCsr, 0); if( rc==SQLITE_OK ){ - Fts5Config *pConfig = pTab->pConfig; - int bContent = (pConfig->abUnindexed[iCol]==0); - sqlite3_value *pVal = sqlite3_column_value(pCsr->pStmt, iCol+1); - sqlite3Fts5ExtractText(pConfig, pVal, bContent, 0, pz, pn); + rc = fts5TextFromStmt(pTab->pConfig, pCsr->pStmt, iCol, pz, pn); + sqlite3Fts5ClearLocale(pTab->pConfig); } } return rc; @@ -2262,17 +2198,15 @@ static int fts5CsrPoslist( rc = fts5SeekCursor(pCsr, 0); } for(i=0; inCol && rc==SQLITE_OK; i++){ - sqlite3_value *pVal = sqlite3_column_value(pCsr->pStmt, i+1); const char *z = 0; int n = 0; - int bReset = 0; - rc = sqlite3Fts5ExtractText(pConfig, pVal, 1, &bReset, &z, &n); + rc = fts5TextFromStmt(pConfig, pCsr->pStmt, i, &z, &n); if( rc==SQLITE_OK ){ rc = sqlite3Fts5ExprPopulatePoslists( pConfig, pCsr->pExpr, aPopulator, i, z, n ); } - if( bReset ) sqlite3Fts5ClearLocale(pConfig); + sqlite3Fts5ClearLocale(pConfig); } sqlite3_free(aPopulator); @@ -2458,17 +2392,14 @@ static int fts5ApiColumnSize(Fts5Context *pCtx, int iCol, int *pnToken){ if( pConfig->abUnindexed[i]==0 ){ const char *z = 0; int n = 0; - int bReset = 0; - sqlite3_value *pVal = sqlite3_column_value(pCsr->pStmt, i+1); - pCsr->aColumnSize[i] = 0; - rc = sqlite3Fts5ExtractText(pConfig, pVal, 1, &bReset, &z, &n); + rc = fts5TextFromStmt(pConfig, pCsr->pStmt, i, &z, &n); if( rc==SQLITE_OK ){ rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_AUX, z, n, (void*)&pCsr->aColumnSize[i], fts5ColumnSizeCb ); - if( bReset ) sqlite3Fts5ClearLocale(pConfig); } + sqlite3Fts5ClearLocale(pConfig); } } } @@ -2740,37 +2671,14 @@ static int fts5ApiColumnLocale( ){ rc = fts5SeekCursor(pCsr, 0); if( rc==SQLITE_OK ){ - /* Load the value into pVal. pVal is a locale/text pair iff: - ** - ** 1) It is an SQLITE_BLOB, and - ** 2) Either the FTS5_LOCALE_HDR header is present, or else the - ** value was loaded from an FTS5_CONTENT_NORMAL table. - ** - ** If condition (1) is met but condition (2) is not, it is an error. - */ - sqlite3_value *pVal = sqlite3_column_value(pCsr->pStmt, iCol+1); - if( sqlite3_value_type(pVal)==SQLITE_BLOB ){ - const u8 *pBlob = (const u8*)sqlite3_value_blob(pVal); - int nBlob = sqlite3_value_bytes(pVal); - if( pConfig->eContent==FTS5_CONTENT_EXTERNAL ){ - if( sqlite3Fts5IsLocaleValue(pConfig, pVal)==0 ){ - rc = SQLITE_ERROR; - } - pBlob += FTS5_LOCALE_HDR_SIZE; - nBlob -= FTS5_LOCALE_HDR_SIZE; - } - if( rc==SQLITE_OK ){ - int nLocale = 0; - for(nLocale=0; nLocalepStmt, iCol, &zDummy, &nDummy); + if( rc==SQLITE_OK ){ + *pzLocale = pConfig->t.pLocale; + *pnLocale = pConfig->t.nLocale; } + sqlite3Fts5ClearLocale(pConfig); } } @@ -2991,58 +2899,6 @@ static int fts5PoslistBlob(sqlite3_context *pCtx, Fts5Cursor *pCsr){ return rc; } -/* -** Value pVal was read from column iCol of the FTS5 table. This function -** returns it to the owner of pCtx via a call to an sqlite3_result_xxx() -** function. This function deals with the same cases as -** sqlite3Fts5ExtractText(): -** -** 1) Ordinary values. These can be returned using sqlite3_result_value(). -** -** 2) Blobs from fts5_locale(). The text is extracted from these and -** returned via sqlite3_result_text(). The locale is discarded. -*/ -static void fts5ExtractValueFromColumn( - sqlite3_context *pCtx, - Fts5Config *pConfig, - int iCol, - sqlite3_value *pVal -){ - assert( pConfig->eContent!=FTS5_CONTENT_NONE ); - - if( pConfig->bLocale - && sqlite3_value_type(pVal)==SQLITE_BLOB - && pConfig->abUnindexed[iCol]==0 - ){ - const u8 *pBlob = sqlite3_value_blob(pVal); - int nBlob = sqlite3_value_bytes(pVal); - int ii; - - if( pConfig->eContent==FTS5_CONTENT_EXTERNAL ){ - if( nBlobpStmt, iCol+1); - fts5ExtractValueFromColumn(pCtx, pConfig, iCol, pVal); + if( pConfig->bLocale + && pConfig->eContent==FTS5_CONTENT_EXTERNAL + && sqlite3Fts5IsLocaleValue(pConfig, pVal) + ){ + const char *z = 0; + int n = 0; + rc = fts5TextFromStmt(pConfig, pCsr->pStmt, iCol, &z, &n); + if( rc==SQLITE_OK ){ + sqlite3_result_text(pCtx, z, n, SQLITE_TRANSIENT); + } + }else{ + sqlite3_result_value(pCtx, pVal); + } } + pConfig->pzErrmsg = 0; } } diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c index 3bf20b3390..e8649c703f 100644 --- a/ext/fts5/fts5_storage.c +++ b/ext/fts5/fts5_storage.c @@ -73,6 +73,30 @@ struct Fts5Storage { #define FTS5_STMT_REPLACE_CONFIG 10 #define FTS5_STMT_SCAN 11 +/* +** Return a pointer to a buffer obtained from sqlite3_malloc() that contains +** nBind comma-separated question marks. e.g. if nBind is passed 5, this +** function returns "?,?,?,?,?". +** +** If *pRc is not SQLITE_OK when this function is called, it is a no-op and +** NULL is returned immediately. Or, if the attempt to malloc a buffer +** fails, then *pRc is set to SQLITE_NOMEM and NULL is returned. Otherwise, +** if it is SQLITE_OK when this function is called and the malloc() succeeds, +** *pRc is left unchanged. +*/ +static char *fts5BindingsList(int *pRc, int nBind){ + char *zBind = sqlite3Fts5MallocZero(pRc, 1 + nBind*2); + if( zBind ){ + int ii; + for(ii=0; iinCol + 1; + case FTS5_STMT_INSERT_CONTENT: { + int nCol = 0; char *zBind; int i; - zBind = sqlite3_malloc64(1 + nCol*2); - if( zBind ){ - for(i=0; inCol; + if( pC->bLocale ){ + for(i=0; inCol; i++){ + if( pC->abUnindexed[i]==0 ) nCol++; } - zBind[i*2-1] = '\0'; + } + + zBind = fts5BindingsList(&rc, nCol); + if( zBind ){ zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName, zBind); sqlite3_free(zBind); } @@ -344,7 +369,7 @@ int sqlite3Fts5StorageOpen( if( bCreate ){ if( pConfig->eContent==FTS5_CONTENT_NORMAL ){ int nDefn = 32 + pConfig->nCol*10; - char *zDefn = sqlite3_malloc64(32 + (sqlite3_int64)pConfig->nCol * 10); + char *zDefn = sqlite3_malloc64(32 + (sqlite3_int64)pConfig->nCol * 20); if( zDefn==0 ){ rc = SQLITE_NOMEM; }else{ @@ -356,6 +381,14 @@ int sqlite3Fts5StorageOpen( sqlite3_snprintf(nDefn-iOff, &zDefn[iOff], ", c%d", i); iOff += (int)strlen(&zDefn[iOff]); } + if( pConfig->bLocale ){ + for(i=0; inCol; i++){ + if( pConfig->abUnindexed[i]==0 ){ + sqlite3_snprintf(nDefn-iOff, &zDefn[iOff], ", l%d", i); + iOff += (int)strlen(&zDefn[iOff]); + } + } + } rc = sqlite3Fts5CreateTable(pConfig, "content", zDefn, 0, pzErr); } sqlite3_free(zDefn); @@ -507,7 +540,8 @@ static int fts5StorageDeleteFromIndex( sqlite3_value *pVal = 0; const char *pText = 0; int nText = 0; - int bReset = 0; + const char *pLoc = 0; + int nLoc = 0; assert( pSeek==0 || apVal==0 ); assert( pSeek!=0 || apVal!=0 ); @@ -517,10 +551,19 @@ static int fts5StorageDeleteFromIndex( pVal = apVal[iCol-1]; } - rc = sqlite3Fts5ExtractText( - pConfig, pVal, pSeek!=0, &bReset, &pText, &nText - ); + if( pConfig->bLocale && sqlite3Fts5IsLocaleValue(pConfig, pVal) ){ + rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc); + }else{ + pText = (const char*)sqlite3_value_text(pVal); + nText = sqlite3_value_bytes(pVal); + if( pConfig->bLocale && pSeek ){ + pLoc = (const char*)sqlite3_column_text(pSeek, iCol + pConfig->nCol); + nLoc = sqlite3_column_bytes(pSeek, iCol + pConfig->nCol); + } + } + if( rc==SQLITE_OK ){ + sqlite3Fts5SetLocale(pConfig, pLoc, nLoc); ctx.szCol = 0; rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT, pText, nText, (void*)&ctx, fts5StorageInsertCallback @@ -529,7 +572,7 @@ static int fts5StorageDeleteFromIndex( if( rc==SQLITE_OK && p->aTotalSize[iCol-1]<0 ){ rc = FTS5_CORRUPT; } - if( bReset ) sqlite3Fts5ClearLocale(pConfig); + sqlite3Fts5ClearLocale(pConfig); } } } @@ -788,20 +831,35 @@ int sqlite3Fts5StorageRebuild(Fts5Storage *p){ for(ctx.iCol=0; rc==SQLITE_OK && ctx.iColnCol; ctx.iCol++){ ctx.szCol = 0; if( pConfig->abUnindexed[ctx.iCol]==0 ){ - int bReset = 0; /* True if tokenizer locale must be reset */ int nText = 0; /* Size of pText in bytes */ const char *pText = 0; /* Pointer to buffer containing text value */ - sqlite3_value *pVal = sqlite3_column_value(pScan, ctx.iCol+1); + int nLoc = 0; /* Size of pLoc in bytes */ + const char *pLoc = 0; /* Pointer to buffer containing text value */ + + sqlite3_value *pVal = sqlite3_column_value(pScan, ctx.iCol+1); + if( pConfig->eContent==FTS5_CONTENT_EXTERNAL + && sqlite3Fts5IsLocaleValue(pConfig, pVal) + ){ + rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc); + }else{ + pText = (const char*)sqlite3_value_text(pVal); + nText = sqlite3_value_bytes(pVal); + if( pConfig->bLocale ){ + int iCol = ctx.iCol + 1 + pConfig->nCol; + pLoc = (const char*)sqlite3_column_text(pScan, iCol); + nLoc = sqlite3_column_bytes(pScan, iCol); + } + } - rc = sqlite3Fts5ExtractText(pConfig, pVal, 1, &bReset, &pText, &nText); if( rc==SQLITE_OK ){ + sqlite3Fts5SetLocale(pConfig, pLoc, nLoc); rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT, pText, nText, (void*)&ctx, fts5StorageInsertCallback ); - if( bReset ) sqlite3Fts5ClearLocale(pConfig); + sqlite3Fts5ClearLocale(pConfig); } } sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol); @@ -884,33 +942,46 @@ int sqlite3Fts5StorageContentInsert( }else{ sqlite3_stmt *pInsert = 0; /* Statement to write %_content table */ int i; /* Counter variable */ + int nIndexed = 0; /* Number indexed columns seen */ rc = fts5StorageGetStmt(p, FTS5_STMT_INSERT_CONTENT, &pInsert, 0); - for(i=1; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){ + if( pInsert ) sqlite3_clear_bindings(pInsert); + + /* Bind the rowid value */ + sqlite3_bind_value(pInsert, 1, apVal[1]); + + /* Loop through values for user-defined columns. i=2 is the leftmost + ** user-defined column. As is column 1 of pSavedRow. */ + for(i=2; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){ + int bUnindexed = pConfig->abUnindexed[i-2]; sqlite3_value *pVal = apVal[i]; + + nIndexed += !bUnindexed; if( sqlite3_value_nochange(pVal) && p->pSavedRow ){ /* This is an UPDATE statement, and column (i-2) was not modified. ** Retrieve the value from Fts5Storage.pSavedRow instead. */ pVal = sqlite3_column_value(p->pSavedRow, i-1); - }else if( sqlite3_value_type(pVal)==SQLITE_BLOB && pConfig->bLocale ){ - assert( pConfig->bLocale ); - assert( i>1 ); - if( pConfig->abUnindexed[i-2] ){ - if( sqlite3Fts5IsLocaleValue(pConfig, pVal) ){ - /* At attempt to insert an fts5_locale() value into an UNINDEXED - ** column. Strip the locale away and just bind the text. */ - const char *pText = 0; - int nText = 0; - rc = sqlite3Fts5ExtractText(pConfig, pVal, 0, 0, &pText, &nText); - sqlite3_bind_text(pInsert, i, pText, nText, SQLITE_TRANSIENT); - continue; - } - }else{ - const u8 *pBlob = (const u8*)sqlite3_value_blob(pVal); - int nBlob = sqlite3_value_bytes(pVal); - assert( nBlob>4 ); - sqlite3_bind_blob(pInsert, i, pBlob+16, nBlob-16, SQLITE_TRANSIENT); - continue; + if( pConfig->bLocale && bUnindexed==0 ){ + sqlite3_bind_value(pInsert, pConfig->nCol + 1 + nIndexed, + sqlite3_column_value(p->pSavedRow, pConfig->nCol + i - 1) + ); } + }else if( sqlite3Fts5IsLocaleValue(pConfig, pVal) ){ + const char *pText = 0; + const char *pLoc = 0; + int nText = 0; + int nLoc = 0; + assert( pConfig->bLocale ); + + rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc); + if( rc==SQLITE_OK ){ + sqlite3_bind_text(pInsert, i, pText, nText, SQLITE_TRANSIENT); + if( bUnindexed==0 ){ + int iLoc = pConfig->nCol + 1 + nIndexed; + sqlite3_bind_text(pInsert, iLoc, pLoc, nLoc, SQLITE_TRANSIENT); + } + } + + continue; } rc = sqlite3_bind_value(pInsert, i, pVal); @@ -948,23 +1019,37 @@ int sqlite3Fts5StorageIndexInsert( for(ctx.iCol=0; rc==SQLITE_OK && ctx.iColnCol; ctx.iCol++){ ctx.szCol = 0; if( pConfig->abUnindexed[ctx.iCol]==0 ){ - int bReset = 0; /* True if tokenizer locale must be reset */ int nText = 0; /* Size of pText in bytes */ const char *pText = 0; /* Pointer to buffer containing text value */ + int nLoc = 0; /* Size of pText in bytes */ + const char *pLoc = 0; /* Pointer to buffer containing text value */ + sqlite3_value *pVal = apVal[ctx.iCol+2]; - int bDisk = 0; if( p->pSavedRow && sqlite3_value_nochange(pVal) ){ pVal = sqlite3_column_value(p->pSavedRow, ctx.iCol+1); - bDisk = 1; + if( pConfig->eContent==FTS5_CONTENT_NORMAL && pConfig->bLocale ){ + int iCol = ctx.iCol + 1 + pConfig->nCol; + pLoc = (const char*)sqlite3_column_text(p->pSavedRow, iCol); + nLoc = sqlite3_column_bytes(p->pSavedRow, iCol); + } + }else{ + pVal = apVal[ctx.iCol+2]; } - rc = sqlite3Fts5ExtractText(pConfig, pVal, bDisk, &bReset, &pText,&nText); + + if( pConfig->bLocale && sqlite3Fts5IsLocaleValue(pConfig, pVal) ){ + rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc); + }else{ + pText = (const char*)sqlite3_value_text(pVal); + nText = sqlite3_value_bytes(pVal); + } + if( rc==SQLITE_OK ){ - assert( bReset==0 || pConfig->bLocale ); + sqlite3Fts5SetLocale(pConfig, pLoc, nLoc); rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT, pText, nText, (void*)&ctx, fts5StorageInsertCallback ); - if( bReset ) sqlite3Fts5ClearLocale(pConfig); + sqlite3Fts5ClearLocale(pConfig); } } sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol); @@ -1129,37 +1214,61 @@ int sqlite3Fts5StorageIntegrity(Fts5Storage *p, int iArg){ rc = sqlite3Fts5TermsetNew(&ctx.pTermset); } for(i=0; rc==SQLITE_OK && inCol; i++){ - if( pConfig->abUnindexed[i] ) continue; - ctx.iCol = i; - ctx.szCol = 0; - if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){ - rc = sqlite3Fts5TermsetNew(&ctx.pTermset); - } - if( rc==SQLITE_OK ){ - int bReset = 0; /* True if tokenizer locale must be reset */ - int nText = 0; /* Size of pText in bytes */ - const char *pText = 0; /* Pointer to buffer containing text value */ + if( pConfig->abUnindexed[i]==0 ){ + const char *pText = 0; + int nText = 0; + const char *pLoc = 0; + int nLoc = 0; + sqlite3_value *pVal = sqlite3_column_value(pScan, i+1); + + if( pConfig->eContent==FTS5_CONTENT_EXTERNAL + && sqlite3Fts5IsLocaleValue(pConfig, pVal) + ){ + rc = sqlite3Fts5DecodeLocaleValue( + pVal, &pText, &nText, &pLoc, &nLoc + ); + }else{ + if( pConfig->eContent==FTS5_CONTENT_NORMAL && pConfig->bLocale ){ + int iCol = i + 1 + pConfig->nCol; + pLoc = (const char*)sqlite3_column_text(pScan, iCol); + nLoc = sqlite3_column_bytes(pScan, iCol); + } + pText = (const char*)sqlite3_value_text(pVal); + nText = sqlite3_value_bytes(pVal); + } + + ctx.iCol = i; + ctx.szCol = 0; + + if( rc==SQLITE_OK && pConfig->eDetail==FTS5_DETAIL_COLUMNS ){ + rc = sqlite3Fts5TermsetNew(&ctx.pTermset); + } - rc = sqlite3Fts5ExtractText(pConfig, - sqlite3_column_value(pScan, i+1), 1, &bReset, &pText, &nText - ); if( rc==SQLITE_OK ){ + sqlite3Fts5SetLocale(pConfig, pLoc, nLoc); rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT, pText, nText, (void*)&ctx, fts5StorageIntegrityCallback ); - if( bReset ) sqlite3Fts5ClearLocale(pConfig); + sqlite3Fts5ClearLocale(pConfig); + } + + /* If this is not a columnsize=0 database, check that the number + ** of tokens in the value matches the aColSize[] value read from + ** the %_docsize table. */ + if( rc==SQLITE_OK + && pConfig->bColumnsize + && ctx.szCol!=aColSize[i] + ){ + rc = FTS5_CORRUPT; + } + aTotalSize[i] += ctx.szCol; + if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){ + sqlite3Fts5TermsetFree(ctx.pTermset); + ctx.pTermset = 0; } - } - if( rc==SQLITE_OK && pConfig->bColumnsize && ctx.szCol!=aColSize[i] ){ - rc = FTS5_CORRUPT; - } - aTotalSize[i] += ctx.szCol; - if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){ - sqlite3Fts5TermsetFree(ctx.pTermset); - ctx.pTermset = 0; } } sqlite3Fts5TermsetFree(ctx.pTermset); diff --git a/ext/fts5/test/fts5blob.test b/ext/fts5/test/fts5blob.test index 4233719fbc..9348554104 100644 --- a/ext/fts5/test/fts5blob.test +++ b/ext/fts5/test/fts5blob.test @@ -103,13 +103,13 @@ do_execsql_test 3.0 { do_catchsql_test 3.1 { INSERT INTO x1(rowid, a, b) VALUES(113, 'hello world', X'123456'); -} {1 {datatype mismatch}} +} {0 {}} do_catchsql_test 3.2 { INSERT INTO x2(rowid, a, b) VALUES(113, 'hello world', X'123456'); -} {1 {datatype mismatch}} +} {0 {}} do_catchsql_test 3.3 { INSERT INTO x3(rowid, a, b) VALUES(113, 'hello world', X'123456'); -} {1 {datatype mismatch}} +} {0 {}} #-------------------------------------------------------------------------- diff --git a/ext/fts5/test/fts5faultI.test b/ext/fts5/test/fts5faultI.test index fe7c0aad50..72f25caee1 100644 --- a/ext/fts5/test/fts5faultI.test +++ b/ext/fts5/test/fts5faultI.test @@ -256,15 +256,16 @@ do_faultsim_test 10.1 -faults oom* -prep { faultsim_test_result {0 hello} } +breakpoint faultsim_save_and_close -do_faultsim_test 10.2 -faults oom* -prep { +do_faultsim_test 10.2 -faults oom-t* -prep { faultsim_restore_and_reopen } -body { execsql { INSERT INTO ft VALUES(zeroblob(10000)); } } -test { - faultsim_test_result {1 {datatype mismatch}} + faultsim_test_result {0 {}} } #------------------------------------------------------------------------- diff --git a/ext/fts5/test/fts5locale.test b/ext/fts5/test/fts5locale.test index f0df4969dc..e5799fb7fd 100644 --- a/ext/fts5/test/fts5locale.test +++ b/ext/fts5/test/fts5locale.test @@ -73,6 +73,7 @@ do_execsql_test 1.2 { SELECT rowid, a FROM t1( fts5_locale('reverse', 'abc') ); } {2 cba} + #------------------------------------------------------------------------- # Test that the locale= option exists and seems to accept values. And # that fts5_locale() values may only be inserted into an internal-content @@ -99,8 +100,11 @@ do_catchsql_test 2.3 { INSERT INTO b1(b1, rank) VALUES('locale', 0); } {1 {SQL logic error}} -do_execsql_test 2.4 { +do_execsql_test 2.4.1 { INSERT INTO b1 VALUES('abc', 'one two three'); +} + +do_execsql_test 2.4.2 { INSERT INTO b1 VALUES('def', fts5_locale('reverse', 'four five six')); } @@ -131,6 +135,7 @@ do_execsql_test 2.12 { SELECT quote(y) FROM b1('ruof') } { do_execsql_test 2.13 { INSERT INTO b1(b1) VALUES('integrity-check'); } + do_execsql_test 2.14 { INSERT INTO b1(b1) VALUES('rebuild'); } @@ -149,7 +154,6 @@ do_execsql_test 2.18 { INSERT INTO b1(rowid, x, y) VALUES( test_setsubtype(45, 76), 'abc def', 'def abc' ); - INSERT INTO b1(b1) VALUES('integrity-check'); } #------------------------------------------------------------------------- @@ -278,9 +282,9 @@ do_execsql_test 5.2 { } do_execsql_test 5.3 { - SELECT typeof(c0), typeof(c1) FROM t1_content + SELECT typeof(c0), typeof(c1), typeof(l0) FROM t1_content } { - blob text + text text text } #------------------------------------------------------------------------- @@ -305,37 +309,37 @@ foreach {tn opt} { fts5_aux_test_functions db - do_execsql_test 5.$tn.3 { + do_execsql_test 6.$tn.3 { SELECT fts5_test_columnsize(y1) FROM y1 } { 2 3 2 4 } - do_execsql_test 5.$tn.4 { + do_execsql_test 6.$tn.4 { SELECT rowid, fts5_test_columnsize(y1) FROM y1('shall'); } { 2 3 } - do_execsql_test 5.$tn.5 { + do_execsql_test 6.$tn.5 { SELECT rowid, fts5_test_columnsize(y1) FROM y1('shall'); } { 2 3 } - do_execsql_test 5.$tn.6 { + do_execsql_test 6.$tn.6 { SELECT rowid, fts5_test_columnsize(y1) FROM y1('have'); } { 4 4 } - do_execsql_test 5.$tn.7 { + do_execsql_test 6.$tn.7 { SELECT rowid, highlight(y1, 0, '[', ']') FROM y1('have'); } { 4 {which it hath been used to [have]} } - do_execsql_test 5.$tn.8 { + do_execsql_test 6.$tn.8 { SELECT rowid, highlight(y1, 0, '[', ']'), snippet(y1, 0, '[', ']', '...', 10) @@ -473,7 +477,7 @@ foreach_detail_mode $::testprefix { } foreach {tn v} { - 1 X'001122' + 1 X'001152' 2 X'0011223344' 3 X'00E0B2EB68656c6c6f' 4 X'00E0B2EB0068656c6c6f' @@ -484,7 +488,7 @@ foreach_detail_mode $::testprefix { do_catchsql_test 10.2.$tn.3 { INSERT INTO ft(ft) VALUES('rebuild'); - } {1 {SQL logic error}} + } {0 {}} do_catchsql_test 10.2.$tn.4 " SELECT * FROM ft( test_setsubtype($v, 76) ); @@ -494,23 +498,23 @@ foreach_detail_mode $::testprefix { INSERT INTO ft(rowid, x) VALUES(1, 'hello world'); } - if {"%DETAIL%"!="full"} { - do_catchsql_test 10.2.$tn.6 { + if {"%DETAIL%"=="full"} { + do_execsql_test 10.2.$tn.6 { SELECT fts5_test_poslist(ft) FROM ft('world'); - } {1 SQLITE_ERROR} + } {0.0.1} - do_catchsql_test 10.2.$tn.7 { + do_execsql_test 10.2.$tn.7.1 { SELECT fts5_test_columnsize(ft) FROM ft('world'); - } {1 SQLITE_ERROR} + } {1} - do_catchsql_test 10.2.$tn.7 { + do_execsql_test 10.2.$tn.7.2 { SELECT fts5_test_columnlocale(ft) FROM ft('world'); - } {1 SQLITE_ERROR} + } {{{}}} } do_catchsql_test 10.2.$tn.8 { - SELECT * FROM ft('hello') - } {1 {SQL logic error}} + SELECT count(*) FROM ft('hello') + } {0 1} do_catchsql_test 10.2.$tn.9 { PRAGMA integrity_check; @@ -527,7 +531,7 @@ foreach_detail_mode $::testprefix { do_catchsql_test 10.2.$tn.12 " INSERT INTO ft(rowid, x) VALUES(2, test_setsubtype($v,76) ) - " {1 {datatype mismatch}} + " {0 {}} do_execsql_test 10.2.$tn.13 { INSERT INTO ft2(rowid, x) VALUES(1, 'hello world'); @@ -536,7 +540,7 @@ foreach_detail_mode $::testprefix { do_catchsql_test 10.2.$tn.15 { PRAGMA integrity_check; - } {1 {SQL logic error}} + } {0 {{malformed inverted index for FTS5 table main.ft2}}} do_execsql_test 10.2.$tn.16 { DELETE FROM ft2_content; @@ -679,13 +683,66 @@ do_execsql_test 14.3 { UPDATE ft SET b = fts5_locale('en_AU', 'world'); } -do_catchsql_test 14.4 { - INSERT INTO ft VALUES(X'abcd', X'1234'); -} {1 {datatype mismatch}} - do_execsql_test 14.4 { - SELECT * FROM ft -} {hello world} + INSERT INTO ft VALUES(X'abcd', X'1234'); +} {} + +do_execsql_test 14.5 { + SELECT quote(a), quote(b) FROM ft +} {'hello' 'world' X'ABCD' X'1234'} + +do_execsql_test 14.6 { + DELETE FROM ft; + INSERT INTO ft VALUES(NULL, 'null'); + INSERT INTO ft VALUES(123, 'int'); + INSERT INTO ft VALUES(345.0, 'real'); + INSERT INTO ft VALUES('abc', 'text'); + INSERT INTO ft VALUES(fts5_locale('abc', 'def'), 'text'); + + SELECT a, typeof(a), b FROM ft +} { + {} null null + 123 integer int + 345.0 real real + abc text text + def text text +} + +do_execsql_test 14.7 { + SELECT quote(c0), typeof(c0) FROM ft_content +} { + NULL null + 123 integer + 345.0 real + 'abc' text + 'def' text +} + +#------------------------------------------------------------------------- +# Check that inserting UNINDEXED columns between indexed columns of a +# locale=1 table does not cause a problem. +# +reset_db +sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create +fts5_aux_test_functions db + +do_execsql_test 15.1 { + CREATE VIRTUAL TABLE ft USING fts5(a, b UNINDEXED, c, locale=1, tokenize=tcl); +} + +do_execsql_test 15.2 { + INSERT INTO ft VALUES('one', 'two', 'three'); + INSERT INTO ft VALUES('one', 'two', fts5_locale('loc', 'three')); +} + +do_execsql_test 15.3 { + SELECT c2, l2 FROM ft_content +} {three {} three loc} + +do_execsql_test 15.4 { + SELECT c, fts5_columnlocale(ft, 2) FROM ft +} {three {} three loc} + finish_test diff --git a/ext/fts5/test/fts5simple.test b/ext/fts5/test/fts5simple.test index ad59bf0d9e..8ca5d334fe 100644 --- a/ext/fts5/test/fts5simple.test +++ b/ext/fts5/test/fts5simple.test @@ -371,6 +371,7 @@ foreach_detail_mode $testprefix { CREATE VIRTUAL TABLE x3 USING fts5(x, detail=%DETAIL%); INSERT INTO x3 VALUES('a b c d e f'); } + breakpoint do_execsql_test 16.1 { SELECT fts5_test_poslist(x3) FROM x3('(a NOT b) OR c'); } {2.0.2} diff --git a/manifest b/manifest index df046ec751..86c0ee142c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sOOM-handling\sproblem\saffecting\slocale=1\sfts5\stables. -D 2024-09-09T19:12:57.172 +C Alternative\simplementation\sof\sfts5\slocale=1\sfeature\sthat\sallows\sblobs\sto\sbe\sstored\sin\sindexed\scolumns\sof\sfts5\slocale=1\stables. +D 2024-09-10T16:19:31.078 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -93,15 +93,15 @@ F ext/fts3/unicode/mkunicode.tcl 63db9624ccf70d4887836c320eda93ab552f21008f3be7e F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb F ext/fts5/extract_api_docs.tcl 009cf59c77afa86d137b0cca3e3b1a5efbe2264faa2df233f9a7aa8563926d15 F ext/fts5/fts5.h efaaac0df3d3bc740383044c144b582f47921aafa21d7b10eb98f42c24c740b0 -F ext/fts5/fts5Int.h 7ab1d838adc4f22fdad5e1ba19182d6899ebded1d3ecadbe995322b0f0de7b9f +F ext/fts5/fts5Int.h 93aba03ca417f403b07b2ab6f50aa0e0c1b8b031917a9026b81520e7047a168e F ext/fts5/fts5_aux.c 65a0468dd177d6093aa9ae1622e6d86b0136b8d267c62c0ad6493ad1e9a3d759 F ext/fts5/fts5_buffer.c 0eec58bff585f1a44ea9147eae5da2447292080ea435957f7488c70673cb6f09 -F ext/fts5/fts5_config.c 353d2a0d12678cae6ab5b9ce54aed8dac0825667b69248b5a4ed81cbefc109ea +F ext/fts5/fts5_config.c da21548ddbc1a457cb42545f527065221ede8ada6a734891b8c34317a7a9506b F ext/fts5/fts5_expr.c 9a56f53700d1860f0ee2f373c2b9074eaf2a7aa0637d0e27a6476de26a3fee33 F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 F ext/fts5/fts5_index.c 571483823193f09439356741669aa8c81da838ae6f5e1bfa7517f7ee2fb3addd -F ext/fts5/fts5_main.c 9124eba418eb0c608c1454c4ad08a5f1ac21a4748c36a44828a0a7a1b32ef896 -F ext/fts5/fts5_storage.c 42cde97eb7d8506a8d2c7ea80b292fc3017b1f5469e1acb0035a69c345e6cf71 +F ext/fts5/fts5_main.c 68b8fd96b0798b4822c081049bb37598f9f3fc34c6447c74cc162f86b18d2ded +F ext/fts5/fts5_storage.c 3332497823c3d171cf56379f2bd8c971ce15a19aadacff961106462022c92470 F ext/fts5/fts5_tcl.c 4db9258a7882c5eac0da4433042132aaf15b87dd1e1636c7a6ca203abd2c8bfe F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee F ext/fts5/fts5_test_tok.c 3cb0a9b508b30d17ef025ccddd26ae3dc8ddffbe76c057616e59a9aa85d36f3b @@ -132,7 +132,7 @@ F ext/fts5/test/fts5auxdata.test 372549088ff792655f73e62b9dfaf4863ce74f5e604c06c F ext/fts5/test/fts5bigid.test 2860854c2561a57594192b00c33a29f91cb85e25f3d6c03b5c2b8f62708f39dd F ext/fts5/test/fts5bigpl.test 8f09858aab866c33593560e6480b2b6975ae7ff29ca32ad7b77e2da61402f8ef F ext/fts5/test/fts5bigtok.test 541119e616c637caea925a8c028c37c2c29e94383e00aa2f9198d530724b6e36 -F ext/fts5/test/fts5blob.test caa33369e93e99ff494cd1103506ae34c5afbc0bcc369ed5e58e135144e33689 +F ext/fts5/test/fts5blob.test 9644a5f917306690e08c5f89a470a3f2489376eaa52026eeca3209d149d6af74 F ext/fts5/test/fts5cat.test bf67dd335f964482ee658287521b81e2b88697b45eb7f73933e15f198ed447cb F ext/fts5/test/fts5circref.test f880dfd0d99f6fb73b88ccacb0927d18e833672fd906cc47d6b4e529419eaa62 F ext/fts5/test/fts5colset.test 544f4998cdbfe06a3123887fc0221612e8aa8192cdaff152872f1aadb10e6897 @@ -178,7 +178,7 @@ F ext/fts5/test/fts5faultE.test 844586ce71dab4be85bb86880e87b624d089f851654cd22e F ext/fts5/test/fts5faultF.test 4abef99f86e99d9f0c6460dd68c586a766b6b9f1f660ada55bf2e8266bd1bbc1 F ext/fts5/test/fts5faultG.test 0544411ffcb3e19b42866f757a8a5e0fb8fef3a62c06f61d14deebc571bb7ea9 F ext/fts5/test/fts5faultH.test 2b2b5b8cb1b3fd7679f488c06e22af44107fbc6137eaf45b3e771dc7b149312d -F ext/fts5/test/fts5faultI.test a1496d6d72b864102f95f9a616a0f583320310a6fb7a463a37c88dfb40d68ae5 +F ext/fts5/test/fts5faultI.test 0706b307b208638554c9e65b4091e1c0dd8c92941535089a301df454ff2c56f4 F ext/fts5/test/fts5first.test bfd685b96905bf541d99d8644e0a7219d1d833455a08ab64e344071a613b6ba9 F ext/fts5/test/fts5full.test 97d263c1072f4a560929cca31e70f65d2ae232610e17e6affcf7e979df59547b F ext/fts5/test/fts5fuzz1.test 238d8c45f3b81342aa384de3e581ff2fa330bf922a7b69e484bbc06051a1080e @@ -189,7 +189,7 @@ F ext/fts5/test/fts5interrupt.test 20d04204d3e341b104c0c24a41596b6393a3a81eba104 F ext/fts5/test/fts5lastrowid.test f36298a1fb9f988bde060a274a7ce638faa9c38a31400f8d2d27ea9373e0c4a1 F ext/fts5/test/fts5leftjoin.test c0b4cafb9661379e576dc4405c0891d8fcc2782680740513c4d1fc114b43d4ad F ext/fts5/test/fts5limits.test 8ab67cf5d311c124b6ceb0062d0297767176df4572d955fce79fa43004dff01c -F ext/fts5/test/fts5locale.test 58ce0515c4f49cbb9905e3711168050d58fc184daf885c9ef7483e20aab63e5a +F ext/fts5/test/fts5locale.test 83ba7ee12628b540d3098f39c39c1de0c0440eddff8f7512c8c698d0c4a3ae3c F ext/fts5/test/fts5matchinfo.test 877520582feb86bbfd95ab780099bcba4526f18ac75ee34979144cf86ba3a5a3 F ext/fts5/test/fts5merge.test 2654df0bcdb2d117c2d38b6aeb0168061be01c643f9e9194b36c43a2970e8082 F ext/fts5/test/fts5merge2.test 3ebad1a59d6ad3fb66eff6523a09e95dc6367cbefb3cd73196801dea0425c8e2 @@ -227,7 +227,7 @@ F ext/fts5/test/fts5secure6.test 74bf04733cc523bccca519bb03d3b4e2ed6f6e3db7c59bf F ext/fts5/test/fts5secure7.test fd03d0868d64340a1db8615b02e5508fea409de13910114e4f19eaefc120777a F ext/fts5/test/fts5secure8.test 808ade9d172ed07b24b85c57dd53b6d2b1aba018b4e634d267ce572221de80e0 F ext/fts5/test/fts5securefault.test c34a28c7cd2f31a8b8907563889e1329a97da975c08df2d951422bcef8e2ebc5 -F ext/fts5/test/fts5simple.test 302cdb4f8a3350b091f4f1bccd82d05610428657f6f9e81c17703ba48267ec40 +F ext/fts5/test/fts5simple.test 5a81d494f269baa176a0acefeae1e4a239d4933700d8f415eff16da214fd11d3 F ext/fts5/test/fts5simple2.test d10d963a357b8ec77b99032e4c816459b4dbdb1f6eee25eada7ef3ed245cb2dc F ext/fts5/test/fts5simple3.test 146ec3dc8f5763d6212641c9f0a2f1cba41679353d2add7b963beceb115dc7f4 F ext/fts5/test/fts5synonym.test becc8cea6cfc958a50b30c572c68cbfdf7455971d0fe988202ce67638d2c6cf6 @@ -2212,8 +2212,11 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P fe0d67e72d4228661c021f227bfc0d5ddb1b726db0f36c7221ead8dd8bd1dc73 -R d215b5e5c87263682db579b6ad049bae +P d8103684f660ff9b3186d0f89afb113ca580bd16f0bf413ed8a9434236b54426 +R 4bff6e3adb2a3e18f21f459a8e5e307a +T *branch * fts5-locale-alternate +T *sym-fts5-locale-alternate * +T -sym-trunk * U dan -Z 3d429b85e0f7dde1fd63e360d413140d +Z 708e8bcf278c2f81c8bdb896ae9792a2 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 739aa6819c..b9473faf29 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d8103684f660ff9b3186d0f89afb113ca580bd16f0bf413ed8a9434236b54426 +55c5c119a0a77fac2c9f46d718ef78c0f33ed3520e10c240cf5bf1801e0586ee From 923423c16f8715ccf0312923b2837c6138ddc174 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 10 Sep 2024 16:40:08 +0000 Subject: [PATCH 32/36] Fix a test in fts3corrupt4.test that would fail if SQLITE_ENABLE_FTS5 was not defined. FossilOrigin-Name: 437849c80851da842b5c4fd37d5c147f821abc541e9b4d6f9000c12983548844 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/fts3corrupt4.test | 4 +++- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index df046ec751..cfcc58cbf1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sOOM-handling\sproblem\saffecting\slocale=1\sfts5\stables. -D 2024-09-09T19:12:57.172 +C Fix\sa\stest\sin\sfts3corrupt4.test\sthat\swould\sfail\sif\sSQLITE_ENABLE_FTS5\swas\snot\sdefined. +D 2024-09-10T16:40:08.960 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1184,7 +1184,7 @@ F test/fts3conf.test c9cd45433b6787d48a43e84949aa2eb8b3b3d242bac7276731c1476290d F test/fts3corrupt.test 6732477c5ace050c5758a40a8b5706c8c0cccd416b9c558e0e15224805a40e57 F test/fts3corrupt2.test e318f0676e5e78d5a4b702637e2bb25265954c08a1b1e4aaf93c7880bb0c67d0 F test/fts3corrupt3.test 0d5b69a0998b4adf868cc301fc78f3d0707745f1d984ce044c205cdb764b491f -F test/fts3corrupt4.test 294684add5f235ea8a77a350b66eb74a80ac8ecee6ac38c07885348f5fb2e233 +F test/fts3corrupt4.test a451033ae31db9c5979a7612dee80fb4f221db104a2eeeabd1c9adcc8e8fe95a F test/fts3corrupt5.test 0549f85ec4bd22e992f645f13c59b99d652f2f5e643dac75568bfd23a6db7ed5 F test/fts3corrupt6.test f417c910254f32c0bc9ead7affa991a1d5aec35b3b32a183ffb05eea78289525 F test/fts3cov.test 7eacdbefd756cfa4dc2241974e3db2834e9b372ca215880e00032222f32194cf @@ -2212,8 +2212,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P fe0d67e72d4228661c021f227bfc0d5ddb1b726db0f36c7221ead8dd8bd1dc73 -R d215b5e5c87263682db579b6ad049bae +P d8103684f660ff9b3186d0f89afb113ca580bd16f0bf413ed8a9434236b54426 +R aef238013a1ce5e62ad259c3eba70261 U dan -Z 3d429b85e0f7dde1fd63e360d413140d +Z ba098087a8d44fd7d814378cea692b59 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 739aa6819c..a7511290dc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d8103684f660ff9b3186d0f89afb113ca580bd16f0bf413ed8a9434236b54426 +437849c80851da842b5c4fd37d5c147f821abc541e9b4d6f9000c12983548844 diff --git a/test/fts3corrupt4.test b/test/fts3corrupt4.test index 7a9ab3fbd0..d09060d6cd 100644 --- a/test/fts3corrupt4.test +++ b/test/fts3corrupt4.test @@ -4376,6 +4376,8 @@ do_test 25.0 { | end crash-dde9e76ed8ab2d.db }]} {} +reset_prng_state + do_catchsql_test 25.1 { PRAGMA writable_schema = 1; WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x%1 FROM c WHERE x<599237) @@ -4404,7 +4406,7 @@ do_catchsql_test 25.5 { do_catchsql_test 25.6 { INSERT INTO t1(t1) SELECT x FROM t2; INSERT INTO t1(t1) SELECT x FROM t2; -} {0 {}} +} {1 {database disk image is malformed}} #------------------------------------------------------------------------- reset_db From 27119c56bd9de173ce7f896b791aaeec8c34b6b5 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 10 Sep 2024 18:38:47 +0000 Subject: [PATCH 33/36] Remove a stray "breakpoint" from a test script. FossilOrigin-Name: 7d87a27a01311153ddee122cedecedc3bcc331618dc2ab1da397a3b257dc21cf --- ext/fts5/fts5_main.c | 25 +++++++++++++++++++++++-- ext/fts5/test/fts5simple.test | 1 - manifest | 17 +++++++---------- manifest.uuid | 2 +- 4 files changed, 31 insertions(+), 14 deletions(-) diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index b4922a8b6f..e57902b6b7 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -1301,7 +1301,17 @@ int sqlite3Fts5IsLocaleValue(Fts5Config *pConfig, sqlite3_value *pVal){ } /* -** Value pVal is guaranteed to be an fts5_locale() value. +** Value pVal is guaranteed to be an fts5_locale() value, according to +** sqlite3Fts5IsLocaleValue(). This function extracts the text and locale +** from the value and returns them separately. +** +** If successful, SQLITE_OK is returned and (*ppText) and (*ppLoc) set +** to point to buffers containing the text and locale, as utf-8, +** respectively. In this case output parameters (*pnText) and (*pnLoc) are +** set to the sizes in bytes of these two buffers. +** +** Or, if an error occurs, then an SQLite error code is returned. The final +** value of the four output parameters is undefined in this case. */ int sqlite3Fts5DecodeLocaleValue( sqlite3_value *pVal, @@ -2105,7 +2115,18 @@ static int fts5ApiPhraseSize(Fts5Context *pCtx, int iPhrase){ } /* -** Argument pStmt is an SQL statement of the type used by Fts5Cursor. +** Argument pStmt is an SQL statement of the type used by Fts5Cursor. This +** function extracts the text value of column iCol of the current row. +** Additionally, if there is an associated locale, it invokes +** sqlite3Fts5SetLocale() to configure the tokenizer. In all cases the caller +** should invoke sqlite3Fts5ClearLocale() to clear the locale at some point +** after this function returns. +** +** If successful, (*ppText) is set to point to a buffer containing the text +** value as utf-8 and SQLITE_OK returned. (*pnText) is set to the size of that +** buffer in bytes. It is not guaranteed to be nul-terminated. If an error +** occurs, an SQLite error code is returned. The final values of the two +** output parameters are undefined in this case. */ static int fts5TextFromStmt( Fts5Config *pConfig, diff --git a/ext/fts5/test/fts5simple.test b/ext/fts5/test/fts5simple.test index 8ca5d334fe..ad59bf0d9e 100644 --- a/ext/fts5/test/fts5simple.test +++ b/ext/fts5/test/fts5simple.test @@ -371,7 +371,6 @@ foreach_detail_mode $testprefix { CREATE VIRTUAL TABLE x3 USING fts5(x, detail=%DETAIL%); INSERT INTO x3 VALUES('a b c d e f'); } - breakpoint do_execsql_test 16.1 { SELECT fts5_test_poslist(x3) FROM x3('(a NOT b) OR c'); } {2.0.2} diff --git a/manifest b/manifest index 86c0ee142c..a62b503f19 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Alternative\simplementation\sof\sfts5\slocale=1\sfeature\sthat\sallows\sblobs\sto\sbe\sstored\sin\sindexed\scolumns\sof\sfts5\slocale=1\stables. -D 2024-09-10T16:19:31.078 +C Remove\sa\sstray\s"breakpoint"\sfrom\sa\stest\sscript. +D 2024-09-10T18:38:47.977 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -100,7 +100,7 @@ F ext/fts5/fts5_config.c da21548ddbc1a457cb42545f527065221ede8ada6a734891b8c3431 F ext/fts5/fts5_expr.c 9a56f53700d1860f0ee2f373c2b9074eaf2a7aa0637d0e27a6476de26a3fee33 F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 F ext/fts5/fts5_index.c 571483823193f09439356741669aa8c81da838ae6f5e1bfa7517f7ee2fb3addd -F ext/fts5/fts5_main.c 68b8fd96b0798b4822c081049bb37598f9f3fc34c6447c74cc162f86b18d2ded +F ext/fts5/fts5_main.c 774a4e6eddfa98e9f050f781c095e2dd49c8b69e82527cb9ccb8cabe78016052 F ext/fts5/fts5_storage.c 3332497823c3d171cf56379f2bd8c971ce15a19aadacff961106462022c92470 F ext/fts5/fts5_tcl.c 4db9258a7882c5eac0da4433042132aaf15b87dd1e1636c7a6ca203abd2c8bfe F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee @@ -227,7 +227,7 @@ F ext/fts5/test/fts5secure6.test 74bf04733cc523bccca519bb03d3b4e2ed6f6e3db7c59bf F ext/fts5/test/fts5secure7.test fd03d0868d64340a1db8615b02e5508fea409de13910114e4f19eaefc120777a F ext/fts5/test/fts5secure8.test 808ade9d172ed07b24b85c57dd53b6d2b1aba018b4e634d267ce572221de80e0 F ext/fts5/test/fts5securefault.test c34a28c7cd2f31a8b8907563889e1329a97da975c08df2d951422bcef8e2ebc5 -F ext/fts5/test/fts5simple.test 5a81d494f269baa176a0acefeae1e4a239d4933700d8f415eff16da214fd11d3 +F ext/fts5/test/fts5simple.test 302cdb4f8a3350b091f4f1bccd82d05610428657f6f9e81c17703ba48267ec40 F ext/fts5/test/fts5simple2.test d10d963a357b8ec77b99032e4c816459b4dbdb1f6eee25eada7ef3ed245cb2dc F ext/fts5/test/fts5simple3.test 146ec3dc8f5763d6212641c9f0a2f1cba41679353d2add7b963beceb115dc7f4 F ext/fts5/test/fts5synonym.test becc8cea6cfc958a50b30c572c68cbfdf7455971d0fe988202ce67638d2c6cf6 @@ -2212,11 +2212,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d8103684f660ff9b3186d0f89afb113ca580bd16f0bf413ed8a9434236b54426 -R 4bff6e3adb2a3e18f21f459a8e5e307a -T *branch * fts5-locale-alternate -T *sym-fts5-locale-alternate * -T -sym-trunk * +P 55c5c119a0a77fac2c9f46d718ef78c0f33ed3520e10c240cf5bf1801e0586ee +R 330e5898e93a7e63c9e09af30983e29e U dan -Z 708e8bcf278c2f81c8bdb896ae9792a2 +Z 71835541a0fd0d777ae33c979ec9a5ae # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b9473faf29..ab537b0361 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -55c5c119a0a77fac2c9f46d718ef78c0f33ed3520e10c240cf5bf1801e0586ee +7d87a27a01311153ddee122cedecedc3bcc331618dc2ab1da397a3b257dc21cf From 2a4a4ec408588541df6a4caac2f52facf890a3b3 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 11 Sep 2024 12:17:26 +0000 Subject: [PATCH 34/36] Disable the debug-use-only functions sqlite3_mutex_held() and sqlite3_mutex_notheld() when compiling with TSAN in as much as those routines cause TSAN to complain. Response to [https://issues.chromium.org/issues/41427446]. FossilOrigin-Name: db702dd78500a0839b0b2810a580d3634df49275470787b170973a86b73826d3 --- manifest | 15 +++++++-------- manifest.uuid | 2 +- src/mutex.c | 15 ++++++++++++++- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index b71c9002b3..8f9468bb4d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Alternative\simplementation\sof\sfts5\slocale=1\sfeature\sthat\sallows\sblobs\sto\sbe\sstored\sin\sindexed\scolumns\sof\sfts5\slocale=1\stables. -D 2024-09-10T20:32:36.789 +C Disable\sthe\sdebug-use-only\sfunctions\ssqlite3_mutex_held()\sand\s\nsqlite3_mutex_notheld()\swhen\scompiling\swith\sTSAN\sin\sas\smuch\sas\sthose\nroutines\scause\sTSAN\sto\scomplain.\s\sResponse\sto\n[https://issues.chromium.org/issues/41427446]. +D 2024-09-11T12:17:26.475 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -740,7 +740,7 @@ F src/mem5.c b7da5c10a726aacacc9ad7cdcb0667deec643e117591cc69cf9b4b9e7f3e96ff F src/memdb.c 16679def118b5fd75292a253166d3feba3ec9c6189205bf209643ecdb2174ecc F src/memjournal.c c283c6c95d940eb9dc70f1863eef3ee40382dbd35e5a1108026e7817c206e8a0 F src/msvc.h 80b35f95d93bf996ccb3e498535255f2ef1118c78764719a7cd15ab4106ccac9 -F src/mutex.c 1b4c7e5e3621b510e0c18397210be27cd54c8084141144fbbafd003fde948e88 +F src/mutex.c 06bcd9c3dbf2d9b21fcd182606c00fafb9bfe0287983c8e17acd13d2c81a2fa9 F src/mutex.h a7b2293c48db5f27007c3bdb21d438873637d12658f5a0bf8ad025bb96803c4a F src/mutex_noop.c 9d4309c075ba9cc7249e19412d3d62f7f94839c4 F src/mutex_unix.c f7ee5a2061a4c11815a2bf4fc0e2bfa6fb8d9dc89390eb613ca0cec32fc9a3d1 @@ -2212,9 +2212,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 437849c80851da842b5c4fd37d5c147f821abc541e9b4d6f9000c12983548844 7d87a27a01311153ddee122cedecedc3bcc331618dc2ab1da397a3b257dc21cf -R cbbfe15fbc887a71ab881832205b2d5a -T +closed 7d87a27a01311153ddee122cedecedc3bcc331618dc2ab1da397a3b257dc21cf -U dan -Z f39f2a1cdc64ffa8dc528a76af023d0e +P 198305de92ebba7045d8ec7d2de98511f3b00924f808a3811f061dca47b01ec7 +R 59c068ea0c953cf7fac1b84818bf6242 +U drh +Z 90039802f2dd6a80594d6a6b132d7045 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index e0567f9ee8..c3b2b6cd9a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -198305de92ebba7045d8ec7d2de98511f3b00924f808a3811f061dca47b01ec7 +db702dd78500a0839b0b2810a580d3634df49275470787b170973a86b73826d3 diff --git a/src/mutex.c b/src/mutex.c index 381ffbdfd5..62e09cb4fa 100644 --- a/src/mutex.c +++ b/src/mutex.c @@ -347,15 +347,28 @@ void sqlite3_mutex_leave(sqlite3_mutex *p){ /* ** The sqlite3_mutex_held() and sqlite3_mutex_notheld() routine are ** intended for use inside assert() statements. +** +** Because these routines raise false-positive alerts in TSAN, disable +** them (make them always return 1) when compiling with TSAN. */ int sqlite3_mutex_held(sqlite3_mutex *p){ +# if defined(__has_feature) +# if __has_feature(thread_sanitizer) + p = 0; +# endif +# endif assert( p==0 || sqlite3GlobalConfig.mutex.xMutexHeld ); return p==0 || sqlite3GlobalConfig.mutex.xMutexHeld(p); } int sqlite3_mutex_notheld(sqlite3_mutex *p){ +# if defined(__has_feature) +# if __has_feature(thread_sanitizer) + p = 0; +# endif +# endif assert( p==0 || sqlite3GlobalConfig.mutex.xMutexNotheld ); return p==0 || sqlite3GlobalConfig.mutex.xMutexNotheld(p); } -#endif +#endif /* NDEBUG */ #endif /* !defined(SQLITE_MUTEX_OMIT) */ From 74c8e234cd2830467849d5329be7279d3fb3f760 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 12 Sep 2024 21:58:31 +0000 Subject: [PATCH 35/36] Enhancement to the "showdb" utility such that the "ptrmap" command shows PTRMAP entries that extend off the end of the database, as long as they appear to be well-formatted. FossilOrigin-Name: a9f95fe5ce90ab9864165e603f3a34013c3c98d03f1db689996f4a32086e2ed6 --- manifest | 12 ++++++------ manifest.uuid | 2 +- tool/showdb.c | 8 ++++++-- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 8f9468bb4d..edc637fa6a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Disable\sthe\sdebug-use-only\sfunctions\ssqlite3_mutex_held()\sand\s\nsqlite3_mutex_notheld()\swhen\scompiling\swith\sTSAN\sin\sas\smuch\sas\sthose\nroutines\scause\sTSAN\sto\scomplain.\s\sResponse\sto\n[https://issues.chromium.org/issues/41427446]. -D 2024-09-11T12:17:26.475 +C Enhancement\sto\sthe\s"showdb"\sutility\ssuch\sthat\sthe\s"ptrmap"\scommand\sshows\nPTRMAP\sentries\sthat\sextend\soff\sthe\send\sof\sthe\sdatabase,\sas\slong\sas\sthey\nappear\sto\sbe\swell-formatted. +D 2024-09-12T21:58:31.168 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -2157,7 +2157,7 @@ F tool/replace.tcl 511c61acfe563dfb58675efb4628bb158a13d48ff8322123ac447e9d25a82 F tool/restore_jrnl.tcl 1079ecba47cc82fa82115b81c1f68097ab1f956f357ee8da5fc4b2589af6bd98 F tool/rollback-test.c 9fc98427d1e23e84429d7e6d07d9094fbdec65a5 F tool/run-speed-test.sh f95d19fd669b68c4c38b6b475242841d47c66076 -F tool/showdb.c 0f74b54cc67076c76cba9b2b7f54d3e05b78d130c70ffc394eb84c5b41bab017 +F tool/showdb.c 14215f764f3e9adcbf49f8ed730a520f27283a1d875c2feb78d9c687c772b66e F tool/showjournal.c 5bad7ae8784a43d2b270d953060423b8bd480818 F tool/showlocks.c 9cc5e66d4ebbf2d194f39db2527ece92077e86ae627ddd233ee48e16e8142564 F tool/showshm.c a0ab6ec32dd1f11218ca2a4018f8fb875b59414801ab8ceed8b2e69b7b45a809 @@ -2212,8 +2212,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 198305de92ebba7045d8ec7d2de98511f3b00924f808a3811f061dca47b01ec7 -R 59c068ea0c953cf7fac1b84818bf6242 +P db702dd78500a0839b0b2810a580d3634df49275470787b170973a86b73826d3 +R 96b15683ae71ad8973e2164a9c586273 U drh -Z 90039802f2dd6a80594d6a6b132d7045 +Z e7a5988d3eebb0fdadfa92bbc6a99e53 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index c3b2b6cd9a..ecf65c614a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -db702dd78500a0839b0b2810a580d3634df49275470787b170973a86b73826d3 +a9f95fe5ce90ab9864165e603f3a34013c3c98d03f1db689996f4a32086e2ed6 diff --git a/tool/showdb.c b/tool/showdb.c index 1b80c7f170..9a38658d23 100644 --- a/tool/showdb.c +++ b/tool/showdb.c @@ -1084,17 +1084,21 @@ static void ptrmap_coverage_report(const char *zDbName){ printf("%5llu: PTRMAP page covering %llu..%llu\n", pgno, pgno+1, pgno+perPage); a = fileRead((pgno-1)*g.pagesize, usable); - for(i=0; i+5<=usable && pgno+1+i/5<=g.mxPage; i+=5){ + for(i=0; i+5<=usable; i+=5){ const char *zType = "???"; u32 iFrom = decodeInt32(&a[i+1]); + const char *zExtra = pgno+1+i/5>g.mxPage ? " (off end of DB)" : ""; switch( a[i] ){ case 1: zType = "b-tree root page"; break; case 2: zType = "freelist page"; break; case 3: zType = "first page of overflow"; break; case 4: zType = "later page of overflow"; break; case 5: zType = "b-tree non-root page"; break; + default: zType = 0; break; + } + if( zType ){ + printf("%5llu: %s, parent=%u%s\n", pgno+1+i/5, zType, iFrom, zExtra); } - printf("%5llu: %s, parent=%u\n", pgno+1+i/5, zType, iFrom); } sqlite3_free(a); } From e92f8e565b5fa6da9a0564228505d7efa0f486ab Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 13 Sep 2024 11:14:10 +0000 Subject: [PATCH 36/36] Further enhancement to PTRMAP display in showdb: Show the details of invalid entries that are within the range of the database file. Continue to ignore invalid entries beyond the end of the database file. FossilOrigin-Name: 4cad385b90eaca2d90e3375e473472145af4134160b81097a8535d06638c2e4a --- manifest | 12 ++++++------ manifest.uuid | 2 +- tool/showdb.c | 11 +++++++++-- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index edc637fa6a..87fe20a989 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhancement\sto\sthe\s"showdb"\sutility\ssuch\sthat\sthe\s"ptrmap"\scommand\sshows\nPTRMAP\sentries\sthat\sextend\soff\sthe\send\sof\sthe\sdatabase,\sas\slong\sas\sthey\nappear\sto\sbe\swell-formatted. -D 2024-09-12T21:58:31.168 +C Further\senhancement\sto\sPTRMAP\sdisplay\sin\sshowdb:\s\sShow\sthe\sdetails\sof\sinvalid\nentries\sthat\sare\swithin\sthe\srange\sof\sthe\sdatabase\sfile.\s\sContinue\sto\signore\ninvalid\sentries\sbeyond\sthe\send\sof\sthe\sdatabase\sfile. +D 2024-09-13T11:14:10.057 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -2157,7 +2157,7 @@ F tool/replace.tcl 511c61acfe563dfb58675efb4628bb158a13d48ff8322123ac447e9d25a82 F tool/restore_jrnl.tcl 1079ecba47cc82fa82115b81c1f68097ab1f956f357ee8da5fc4b2589af6bd98 F tool/rollback-test.c 9fc98427d1e23e84429d7e6d07d9094fbdec65a5 F tool/run-speed-test.sh f95d19fd669b68c4c38b6b475242841d47c66076 -F tool/showdb.c 14215f764f3e9adcbf49f8ed730a520f27283a1d875c2feb78d9c687c772b66e +F tool/showdb.c 81b04bfaa9a63665f75945947323aa68b820570aa156b1574f440fc8276092c6 F tool/showjournal.c 5bad7ae8784a43d2b270d953060423b8bd480818 F tool/showlocks.c 9cc5e66d4ebbf2d194f39db2527ece92077e86ae627ddd233ee48e16e8142564 F tool/showshm.c a0ab6ec32dd1f11218ca2a4018f8fb875b59414801ab8ceed8b2e69b7b45a809 @@ -2212,8 +2212,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P db702dd78500a0839b0b2810a580d3634df49275470787b170973a86b73826d3 -R 96b15683ae71ad8973e2164a9c586273 +P a9f95fe5ce90ab9864165e603f3a34013c3c98d03f1db689996f4a32086e2ed6 +R 5fde967f0ed3ce1cc53a6d5bcb8024a9 U drh -Z e7a5988d3eebb0fdadfa92bbc6a99e53 +Z 8b78883255183c9952b265920384a36e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index ecf65c614a..cf7caf5259 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a9f95fe5ce90ab9864165e603f3a34013c3c98d03f1db689996f4a32086e2ed6 +4cad385b90eaca2d90e3375e473472145af4134160b81097a8535d06638c2e4a diff --git a/tool/showdb.c b/tool/showdb.c index 9a38658d23..12c2e271b7 100644 --- a/tool/showdb.c +++ b/tool/showdb.c @@ -1085,7 +1085,7 @@ static void ptrmap_coverage_report(const char *zDbName){ pgno+1, pgno+perPage); a = fileRead((pgno-1)*g.pagesize, usable); for(i=0; i+5<=usable; i+=5){ - const char *zType = "???"; + const char *zType; u32 iFrom = decodeInt32(&a[i+1]); const char *zExtra = pgno+1+i/5>g.mxPage ? " (off end of DB)" : ""; switch( a[i] ){ @@ -1094,7 +1094,14 @@ static void ptrmap_coverage_report(const char *zDbName){ case 3: zType = "first page of overflow"; break; case 4: zType = "later page of overflow"; break; case 5: zType = "b-tree non-root page"; break; - default: zType = 0; break; + default: { + if( zExtra[0]==0 ){ + printf("%5llu: invalid (0x%02x), parent=%u\n", + pgno+1+i/5, a[i], iFrom); + } + zType = 0; + break; + } } if( zType ){ printf("%5llu: %s, parent=%u%s\n", pgno+1+i/5, zType, iFrom, zExtra);